Skip to content

feat: kubelet proxy #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lylingzhen merged 4 commits into from
Aug 20, 2025
Merged

feat: kubelet proxy #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0713922
feat: kubelet proxy
BetaCat0 Aug 7, 2025
b79c8b0
chore: update golangci-lint configuration to skip test files
BetaCat0 Aug 19, 2025
d4b913b
fix: lint issues
BetaCat0 Aug 19, 2025
405833a
fix: ai issues
BetaCat0 Aug 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .golangci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,7 @@ formatters:
enable:
- gofumpt
- goimports


run:
tests: false
4 changes: 2 additions & 2 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Build the manager binary
FROM golang:1.22 as builder
FROM golang:1.24 as builder
ARG TARGETOS
ARG TARGETARCH

Expand Down Expand Up @@ -36,4 +36,4 @@ EXPOSE 9090
EXPOSE 8080
EXPOSE 7777

ENTRYPOINT ["./module_controller"]
ENTRYPOINT ["./module_controller"]
82 changes: 72 additions & 10 deletions cmd/module-controller/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,37 +18,46 @@ import (
"context"
"errors"
"fmt"
"github.com/koupleless/module_controller/common/zaplogger"
"github.com/koupleless/module_controller/report_server"
"github.com/koupleless/virtual-kubelet/vnode_controller"
"os"
"os/signal"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"strconv"
"syscall"

"github.com/google/uuid"
"github.com/koupleless/module_controller/common/model"
"github.com/koupleless/module_controller/common/zaplogger"
"github.com/koupleless/module_controller/controller/module_deployment_controller"
"github.com/koupleless/module_controller/module_tunnels/koupleless_http_tunnel"
"github.com/koupleless/module_controller/module_tunnels/koupleless_mqtt_tunnel"
"github.com/koupleless/module_controller/report_server"
"github.com/koupleless/module_controller/staging/kubelet_proxy"
"github.com/koupleless/virtual-kubelet/common/tracker"
"github.com/koupleless/virtual-kubelet/common/utils"
vkModel "github.com/koupleless/virtual-kubelet/model"
"github.com/koupleless/virtual-kubelet/tunnel"
"github.com/koupleless/virtual-kubelet/vnode_controller"
"github.com/sirupsen/logrus"
"github.com/virtual-kubelet/virtual-kubelet/log"
logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
"github.com/virtual-kubelet/virtual-kubelet/trace"
"github.com/virtual-kubelet/virtual-kubelet/trace/opencensus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
)

const (
certFilePath = "/etc/virtual-kubelet/tls/tls.crt"
keyFilePath = "/etc/virtual-kubelet/tls/tls.key"
DefaultKubeletHttpListenAddr = "10250" // Default listen address for the HTTP server
)

// Main function for the module controller
// Responsibilities:
// 1. Sets up signal handling for graceful shutdown
Expand All @@ -59,6 +68,7 @@ import (
// 6. Creates and configures the VNode controller
// 7. Optionally creates module deployment controller
// 8. Starts all tunnels and the manager
// 9. Starts the kubelet proxy server(if enabled)

func main() {
ctx, cancel := context.WithCancel(context.Background())
Expand All @@ -75,27 +85,29 @@ func main() {
trace.T = opencensus.Adapter{}

// Get configuration from environment variables
clientID := utils.GetEnv("CLIENT_ID", uuid.New().String())
clientID := utils.GetEnv(model.EnvKeyOfClientID, uuid.New().String())
env := utils.GetEnv("ENV", "dev")

zlogger := zaplogger.GetLogger()
ctx = zaplogger.WithLogger(ctx, zlogger)

// Parse configuration with defaults
isCluster := utils.GetEnv("IS_CLUSTER", "") == "true"
workloadMaxLevel, err := strconv.Atoi(utils.GetEnv("WORKLOAD_MAX_LEVEL", "3"))
isCluster := utils.GetEnv(model.EnvKeyOfClusterModeEnabled, "") == "true"
workloadMaxLevel, err := strconv.Atoi(utils.GetEnv(model.EnvKeyOfWorkloadMaxLevel, "3"))

if err != nil {
zlogger.Error(err, "failed to parse WORKLOAD_MAX_LEVEL, will be set to 3 default")
workloadMaxLevel = 3
}

vnodeWorkerNum, err := strconv.Atoi(utils.GetEnv("VNODE_WORKER_NUM", "8"))
vnodeWorkerNum, err := strconv.Atoi(utils.GetEnv(model.EnvKeyOfVNodeWorkerNum, "8"))
if err != nil {
zlogger.Error(err, "failed to parse VNODE_WORKER_NUM, will be set to 8 default")
vnodeWorkerNum = 8
}

deployNamespace := utils.GetEnv(model.EnvKeyOfNamespace, "default")

// Initialize controller manager
kubeConfig := config.GetConfigOrDie()
// TODO: should support to set from parameter
Expand All @@ -111,14 +123,25 @@ func main() {
BindAddress: ":9090",
},
})

if err != nil {
zlogger.Error(err, "unable to set up overall controller manager")
os.Exit(1)
}

tracker.SetTracker(&tracker.DefaultTracker{})

k8sClientSet := kubernetes.NewForConfigOrDie(kubeConfig)

var moduleControllerServiceIP string
var kubeletProxyEnabled bool
if os.Getenv(model.EnvKeyOfKubeletProxyEnabled) == "true" {
moduleControllerServiceIP, err = lookupProxyServiceIP(ctx, deployNamespace, k8sClientSet)
if err != nil {
log.G(ctx).Fatalf("Failed to lookup kubelet proxy service IP: %v", err)
}
kubeletProxyEnabled = true
}

Comment on lines +136 to +144
Copy link

@coderabbitai coderabbitai bot Aug 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent

🧩 Analysis chain

Verify label exists on the Service manifest used for discovery

Ensure your kubelet proxy Service is labeled with virtual-kubelet.koupleless.io/kubelet-proxy-service=true in the deployNamespace; otherwise discovery will fail.

Run this script to confirm the label exists in your repo manifests:

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "Searching for kubelet proxy Service label..."
rg -n -C2 'virtual-kubelet\.koupleless\.io/kubelet-proxy-service\s*:\s*"?true"?' -g '!**/vendor/**' -g '!**/node_modules/**'

echo "Checking Services in code (if generated in Go):"
rg -n -C3 'Labels:\s*map\[string\]string\{[^}]*virtual-kubelet\.koupleless\.io/kubelet-proxy-service' --type=go -g '!**/vendor/**'

Length of output: 245

Add missing discovery label to the kubelet proxy Service manifest

I ran the provided search across the repository and found no occurrences of
virtual-kubelet.koupleless.io/kubelet-proxy-service: "true"
in your YAML manifests or Go code. Without this label, lookupProxyServiceIP will be unable to find the Service when KUBELET_PROXY_ENABLED=true, causing runtime failures.

Please update the Service manifest used for discovery (e.g., under your deploy/ directory) to include:

metadata:
  labels:
    virtual-kubelet.koupleless.io/kubelet-proxy-service: "true"
🤖 Prompt for AI Agents 
In cmd/module-controller/main.go around lines 136 to 144, the code enables
kubelet proxy discovery but the Service manifest lacks the discovery label so
lookupProxyServiceIP cannot find it at runtime; update the Service manifest used
for deployment (e.g., deploy/ directory) to add the metadata.labels key
virtual-kubelet.koupleless.io/kubelet-proxy-service: "true" to the kubelet proxy
Service so lookupProxyServiceIP can discover the Service when
KUBELET_PROXY_ENABLED=true.

// Configure and create VNode controller
vNodeControllerConfig := vkModel.BuildVNodeControllerConfig{
ClientID: clientID,
Expand All @@ -127,6 +150,8 @@ func main() {
IsCluster: isCluster,
WorkloadMaxLevel: workloadMaxLevel,
VNodeWorkerNum: vnodeWorkerNum,
// vnode ip will fall back to the ip of the base pod if not set
PseudoNodeIP: moduleControllerServiceIP,
}

moduleDeploymentController, err := module_deployment_controller.NewModuleDeploymentController(env)
Expand Down Expand Up @@ -164,6 +189,21 @@ func main() {
os.Exit(1)
}

if kubeletProxyEnabled {
zlogger.Info("starting kubelet proxy server")
err := kubelet_proxy.StartKubeletProxy(
ctx,
certFilePath,
keyFilePath,
":"+utils.GetEnv(model.EnvKeyOfKubeletProxyPort, DefaultKubeletHttpListenAddr),
k8sClientSet,
)
if err != nil {
zlogger.Error(err, "failed to start kubelet proxy server")
os.Exit(1)
}
}

zlogger.Info("Module controller running")
err = k8sControllerManager.Start(signals.SetupSignalHandler())
if err != nil {
Expand Down Expand Up @@ -218,3 +258,25 @@ func startTunnels(ctx context.Context, clientId string, env string, mgr manager.
// we only using one tunnel for now
return tunnels[0]
}

// lookupProxyServiceIP retrieves the ClusterIP of the kubelet proxy service in the specified namespace.
func lookupProxyServiceIP(ctx context.Context, namespace string, clientSet kubernetes.Interface) (string, error) {
svcList, err := clientSet.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", model.LabelKeyOfKubeletProxyService, "true"),
})
if err != nil {
return "", fmt.Errorf("failed to list services in namespace %s: %w", namespace, err)
}
if len(svcList.Items) == 0 {
return "", fmt.Errorf("no kubelet proxy service found in namespace %s", namespace)
}
if len(svcList.Items) > 1 {
return "", fmt.Errorf("multiple kubelet proxy services deteched in namespace %s, expected only one", namespace)
}

firstSvc := svcList.Items[0]
if firstSvc.Spec.ClusterIP == "" || firstSvc.Spec.ClusterIP == "None" {
return "", fmt.Errorf("kubelet proxy service %s in namespace %s has no valid ClusterIP", firstSvc.Name, namespace)
}
return firstSvc.Spec.ClusterIP, nil
}
22 changes: 22 additions & 0 deletions common/model/consts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,28 @@ const (
LabelKeyOfSkipReplicasControl = "virtual-kubelet.koupleless.io/replicas-control"
// LabelKeyOfVPodDeploymentStrategy specifies the deployment strategy
LabelKeyOfVPodDeploymentStrategy = "virtual-kubelet.koupleless.io/strategy"
// LabelKeyOfKubeletProxyService indicates the kubelet proxy service
LabelKeyOfKubeletProxyService = "virtual-kubelet.koupleless.io/kubelet-proxy-service"
)

// Env keys for module controller
const (
// EnvKeyOfClientID is the environment variable key for the client ID
EnvKeyOfClientID = "CLIENT_ID"
// EnvKeyOfNamespace is the environment variable key for the deployment namespace, default to "default"
EnvKeyOfNamespace = "NAMESPACE"
// EnvKeyOfENV is the environment variable key for the environment label
EnvKeyOfENV = "ENV"
// EnvKeyOfClusterModeEnabled is the environment variable key for the cluster flag, use "true" or "false"
EnvKeyOfClusterModeEnabled = "IS_CLUSTER"
// EnvKeyOfWorkloadMaxLevel is the environment variable key for the maximum workload level
EnvKeyOfWorkloadMaxLevel = "WORKLOAD_MAX_LEVEL"
// EnvKeyOfVNodeWorkerNum is the environment variable key for the number of vnode worker threads
EnvKeyOfVNodeWorkerNum = "VNODE_WORKER_NUM"
// EnvKeyOfKubeletProxyEnabled is the environment variable key for enabling kubelet proxy
EnvKeyOfKubeletProxyEnabled = "KUBELET_PROXY_ENABLED"
// EnvKeyOfKubeletProxyPort is the environment variable key for the kubelet proxy port
EnvKeyOfKubeletProxyPort = "KUBELET_PROXY_PORT"
)

// Component types
Expand Down
4 changes: 2 additions & 2 deletions debug.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Build the manager binary
FROM golang:1.22 as builder
FROM golang:1.24 as builder
ARG TARGETOS
ARG TARGETARCH

Expand Down Expand Up @@ -43,4 +43,4 @@ EXPOSE 7777
EXPOSE 2345

#ENTRYPOINT ["dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./module_controller"]
CMD ["/bin/sh", "-c", "tail -f /dev/null"]
CMD ["/bin/sh", "-c", "tail -f /dev/null"]
37 changes: 22 additions & 15 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
module github.com/koupleless/module_controller

go 1.22.0
go 1.23.0

toolchain go1.22.4
toolchain go1.24.5

require (
github.com/eclipse/paho.mqtt.golang v1.5.0
github.com/go-logr/logr v1.4.2
github.com/go-resty/resty/v2 v2.11.0
github.com/google/uuid v1.6.0
github.com/koupleless/arkctl v0.2.4-0.20250106035535-5ed5cb871995
github.com/koupleless/virtual-kubelet v0.3.8
github.com/koupleless/virtual-kubelet v0.3.9
github.com/onsi/ginkgo/v2 v2.19.0
github.com/onsi/gomega v1.33.1
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
github.com/stretchr/testify v1.10.0
github.com/virtual-kubelet/virtual-kubelet v1.11.0
github.com/wind-c/comqtt/v2 v2.6.0
k8s.io/api v0.31.0
Expand All @@ -27,6 +27,7 @@ require (

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
Expand All @@ -42,22 +43,26 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/moby/spdystream v0.4.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/client_golang v1.20.4 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.64.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
Expand All @@ -66,21 +71,23 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.25.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.23.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.31.0 // indirect
k8s.io/apiserver v0.31.0 // indirect
k8s.io/component-base v0.31.0 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
Expand Down
Loading
Loading