diff --git a/.golangci.yml b/.golangci.yml index 6c48152..3035363 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -17,3 +17,7 @@ formatters: enable: - gofumpt - goimports + + +run: + tests: false diff --git a/Dockerfile b/Dockerfile index 56d70a6..536c4c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.22 as builder +FROM golang:1.24 as builder ARG TARGETOS ARG TARGETARCH @@ -36,4 +36,4 @@ EXPOSE 9090 EXPOSE 8080 EXPOSE 7777 -ENTRYPOINT ["./module_controller"] \ No newline at end of file +ENTRYPOINT ["./module_controller"] diff --git a/cmd/module-controller/main.go b/cmd/module-controller/main.go index b140bd6..720f442 100644 --- a/cmd/module-controller/main.go +++ b/cmd/module-controller/main.go @@ -18,37 +18,46 @@ import ( "context" "errors" "fmt" - "github.com/koupleless/module_controller/common/zaplogger" - "github.com/koupleless/module_controller/report_server" - "github.com/koupleless/virtual-kubelet/vnode_controller" "os" "os/signal" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/healthz" "strconv" "syscall" "github.com/google/uuid" "github.com/koupleless/module_controller/common/model" + "github.com/koupleless/module_controller/common/zaplogger" "github.com/koupleless/module_controller/controller/module_deployment_controller" "github.com/koupleless/module_controller/module_tunnels/koupleless_http_tunnel" "github.com/koupleless/module_controller/module_tunnels/koupleless_mqtt_tunnel" + "github.com/koupleless/module_controller/report_server" + "github.com/koupleless/module_controller/staging/kubelet_proxy" "github.com/koupleless/virtual-kubelet/common/tracker" "github.com/koupleless/virtual-kubelet/common/utils" vkModel "github.com/koupleless/virtual-kubelet/model" "github.com/koupleless/virtual-kubelet/tunnel" + "github.com/koupleless/virtual-kubelet/vnode_controller" "github.com/sirupsen/logrus" "github.com/virtual-kubelet/virtual-kubelet/log" logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" "github.com/virtual-kubelet/virtual-kubelet/trace" "github.com/virtual-kubelet/virtual-kubelet/trace/opencensus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) +const ( + certFilePath = "/etc/virtual-kubelet/tls/tls.crt" + keyFilePath = "/etc/virtual-kubelet/tls/tls.key" + DefaultKubeletHttpListenAddr = "10250" // Default listen address for the HTTP server +) + // Main function for the module controller // Responsibilities: // 1. Sets up signal handling for graceful shutdown @@ -59,6 +68,7 @@ import ( // 6. Creates and configures the VNode controller // 7. Optionally creates module deployment controller // 8. Starts all tunnels and the manager +// 9. Starts the kubelet proxy server(if enabled) func main() { ctx, cancel := context.WithCancel(context.Background()) @@ -75,27 +85,29 @@ func main() { trace.T = opencensus.Adapter{} // Get configuration from environment variables - clientID := utils.GetEnv("CLIENT_ID", uuid.New().String()) + clientID := utils.GetEnv(model.EnvKeyOfClientID, uuid.New().String()) env := utils.GetEnv("ENV", "dev") zlogger := zaplogger.GetLogger() ctx = zaplogger.WithLogger(ctx, zlogger) // Parse configuration with defaults - isCluster := utils.GetEnv("IS_CLUSTER", "") == "true" - workloadMaxLevel, err := strconv.Atoi(utils.GetEnv("WORKLOAD_MAX_LEVEL", "3")) + isCluster := utils.GetEnv(model.EnvKeyOfClusterModeEnabled, "") == "true" + workloadMaxLevel, err := strconv.Atoi(utils.GetEnv(model.EnvKeyOfWorkloadMaxLevel, "3")) if err != nil { zlogger.Error(err, "failed to parse WORKLOAD_MAX_LEVEL, will be set to 3 default") workloadMaxLevel = 3 } - vnodeWorkerNum, err := strconv.Atoi(utils.GetEnv("VNODE_WORKER_NUM", "8")) + vnodeWorkerNum, err := strconv.Atoi(utils.GetEnv(model.EnvKeyOfVNodeWorkerNum, "8")) if err != nil { zlogger.Error(err, "failed to parse VNODE_WORKER_NUM, will be set to 8 default") vnodeWorkerNum = 8 } + deployNamespace := utils.GetEnv(model.EnvKeyOfNamespace, "default") + // Initialize controller manager kubeConfig := config.GetConfigOrDie() // TODO: should support to set from parameter @@ -111,7 +123,6 @@ func main() { BindAddress: ":9090", }, }) - if err != nil { zlogger.Error(err, "unable to set up overall controller manager") os.Exit(1) @@ -119,6 +130,18 @@ func main() { tracker.SetTracker(&tracker.DefaultTracker{}) + k8sClientSet := kubernetes.NewForConfigOrDie(kubeConfig) + + var moduleControllerServiceIP string + var kubeletProxyEnabled bool + if os.Getenv(model.EnvKeyOfKubeletProxyEnabled) == "true" { + moduleControllerServiceIP, err = lookupProxyServiceIP(ctx, deployNamespace, k8sClientSet) + if err != nil { + log.G(ctx).Fatalf("Failed to lookup kubelet proxy service IP: %v", err) + } + kubeletProxyEnabled = true + } + // Configure and create VNode controller vNodeControllerConfig := vkModel.BuildVNodeControllerConfig{ ClientID: clientID, @@ -127,6 +150,8 @@ func main() { IsCluster: isCluster, WorkloadMaxLevel: workloadMaxLevel, VNodeWorkerNum: vnodeWorkerNum, + // vnode ip will fall back to the ip of the base pod if not set + PseudoNodeIP: moduleControllerServiceIP, } moduleDeploymentController, err := module_deployment_controller.NewModuleDeploymentController(env) @@ -164,6 +189,21 @@ func main() { os.Exit(1) } + if kubeletProxyEnabled { + zlogger.Info("starting kubelet proxy server") + err := kubelet_proxy.StartKubeletProxy( + ctx, + certFilePath, + keyFilePath, + ":"+utils.GetEnv(model.EnvKeyOfKubeletProxyPort, DefaultKubeletHttpListenAddr), + k8sClientSet, + ) + if err != nil { + zlogger.Error(err, "failed to start kubelet proxy server") + os.Exit(1) + } + } + zlogger.Info("Module controller running") err = k8sControllerManager.Start(signals.SetupSignalHandler()) if err != nil { @@ -218,3 +258,25 @@ func startTunnels(ctx context.Context, clientId string, env string, mgr manager. // we only using one tunnel for now return tunnels[0] } + +// lookupProxyServiceIP retrieves the ClusterIP of the kubelet proxy service in the specified namespace. +func lookupProxyServiceIP(ctx context.Context, namespace string, clientSet kubernetes.Interface) (string, error) { + svcList, err := clientSet.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", model.LabelKeyOfKubeletProxyService, "true"), + }) + if err != nil { + return "", fmt.Errorf("failed to list services in namespace %s: %w", namespace, err) + } + if len(svcList.Items) == 0 { + return "", fmt.Errorf("no kubelet proxy service found in namespace %s", namespace) + } + if len(svcList.Items) > 1 { + return "", fmt.Errorf("multiple kubelet proxy services deteched in namespace %s, expected only one", namespace) + } + + firstSvc := svcList.Items[0] + if firstSvc.Spec.ClusterIP == "" || firstSvc.Spec.ClusterIP == "None" { + return "", fmt.Errorf("kubelet proxy service %s in namespace %s has no valid ClusterIP", firstSvc.Name, namespace) + } + return firstSvc.Spec.ClusterIP, nil +} diff --git a/common/model/consts.go b/common/model/consts.go index 911b69d..09cfd75 100644 --- a/common/model/consts.go +++ b/common/model/consts.go @@ -14,6 +14,28 @@ const ( LabelKeyOfSkipReplicasControl = "virtual-kubelet.koupleless.io/replicas-control" // LabelKeyOfVPodDeploymentStrategy specifies the deployment strategy LabelKeyOfVPodDeploymentStrategy = "virtual-kubelet.koupleless.io/strategy" + // LabelKeyOfKubeletProxyService indicates the kubelet proxy service + LabelKeyOfKubeletProxyService = "virtual-kubelet.koupleless.io/kubelet-proxy-service" +) + +// Env keys for module controller +const ( + // EnvKeyOfClientID is the environment variable key for the client ID + EnvKeyOfClientID = "CLIENT_ID" + // EnvKeyOfNamespace is the environment variable key for the deployment namespace, default to "default" + EnvKeyOfNamespace = "NAMESPACE" + // EnvKeyOfENV is the environment variable key for the environment label + EnvKeyOfENV = "ENV" + // EnvKeyOfClusterModeEnabled is the environment variable key for the cluster flag, use "true" or "false" + EnvKeyOfClusterModeEnabled = "IS_CLUSTER" + // EnvKeyOfWorkloadMaxLevel is the environment variable key for the maximum workload level + EnvKeyOfWorkloadMaxLevel = "WORKLOAD_MAX_LEVEL" + // EnvKeyOfVNodeWorkerNum is the environment variable key for the number of vnode worker threads + EnvKeyOfVNodeWorkerNum = "VNODE_WORKER_NUM" + // EnvKeyOfKubeletProxyEnabled is the environment variable key for enabling kubelet proxy + EnvKeyOfKubeletProxyEnabled = "KUBELET_PROXY_ENABLED" + // EnvKeyOfKubeletProxyPort is the environment variable key for the kubelet proxy port + EnvKeyOfKubeletProxyPort = "KUBELET_PROXY_PORT" ) // Component types diff --git a/debug.Dockerfile b/debug.Dockerfile index fac03ce..f3cbde8 100644 --- a/debug.Dockerfile +++ b/debug.Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.22 as builder +FROM golang:1.24 as builder ARG TARGETOS ARG TARGETARCH @@ -43,4 +43,4 @@ EXPOSE 7777 EXPOSE 2345 #ENTRYPOINT ["dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./module_controller"] -CMD ["/bin/sh", "-c", "tail -f /dev/null"] \ No newline at end of file +CMD ["/bin/sh", "-c", "tail -f /dev/null"] diff --git a/go.mod b/go.mod index 0de7977..08cad0c 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/koupleless/module_controller -go 1.22.0 +go 1.23.0 -toolchain go1.22.4 +toolchain go1.24.5 require ( github.com/eclipse/paho.mqtt.golang v1.5.0 @@ -10,11 +10,11 @@ require ( github.com/go-resty/resty/v2 v2.11.0 github.com/google/uuid v1.6.0 github.com/koupleless/arkctl v0.2.4-0.20250106035535-5ed5cb871995 - github.com/koupleless/virtual-kubelet v0.3.8 + github.com/koupleless/virtual-kubelet v0.3.9 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/virtual-kubelet/virtual-kubelet v1.11.0 github.com/wind-c/comqtt/v2 v2.6.0 k8s.io/api v0.31.0 @@ -27,6 +27,7 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -42,22 +43,26 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/client_golang v1.20.4 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rs/xid v1.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -66,21 +71,23 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.14.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/term v0.32.0 // indirect + golang.org/x/text v0.25.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.23.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect + k8s.io/apiserver v0.31.0 // indirect + k8s.io/component-base v0.31.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index a89b7d2..4cc90f1 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -70,8 +74,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -80,6 +84,8 @@ github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -92,10 +98,12 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/koupleless/arkctl v0.2.4-0.20250106035535-5ed5cb871995 h1:dkBdI/WczkOJ4LaoZteX3uz8r+WKxBqYQjpzwyVDvyw= github.com/koupleless/arkctl v0.2.4-0.20250106035535-5ed5cb871995/go.mod h1:nbnAiPEv7x/ZDQ+QsjFWkqwxMDofGmqnFPHa3XpXHyE= -github.com/koupleless/virtual-kubelet v0.3.8 h1:iCzHorbxRZSq3B5DHMvKlP6Uovw5OPMKxfl7xUP8i3c= -github.com/koupleless/virtual-kubelet v0.3.8/go.mod h1:V/RjXRvoSNr55I9KMV+tgtOp6duxxBMcwyDTH04XiX0= +github.com/koupleless/virtual-kubelet v0.3.9 h1:7O+unpomhxiLvgF4R856p4YUVr10rtz27zhxRylgJuM= +github.com/koupleless/virtual-kubelet v0.3.9/go.mod h1:V/RjXRvoSNr55I9KMV+tgtOp6duxxBMcwyDTH04XiX0= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -103,10 +111,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= +github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -114,6 +126,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= @@ -123,13 +137,13 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= +github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= +github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= @@ -148,8 +162,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/virtual-kubelet/virtual-kubelet v1.11.0 h1:LOMcZQfP083xmYH9mYtyHAR+ybFbK1uMaRA+EtDcd1I= github.com/virtual-kubelet/virtual-kubelet v1.11.0/go.mod h1:WQfPHbIlzfhMNYkh6hFXF1ctGfNM8UJCYLYpLa/trxc= github.com/wind-c/comqtt/v2 v2.6.0 h1:huLdOwYDrwMTrEwH7+mSs1GftHZ/tDqJw8nOz3iX7kc= @@ -196,11 +210,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -208,8 +222,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -222,23 +236,23 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -279,8 +293,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -305,8 +319,12 @@ k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24 k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY= +k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk= k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs= +k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/staging/kubelet_proxy/pod_handler.go b/staging/kubelet_proxy/pod_handler.go new file mode 100644 index 0000000..551b358 --- /dev/null +++ b/staging/kubelet_proxy/pod_handler.go @@ -0,0 +1,121 @@ +package kubelet_proxy + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/koupleless/virtual-kubelet/common/utils" + vkModel "github.com/koupleless/virtual-kubelet/model" + "github.com/virtual-kubelet/virtual-kubelet/errdefs" + "github.com/virtual-kubelet/virtual-kubelet/log" + "github.com/virtual-kubelet/virtual-kubelet/node/api" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" +) + +func NewPodHandler(clientSet kubernetes.Interface) (*PodHandler, error) { + if clientSet == nil { + return nil, fmt.Errorf("clientSet can't be nil") + } + return &PodHandler{clientSet: clientSet}, nil +} + +type PodHandler struct { + clientSet kubernetes.Interface // Kubernetes clientSet to interact with the cluster +} + +func (f *PodHandler) AttachPodRoutes(mux *http.ServeMux) { + mux.Handle("/", api.PodHandler(api.PodHandlerConfig{ + GetContainerLogs: f.getContainerLogs, + }, true)) +} + +func (f *PodHandler) getContainerLogs( + ctx context.Context, + namespace string, + podName string, + containerName string, + opts api.ContainerLogOpts, +) (io.ReadCloser, error) { + log.L.Debugf("Forwarding log request for pod %s in namespace %s, container %s", podName, namespace, containerName) + basePod, err := f.lookupBasePod(ctx, namespace, podName) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, errdefs.NotFound(err.Error()) + } + return nil, err + } + + podLogOptions := &corev1.PodLogOptions{ + Container: utils.OrElse(basePod.Labels[vkModel.LabelKeyOfBaseContainerName], "base"), + Follow: opts.Follow, + Previous: opts.Previous, + Timestamps: opts.Timestamps, + TailLines: func() *int64 { + if opts.Tail > 0 { + return ptr.To[int64](int64(opts.Tail)) + } + return nil + }(), + LimitBytes: func() *int64 { + if opts.LimitBytes > 0 { + return ptr.To[int64](int64(opts.LimitBytes)) + } + return nil + }(), + SinceTime: func() *metav1.Time { + if !opts.SinceTime.IsZero() { + return &metav1.Time{Time: opts.SinceTime} + } + return nil + }(), + SinceSeconds: func() *int64 { + if opts.SinceSeconds > 0 { + return ptr.To[int64](int64(opts.SinceSeconds)) + } + return nil + }(), + } + rc, err := f.clientSet.CoreV1().Pods(basePod.Namespace).GetLogs(basePod.Name, podLogOptions).Stream(ctx) + if err != nil { + return nil, err + } + return rc, nil +} + +// lookupBasePod retrieves the base pod for a given virtual pod name in the specified namespace. +func (f *PodHandler) lookupBasePod(ctx context.Context, namespace, vPodName string) (*corev1.Pod, error) { + vPod, err := f.clientSet.CoreV1().Pods(namespace).Get(ctx, vPodName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, apierrors.NewNotFound(corev1.Resource("pods"), vPodName) + } + log.L.Errorf("Fail to pod %s in namespace %s: %v", vPodName, namespace, err) + return nil, err + } + + vNodeName := vPod.Spec.NodeName + + vNode, err := f.clientSet.CoreV1().Nodes().Get(ctx, vNodeName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, apierrors.NewNotFound(corev1.Resource("nodes"), vNodeName) + } + log.L.Errorf("Error getting node %s: %v", vNodeName, err) + return nil, err + } + // compatible with old style label + basePodName := utils.OrElse( + vNode.Labels[vkModel.LabelKeyOfBaseHostName], + vNode.Labels[corev1.LabelHostname], + ) + if basePodName == "" { + return nil, apierrors.NewNotFound(corev1.Resource("pods"), vPodName) + } + return f.clientSet.CoreV1().Pods(namespace).Get(ctx, basePodName, metav1.GetOptions{}) +} diff --git a/staging/kubelet_proxy/pod_handler_test.go b/staging/kubelet_proxy/pod_handler_test.go new file mode 100644 index 0000000..cb32fa5 --- /dev/null +++ b/staging/kubelet_proxy/pod_handler_test.go @@ -0,0 +1,132 @@ +package kubelet_proxy + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + vkModel "github.com/koupleless/virtual-kubelet/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestNewPodHandler(t *testing.T) { + clientSet := fake.NewClientset() + handler, err := NewPodHandler(clientSet) + require.NoError(t, err) + require.NotNil(t, handler) + assert.Equal(t, clientSet, handler.clientSet) + + _, err = NewPodHandler(nil) + require.Error(t, err) +} + +func TestPodHandler_AttachPodRoutes(t *testing.T) { + vPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vpod", + Namespace: "vpod-namespace", + }, + Spec: corev1.PodSpec{ + NodeName: "vnode", + }, + } + vNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vnode", + Labels: map[string]string{ + vkModel.LabelKeyOfBaseHostName: "base-pod", + }, + }, + } + basePod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base-pod", + Namespace: "vpod-namespace", + }, + } + + handler, err := NewPodHandler(fake.NewClientset(vPod, vNode, basePod)) + require.NoError(t, err) + + mux := http.NewServeMux() + handler.AttachPodRoutes(mux) + + t.Run("normal case", func(t *testing.T) { + t.Run("with default params", func(t *testing.T) { + recorder := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/containerLogs/vpod-namespace/vpod/ignored", nil) + + mux.ServeHTTP(recorder, req) + assert.EqualValues(t, http.StatusOK, recorder.Code) + assert.EqualValues(t, "fake logs", recorder.Body.String()) + }) + t.Run("with custom params", func(t *testing.T) { + recorder := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/containerLogs/vpod-namespace/vpod/ignored", nil) + queryParams := make(url.Values) + queryParams.Add("tailLines", "1") + queryParams.Add("follow", "true") + queryParams.Add("timestamps", "true") + queryParams.Add("previous", "false") + queryParams.Add("limitBytes", "1000") + queryParams.Add("sinceTime", "2023-10-01T00:00:00Z") + req.URL.RawQuery = queryParams.Encode() + + mux.ServeHTTP(recorder, req) + assert.EqualValues(t, http.StatusOK, recorder.Code) + assert.EqualValues(t, "fake logs", recorder.Body.String()) + + queryParams.Del("sinceTime") + queryParams.Add("sinceSeconds", "60") + req.URL.RawQuery = queryParams.Encode() + + recorder = httptest.NewRecorder() + mux.ServeHTTP(recorder, req) + assert.EqualValues(t, http.StatusOK, recorder.Code) + assert.EqualValues(t, "fake logs", recorder.Body.String()) + }) + }) + + t.Run("n: vpod not found", func(t *testing.T) { + recorder := httptest.NewRecorder() + mux.ServeHTTP( + recorder, + httptest.NewRequest("GET", "/containerLogs/vpod-namespace/nonexistent/ignored", nil), + ) + assert.EqualValues(t, http.StatusNotFound, recorder.Code) + }) + + t.Run("n: vnode not found", func(t *testing.T) { + vPodCopy := vPod.DeepCopy() + vPodCopy.Spec.NodeName = "nonexistent-vnode" + clientSet := fake.NewClientset(vPodCopy, vNode) + + handler, err := NewPodHandler(clientSet) + require.NoError(t, err) + mux := http.NewServeMux() + handler.AttachPodRoutes(mux) + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, httptest.NewRequest("GET", "/containerLogs/vpod-namespace/vpod/ignored", nil)) + + assert.EqualValues(t, http.StatusNotFound, recorder.Code) + + }) + + t.Run("n: base pod not found", func(t *testing.T) { + clientSet := fake.NewClientset(vPod, vNode) + + handler, err := NewPodHandler(clientSet) + require.NoError(t, err) + mux := http.NewServeMux() + handler.AttachPodRoutes(mux) + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, httptest.NewRequest("GET", "/containerLogs/vpod-namespace/vpod/ignored", nil)) + + assert.EqualValues(t, http.StatusNotFound, recorder.Code) + }) +} diff --git a/staging/kubelet_proxy/proxy.go b/staging/kubelet_proxy/proxy.go new file mode 100644 index 0000000..af2f7fb --- /dev/null +++ b/staging/kubelet_proxy/proxy.go @@ -0,0 +1,74 @@ +package kubelet_proxy + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/http" + "time" + + "github.com/virtual-kubelet/virtual-kubelet/log" + "k8s.io/client-go/kubernetes" +) + +const ( + DefaultHeaderRequestTimeout = 10 * time.Second +) + +// StartKubeletProxy initializes and starts the kubelet proxy server. +func StartKubeletProxy( + ctx context.Context, + certFile, keyFile string, + listenAddr string, + clientSet kubernetes.Interface, +) error { + proxy, err := NewPodHandler(clientSet) + if err != nil { + return err + } + + tlsConfig, err := loadTLSConfig(certFile, keyFile) + if err != nil { + return err + } + + mux := http.NewServeMux() + proxy.AttachPodRoutes(mux) + + srv := &http.Server{ + Addr: listenAddr, + Handler: mux, + TLSConfig: tlsConfig, + ReadHeaderTimeout: DefaultHeaderRequestTimeout, + } + + go func() { + log.G(ctx).Infof("Starting kubelet proxy server on %s", listenAddr) + if err := srv.ListenAndServeTLS(certFile, keyFile); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.G(ctx).Fatalf("Failed to start kubelet proxy server: %v", err) + } + }() + + go func() { + <-ctx.Done() + log.G(ctx).Info("Shutting down kubelet proxy server") + if err := srv.Shutdown(context.Background()); err != nil { + log.G(ctx).Errorf("Failed to gracefully shutdown kubelet proxy server: %v", err) + } + }() + + return nil +} + +func loadTLSConfig(certFile, keyFile string) (*tls.Config, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("fail to load TLS certificate and key: %w", err) + } + + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS12, + }, nil +} diff --git a/staging/kubelet_proxy/proxy_test.go b/staging/kubelet_proxy/proxy_test.go new file mode 100644 index 0000000..2f66eb9 --- /dev/null +++ b/staging/kubelet_proxy/proxy_test.go @@ -0,0 +1,90 @@ +package kubelet_proxy + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes/fake" +) + +func TestStartKubeletProxy(t *testing.T) { + cert, key := genTestCertPair() + defer func() { + os.Remove(cert) + os.Remove(key) + }() + + t.Run("normal case", func(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + err := StartKubeletProxy(ctx, cert, key, ":10250", fake.NewClientset()) + require.NoError(t, err) + }) + + t.Run("negative cases", func(t *testing.T) { + t.Run("empty cert file", func(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + err := StartKubeletProxy(ctx, "", key, ":10250", fake.NewClientset()) + require.Error(t, err) + }) + t.Run("empty key file", func(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + err := StartKubeletProxy(ctx, cert, "", ":10250", fake.NewClientset()) + require.Error(t, err) + }) + }) +} + +func genTestCertPair() (certFileLoc string, keyFileLoc string) { + certFileLoc = "test_cert.pem" + keyFileLoc = "test_key.pem" + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + panic("failed to generate private key: " + err.Error()) + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(8760 * time.Hour), // 365 天 + DNSNames: []string{"localhost"}, + } + cert, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + panic("failed to create certificate: " + err.Error()) + } + certFile, err := os.Create(certFileLoc) + if err != nil { + panic("failed to create cert file: " + err.Error()) + } + defer certFile.Close() + err = pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: cert}) + if err != nil { + panic("failed to write cert to file: " + err.Error()) + } + keyFile, err := os.Create(keyFileLoc) + if err != nil { + panic("failed to create key file: " + err.Error()) + } + defer keyFile.Close() + err = pem.Encode(keyFile, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + if err != nil { + panic("failed to write key to file: " + err.Error()) + } + return certFileLoc, keyFileLoc +}