From 8f642b71b33abcbb8100165ed334bafbea72d245 Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Fri, 5 Jan 2024 15:09:28 +0800 Subject: [PATCH] Clean up components patches --- pkg/kwokctl/components/etcd.go | 29 ++--- pkg/kwokctl/components/jaeger.go | 4 - pkg/kwokctl/components/kube_apiserver.go | 6 - .../components/kube_controller_manager.go | 6 - pkg/kwokctl/components/kube_scheduler.go | 6 - pkg/kwokctl/components/kwok_controller.go | 7 - pkg/kwokctl/components/prometheus.go | 6 - pkg/kwokctl/components/utils.go | 6 - pkg/kwokctl/runtime/binary/cluster.go | 67 ++++------ pkg/kwokctl/runtime/compose/cluster.go | 122 ++++++------------ pkg/kwokctl/runtime/kind/cluster.go | 114 ++++++++-------- pkg/kwokctl/runtime/kind/kind.yaml.tpl | 8 +- pkg/kwokctl/runtime/util.go | 20 +++ .../binary/create_cluster_with_extra.txt | 12 +- .../docker/create_cluster_with_extra.txt | 12 +- .../kind-podman/create_cluster_with_extra.txt | 74 +++++------ .../create_cluster_with_verbosity.txt | 20 +-- .../kind/create_cluster_with_extra.txt | 74 +++++------ .../kind/create_cluster_with_verbosity.txt | 20 +-- .../nerdctl/create_cluster_with_extra.txt | 12 +- .../podman/create_cluster_with_extra.txt | 12 +- 21 files changed, 274 insertions(+), 363 deletions(-) diff --git a/pkg/kwokctl/components/etcd.go b/pkg/kwokctl/components/etcd.go index 25ef805ca..f1de73dac 100644 --- a/pkg/kwokctl/components/etcd.go +++ b/pkg/kwokctl/components/etcd.go @@ -29,20 +29,17 @@ import ( // BuildEtcdComponentConfig is the configuration for building an etcd component. type BuildEtcdComponentConfig struct { - Runtime string - Binary string - Image string - ProjectName string - Version version.Version - DataPath string - Workdir string - BindAddress string - Port uint32 - PeerPort uint32 - Verbosity log.Level - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume - ExtraEnvs []internalversion.Env + Runtime string + Binary string + Image string + ProjectName string + Version version.Version + DataPath string + Workdir string + BindAddress string + Port uint32 + PeerPort uint32 + Verbosity log.Level } // BuildEtcdComponent builds an etcd component. @@ -59,7 +56,6 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio } var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port etcdArgs := []string{ @@ -67,7 +63,7 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio "--auto-compaction-retention=1", "--quota-backend-bytes=8589934592", } - etcdArgs = append(etcdArgs, extraArgsToStrings(conf.ExtraArgs)...) + var metric *internalversion.ComponentMetric if GetRuntimeMode(conf.Runtime) != RuntimeModeNative { @@ -149,7 +145,6 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio Value: runtime.GOARCH, }) } - envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ Name: consts.ComponentEtcd, diff --git a/pkg/kwokctl/components/jaeger.go b/pkg/kwokctl/components/jaeger.go index 94114595a..7c5e7d938 100644 --- a/pkg/kwokctl/components/jaeger.go +++ b/pkg/kwokctl/components/jaeger.go @@ -36,17 +36,13 @@ type BuildJaegerComponentConfig struct { Port uint32 OtlpGrpcPort uint32 Verbosity log.Level - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume } // BuildJaegerComponent builds a jaeger component. func BuildJaegerComponent(conf BuildJaegerComponentConfig) (component internalversion.Component, err error) { jaegerArgs := []string{"--collector.otlp.enabled=true"} - jaegerArgs = append(jaegerArgs, extraArgsToStrings(conf.ExtraArgs)...) var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port if GetRuntimeMode(conf.Runtime) != RuntimeModeNative { diff --git a/pkg/kwokctl/components/kube_apiserver.go b/pkg/kwokctl/components/kube_apiserver.go index 25d878100..e544a25a2 100644 --- a/pkg/kwokctl/components/kube_apiserver.go +++ b/pkg/kwokctl/components/kube_apiserver.go @@ -53,9 +53,6 @@ type BuildKubeApiserverComponentConfig struct { Verbosity log.Level DisableQPSLimits bool TracingConfigPath string - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume - ExtraEnvs []internalversion.Env } // BuildKubeApiserverComponent builds a kube-apiserver component. @@ -80,7 +77,6 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon ) } - kubeApiserverArgs = append(kubeApiserverArgs, extraArgsToStrings(conf.ExtraArgs)...) if conf.KubeRuntimeConfig != "" { kubeApiserverArgs = append(kubeApiserverArgs, "--runtime-config="+conf.KubeRuntimeConfig, @@ -122,7 +118,6 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon var ports []internalversion.Port var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) var metric *internalversion.ComponentMetric if GetRuntimeMode(conf.Runtime) != RuntimeModeNative { @@ -285,7 +280,6 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon } envs := []internalversion.Env{} - envs = append(envs, conf.ExtraEnvs...) links := []string{consts.ComponentEtcd} if conf.TracingConfigPath != "" { diff --git a/pkg/kwokctl/components/kube_controller_manager.go b/pkg/kwokctl/components/kube_controller_manager.go index a4afb7714..09535a972 100644 --- a/pkg/kwokctl/components/kube_controller_manager.go +++ b/pkg/kwokctl/components/kube_controller_manager.go @@ -48,15 +48,11 @@ type BuildKubeControllerManagerComponentConfig struct { NodeMonitorGracePeriodMilliseconds int64 Verbosity log.Level DisableQPSLimits bool - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume - ExtraEnvs []internalversion.Env } // BuildKubeControllerManagerComponent builds a kube-controller-manager component. func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponentConfig) (component internalversion.Component, err error) { kubeControllerManagerArgs := []string{} - kubeControllerManagerArgs = append(kubeControllerManagerArgs, extraArgsToStrings(conf.ExtraArgs)...) if conf.KubeFeatureGates != "" { kubeControllerManagerArgs = append(kubeControllerManagerArgs, @@ -77,7 +73,6 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen } var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port var metric *internalversion.ComponentMetric @@ -224,7 +219,6 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen } envs := []internalversion.Env{} - envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ Name: consts.ComponentKubeControllerManager, diff --git a/pkg/kwokctl/components/kube_scheduler.go b/pkg/kwokctl/components/kube_scheduler.go index 6137468c7..07316fdaa 100644 --- a/pkg/kwokctl/components/kube_scheduler.go +++ b/pkg/kwokctl/components/kube_scheduler.go @@ -44,15 +44,11 @@ type BuildKubeSchedulerComponentConfig struct { KubeFeatureGates string Verbosity log.Level DisableQPSLimits bool - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume - ExtraEnvs []internalversion.Env } // BuildKubeSchedulerComponent builds a kube-scheduler component. func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (component internalversion.Component, err error) { kubeSchedulerArgs := []string{} - kubeSchedulerArgs = append(kubeSchedulerArgs, extraArgsToStrings(conf.ExtraArgs)...) if conf.KubeFeatureGates != "" { kubeSchedulerArgs = append(kubeSchedulerArgs, @@ -61,7 +57,6 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon } var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port var metric *internalversion.ComponentMetric @@ -214,7 +209,6 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon } envs := []internalversion.Env{} - envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ Name: consts.ComponentKubeScheduler, diff --git a/pkg/kwokctl/components/kwok_controller.go b/pkg/kwokctl/components/kwok_controller.go index 0099a09e4..99d34021a 100644 --- a/pkg/kwokctl/components/kwok_controller.go +++ b/pkg/kwokctl/components/kwok_controller.go @@ -48,9 +48,6 @@ type BuildKwokControllerComponentConfig struct { Verbosity log.Level NodeLeaseDurationSeconds uint EnableCRDs []string - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume - ExtraEnvs []internalversion.Env } // BuildKwokControllerComponent builds a kwok controller component. @@ -67,10 +64,7 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp ) } - kwokControllerArgs = append(kwokControllerArgs, extraArgsToStrings(conf.ExtraArgs)...) - var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port if GetRuntimeMode(conf.Runtime) != RuntimeModeNative { @@ -170,7 +164,6 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp } envs := []internalversion.Env{} - envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ Name: consts.ComponentKwokController, diff --git a/pkg/kwokctl/components/prometheus.go b/pkg/kwokctl/components/prometheus.go index 6552b227e..31c0a66b0 100644 --- a/pkg/kwokctl/components/prometheus.go +++ b/pkg/kwokctl/components/prometheus.go @@ -38,18 +38,13 @@ type BuildPrometheusComponentConfig struct { AdminCertPath string AdminKeyPath string Verbosity log.Level - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume - ExtraEnvs []internalversion.Env } // BuildPrometheusComponent builds a prometheus component. func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component internalversion.Component, err error) { prometheusArgs := []string{} - prometheusArgs = append(prometheusArgs, extraArgsToStrings(conf.ExtraArgs)...) var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port var metric *internalversion.ComponentMetric @@ -99,7 +94,6 @@ func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component in } envs := []internalversion.Env{} - envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ Name: consts.ComponentPrometheus, diff --git a/pkg/kwokctl/components/utils.go b/pkg/kwokctl/components/utils.go index b733991db..21d26febc 100644 --- a/pkg/kwokctl/components/utils.go +++ b/pkg/kwokctl/components/utils.go @@ -67,12 +67,6 @@ func GroupByLinks(components []internalversion.Component) ([][]internalversion.C return groups, nil } -func extraArgsToStrings(args []internalversion.ExtraArgs) []string { - return slices.Map(args, func(arg internalversion.ExtraArgs) string { - return fmt.Sprintf("--%s=%s", arg.Key, arg.Value) - }) -} - // The following runtime mode is classification of runtime for components. const ( RuntimeModeNative = "native" diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index 736b65f85..2610a1277 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -374,21 +374,17 @@ func (c *Cluster) addEtcd(ctx context.Context, env *env) (err error) { return err } - etcdComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentEtcd) etcdComponent, err := components.BuildEtcdComponent(components.BuildEtcdComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Binary: etcdPath, - Version: etcdVersion, - BindAddress: conf.BindAddress, - DataPath: env.etcdDataPath, - Port: conf.EtcdPort, - PeerPort: conf.EtcdPeerPort, - Verbosity: env.verbosity, - ExtraArgs: etcdComponentPatches.ExtraArgs, - ExtraVolumes: etcdComponentPatches.ExtraVolumes, - ExtraEnvs: etcdComponentPatches.ExtraEnvs, + Runtime: conf.Runtime, + ProjectName: c.Name(), + Workdir: env.workdir, + Binary: etcdPath, + Version: etcdVersion, + BindAddress: conf.BindAddress, + DataPath: env.etcdDataPath, + Port: conf.EtcdPort, + PeerPort: conf.EtcdPeerPort, + Verbosity: env.verbosity, }) if err != nil { return err @@ -432,7 +428,6 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { } } - kubeApiserverComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeApiserver) kubeApiserverComponent, err := components.BuildKubeApiserverComponent(components.BuildKubeApiserverComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -456,9 +451,6 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, TracingConfigPath: kubeApiserverTracingConfigPath, - ExtraArgs: kubeApiserverComponentPatches.ExtraArgs, - ExtraVolumes: kubeApiserverComponentPatches.ExtraVolumes, - ExtraEnvs: kubeApiserverComponentPatches.ExtraEnvs, }) if err != nil { return err @@ -487,7 +479,6 @@ func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err e return err } - kubeControllerManagerPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeControllerManager) kubeControllerManagerComponent, err := components.BuildKubeControllerManagerComponent(components.BuildKubeControllerManagerComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -507,9 +498,6 @@ func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err e NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, - ExtraArgs: kubeControllerManagerPatches.ExtraArgs, - ExtraVolumes: kubeControllerManagerPatches.ExtraVolumes, - ExtraEnvs: kubeControllerManagerPatches.ExtraEnvs, }) if err != nil { return err @@ -548,7 +536,6 @@ func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { return err } - kubeSchedulerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeScheduler) kubeSchedulerComponent, err := components.BuildKubeSchedulerComponent(components.BuildKubeSchedulerComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -566,9 +553,6 @@ func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { KubeFeatureGates: conf.KubeFeatureGates, Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, - ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, - ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, - ExtraEnvs: kubeSchedulerComponentPatches.ExtraEnvs, }) if err != nil { return err @@ -589,8 +573,6 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return err } - kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) - kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -608,8 +590,6 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { Verbosity: env.verbosity, NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, EnableCRDs: conf.EnableCRDs, - ExtraArgs: kwokControllerComponentPatches.ExtraArgs, - ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, }) if err != nil { return err @@ -655,19 +635,15 @@ func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { return err } - prometheusComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) prometheusComponent, err := components.BuildPrometheusComponent(components.BuildPrometheusComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Binary: prometheusPath, - Version: prometheusVersion, - BindAddress: conf.BindAddress, - Port: conf.PrometheusPort, - ConfigPath: prometheusConfigPath, - Verbosity: env.verbosity, - ExtraArgs: prometheusComponentPatches.ExtraArgs, - ExtraVolumes: prometheusComponentPatches.ExtraVolumes, - ExtraEnvs: prometheusComponentPatches.ExtraEnvs, + Runtime: conf.Runtime, + Workdir: env.workdir, + Binary: prometheusPath, + Version: prometheusVersion, + BindAddress: conf.BindAddress, + Port: conf.PrometheusPort, + ConfigPath: prometheusConfigPath, + Verbosity: env.verbosity, }) if err != nil { return err @@ -689,7 +665,6 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) error { return err } - jaegerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentJaeger) jaegerComponent, err := components.BuildJaegerComponent(components.BuildJaegerComponentConfig{ Runtime: conf.Runtime, Workdir: env.workdir, @@ -699,8 +674,6 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) error { Port: conf.JaegerPort, OtlpGrpcPort: conf.JaegerOtlpGrpcPort, Verbosity: env.verbosity, - ExtraArgs: jaegerComponentPatches.ExtraArgs, - ExtraVolumes: jaegerComponentPatches.ExtraVolumes, }) if err != nil { return err @@ -713,6 +686,10 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) error { func (c *Cluster) finishInstall(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options + for i := range env.kwokctlConfig.Components { + runtime.ApplyComponentPatches(&env.kwokctlConfig.Components[i], env.kwokctlConfig.ComponentsPatches) + } + // Setup kubeconfig kubeconfigData, err := kubeconfig.EncodeKubeconfig(kubeconfig.BuildKubeconfig(kubeconfig.BuildKubeconfigConfig{ ProjectName: c.Name(), diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index b46c2c3e4..98bad01d0 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -326,6 +326,11 @@ func (c *Cluster) Install(ctx context.Context) error { return err } + err = c.preInstall(ctx, env) + if err != nil { + return err + } + err = c.setup(ctx, env) if err != nil { return err @@ -406,24 +411,16 @@ func (c *Cluster) addEtcd(ctx context.Context, env *env) (err error) { return err } - etcdComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentEtcd) - etcdComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(etcdComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for etcd component: %w", err) - } etcdComponent, err := components.BuildEtcdComponent(components.BuildEtcdComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Image: conf.EtcdImage, - Version: etcdVersion, - BindAddress: net.PublicAddress, - Port: conf.EtcdPort, - DataPath: env.etcdDataPath, - Verbosity: env.verbosity, - ExtraArgs: etcdComponentPatches.ExtraArgs, - ExtraVolumes: etcdComponentPatches.ExtraVolumes, - ExtraEnvs: etcdComponentPatches.ExtraEnvs, + Runtime: conf.Runtime, + ProjectName: c.Name(), + Workdir: env.workdir, + Image: conf.EtcdImage, + Version: etcdVersion, + BindAddress: net.PublicAddress, + Port: conf.EtcdPort, + DataPath: env.etcdDataPath, + Verbosity: env.verbosity, }) if err != nil { return err @@ -441,12 +438,6 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { return err } - kubeApiserverComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeApiserver) - kubeApiserverComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kubeApiserverComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kube api server component: %w", err) - } - kubeApiserverTracingConfigPath := "" if conf.JaegerPort != 0 { kubeApiserverTracingConfigData, err := k8s.BuildKubeApiserverTracingConfig(k8s.BuildKubeApiserverTracingConfigParam{ @@ -486,9 +477,6 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, TracingConfigPath: kubeApiserverTracingConfigPath, - ExtraArgs: kubeApiserverComponentPatches.ExtraArgs, - ExtraVolumes: kubeApiserverComponentPatches.ExtraVolumes, - ExtraEnvs: kubeApiserverComponentPatches.ExtraEnvs, }) if err != nil { return err @@ -507,11 +495,6 @@ func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err e return err } - kubeControllerManagerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeControllerManager) - kubeControllerManagerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kubeControllerManagerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kube controller manager component: %w", err) - } kubeControllerManagerComponent, err := components.BuildKubeControllerManagerComponent(components.BuildKubeControllerManagerComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -531,9 +514,6 @@ func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err e DisableQPSLimits: conf.DisableQPSLimits, NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, - ExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs, - ExtraVolumes: kubeControllerManagerComponentPatches.ExtraVolumes, - ExtraEnvs: kubeControllerManagerComponentPatches.ExtraEnvs, }) if err != nil { return err @@ -562,11 +542,6 @@ func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { return err } - kubeSchedulerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeScheduler) - kubeSchedulerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kubeSchedulerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kube scheduler component: %w", err) - } kubeSchedulerComponent, err := components.BuildKubeSchedulerComponent(components.BuildKubeSchedulerComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -584,9 +559,6 @@ func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { KubeFeatureGates: conf.KubeFeatureGates, Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, - ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, - ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, - ExtraEnvs: kubeSchedulerComponentPatches.ExtraEnvs, }) if err != nil { return err @@ -605,15 +577,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return err } - kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) - kwokControllerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kwokControllerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kwok controller component: %w", err) - } - logVolumes := runtime.GetLogVolumes(ctx) - kwokControllerExtraVolumes := kwokControllerComponentPatches.ExtraVolumes - kwokControllerExtraVolumes = append(kwokControllerExtraVolumes, logVolumes...) kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ Runtime: conf.Runtime, @@ -632,10 +596,9 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { Verbosity: env.verbosity, NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, EnableCRDs: conf.EnableCRDs, - ExtraArgs: kwokControllerComponentPatches.ExtraArgs, - ExtraVolumes: kwokControllerExtraVolumes, - ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, }) + kwokControllerComponent.Volumes = append(kwokControllerComponent.Volumes, logVolumes...) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kwokControllerComponent) return nil } @@ -675,11 +638,6 @@ func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { return err } - prometheusComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) - prometheusComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(prometheusComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for prometheus component: %w", err) - } prometheusComponent, err := components.BuildPrometheusComponent(components.BuildPrometheusComponentConfig{ Runtime: conf.Runtime, Workdir: env.workdir, @@ -691,9 +649,6 @@ func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { AdminCertPath: env.adminCertPath, AdminKeyPath: env.adminKeyPath, Verbosity: env.verbosity, - ExtraArgs: prometheusComponentPatches.ExtraArgs, - ExtraVolumes: prometheusComponentPatches.ExtraVolumes, - ExtraEnvs: prometheusComponentPatches.ExtraEnvs, }) if err != nil { return err @@ -707,11 +662,6 @@ func (c *Cluster) addDashboard(_ context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options if conf.DashboardPort != 0 { - dashboardComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentDashboard) - dashboardComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(dashboardComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for dashboard component: %w", err) - } dashboardComponent, err := components.BuildDashboardComponent(components.BuildDashboardComponentConfig{ Runtime: conf.Runtime, Workdir: env.workdir, @@ -742,21 +692,14 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) error { return err } - jaegerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentJaeger) - jaegerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(jaegerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for jaeger component: %w", err) - } jaegerComponent, err := components.BuildJaegerComponent(components.BuildJaegerComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Image: conf.JaegerImage, - Version: jaegerVersion, - BindAddress: net.PublicAddress, - Port: conf.JaegerPort, - Verbosity: env.verbosity, - ExtraArgs: jaegerComponentPatches.ExtraArgs, - ExtraVolumes: jaegerComponentPatches.ExtraVolumes, + Runtime: conf.Runtime, + Workdir: env.workdir, + Image: conf.JaegerImage, + Version: jaegerVersion, + BindAddress: net.PublicAddress, + Port: conf.JaegerPort, + Verbosity: env.verbosity, }) if err != nil { return err @@ -766,9 +709,28 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) error { return nil } +func (c *Cluster) preInstall(_ context.Context, env *env) error { + for i, patch := range env.kwokctlConfig.ComponentsPatches { + if len(patch.ExtraVolumes) == 0 { + continue + } + volumes, err := runtime.ExpandVolumesHostPaths(patch.ExtraVolumes) + if err != nil { + return fmt.Errorf("failed to expand host volumes for %q component: %w", patch.Name, err) + } + + env.kwokctlConfig.ComponentsPatches[i].ExtraVolumes = volumes + } + return nil +} + func (c *Cluster) finishInstall(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options + for i := range env.kwokctlConfig.Components { + runtime.ApplyComponentPatches(&env.kwokctlConfig.Components[i], env.kwokctlConfig.ComponentsPatches) + } + // Setup kubeconfig kubeconfigData, err := kubeconfig.EncodeKubeconfig(kubeconfig.BuildKubeconfig(kubeconfig.BuildKubeconfigConfig{ ProjectName: c.Name(), diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 142b02f25..c81449899 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -191,6 +191,11 @@ func (c *Cluster) Install(ctx context.Context) error { return err } + err = c.preInstall(ctx, env) + if err != nil { + return err + } + // This is not necessary when creating a cluster use kind, but in Linux the cluster is created as root, // and the files here may not have permissions when deleted, so we create them first. err = c.setup(ctx, env) @@ -473,21 +478,12 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return err } - kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) - kwokControllerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kwokControllerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kwok controller component: %w", err) - } - logVolumes := runtime.GetLogVolumes(ctx) logVolumes = slices.Map(logVolumes, func(v internalversion.Volume) internalversion.Volume { v.HostPath = path.Join("/var/components/controller", v.HostPath) return v }) - kwokControllerExtraVolumes := kwokControllerComponentPatches.ExtraVolumes - kwokControllerExtraVolumes = append(kwokControllerExtraVolumes, logVolumes...) - kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -507,10 +503,10 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { Verbosity: env.verbosity, NodeLeaseDurationSeconds: 40, EnableCRDs: conf.EnableCRDs, - ExtraArgs: kwokControllerComponentPatches.ExtraArgs, - ExtraVolumes: kwokControllerExtraVolumes, - ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, }) + kwokControllerComponent.Volumes = append(kwokControllerComponent.Volumes, logVolumes...) + + runtime.ApplyComponentPatches(&kwokControllerComponent, env.kwokctlConfig.ComponentsPatches) pod := components.ConvertToPod(kwokControllerComponent) pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ @@ -560,6 +556,8 @@ func (c *Cluster) addDashboard(ctx context.Context, env *env) (err error) { return fmt.Errorf("failed to build dashboard component: %w", err) } + runtime.ApplyComponentPatches(&dashboardComponent, env.kwokctlConfig.ComponentsPatches) + dashboardPod, err := yaml.Marshal(components.ConvertToPod(dashboardComponent)) if err != nil { return fmt.Errorf("failed to marshal dashboard pod: %w", err) @@ -602,25 +600,6 @@ func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { return err } - prometheusComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) - prometheusComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(prometheusComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for prometheus component: %w", err) - } - - prometheusComponentPatches.ExtraVolumes = append(prometheusComponentPatches.ExtraVolumes, - internalversion.Volume{ - HostPath: "/etc/kubernetes/pki/apiserver-etcd-client.crt", - MountPath: "/etc/kubernetes/pki/apiserver-etcd-client.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: "/etc/kubernetes/pki/apiserver-etcd-client.key", - MountPath: "/etc/kubernetes/pki/apiserver-etcd-client.key", - ReadOnly: true, - }, - ) - prometheusComponent, err := components.BuildPrometheusComponent(components.BuildPrometheusComponentConfig{ Runtime: conf.Runtime, Workdir: env.workdir, @@ -632,14 +611,26 @@ func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { AdminCertPath: env.adminCertPath, AdminKeyPath: env.adminKeyPath, Verbosity: env.verbosity, - ExtraArgs: prometheusComponentPatches.ExtraArgs, - ExtraVolumes: prometheusComponentPatches.ExtraVolumes, - ExtraEnvs: prometheusComponentPatches.ExtraEnvs, }) if err != nil { return err } + prometheusComponent.Volumes = append(prometheusComponent.Volumes, + internalversion.Volume{ + HostPath: "/etc/kubernetes/pki/apiserver-etcd-client.crt", + MountPath: "/etc/kubernetes/pki/apiserver-etcd-client.crt", + ReadOnly: true, + }, + internalversion.Volume{ + HostPath: "/etc/kubernetes/pki/apiserver-etcd-client.key", + MountPath: "/etc/kubernetes/pki/apiserver-etcd-client.key", + ReadOnly: true, + }, + ) + + runtime.ApplyComponentPatches(&prometheusComponent, env.kwokctlConfig.ComponentsPatches) + prometheusPod, err := yaml.Marshal(components.ConvertToPod(prometheusComponent)) if err != nil { return fmt.Errorf("failed to marshal prometheus pod: %w", err) @@ -663,26 +654,21 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) error { return err } - jaegerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentJaeger) - jaegerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(jaegerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for jaeger component: %w", err) - } jaegerComponent, err := components.BuildJaegerComponent(components.BuildJaegerComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Image: conf.JaegerImage, - Version: jaegerVersion, - BindAddress: net.PublicAddress, - Port: 16686, - Verbosity: env.verbosity, - ExtraArgs: jaegerComponentPatches.ExtraArgs, - ExtraVolumes: jaegerComponentPatches.ExtraVolumes, + Runtime: conf.Runtime, + Workdir: env.workdir, + Image: conf.JaegerImage, + Version: jaegerVersion, + BindAddress: net.PublicAddress, + Port: 16686, + Verbosity: env.verbosity, }) if err != nil { return err } + runtime.ApplyComponentPatches(&jaegerComponent, env.kwokctlConfig.ComponentsPatches) + jaegerPod, err := yaml.Marshal(components.ConvertToPod(jaegerComponent)) if err != nil { return fmt.Errorf("failed to marshal jaeger pod: %w", err) @@ -697,6 +683,21 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) error { return nil } +func (c *Cluster) preInstall(_ context.Context, env *env) error { + for i, patch := range env.kwokctlConfig.ComponentsPatches { + if len(patch.ExtraVolumes) == 0 { + continue + } + volumes, err := runtime.ExpandVolumesHostPaths(patch.ExtraVolumes) + if err != nil { + return fmt.Errorf("failed to expand host volumes for %q component: %w", patch.Name, err) + } + + env.kwokctlConfig.ComponentsPatches[i].ExtraVolumes = volumes + } + return nil +} + // Up starts the cluster. func (c *Cluster) Up(ctx context.Context) error { config, err := c.Config(ctx) @@ -820,16 +821,13 @@ func (c *Cluster) listAllImages(ctx context.Context) ([]string, error) { if err != nil { return nil, err } - conf := &config.Options - images := []string{conf.KwokControllerImage} - if conf.DashboardPort != 0 { - images = append(images, conf.DashboardImage) - } - if conf.PrometheusPort != 0 { - images = append(images, conf.PrometheusImage) - } - if conf.JaegerPort != 0 { - images = append(images, conf.JaegerImage) + + images := []string{} + for _, component := range config.Components { + if component.Image == "" { + continue + } + images = append(images, component.Image) } return images, nil diff --git a/pkg/kwokctl/runtime/kind/kind.yaml.tpl b/pkg/kwokctl/runtime/kind/kind.yaml.tpl index eff962f40..879c4f6f0 100644 --- a/pkg/kwokctl/runtime/kind/kind.yaml.tpl +++ b/pkg/kwokctl/runtime/kind/kind.yaml.tpl @@ -47,7 +47,7 @@ nodes: {{ if .EtcdExtraArgs }} extraArgs: {{ range .EtcdExtraArgs }} - "{{.Key}}": "{{.Value}}" + "{{ .Key }}": "{{ .Value }}" {{ end }} {{ end }} @@ -71,7 +71,7 @@ nodes: {{ if .ApiserverExtraArgs }} extraArgs: {{ range .ApiserverExtraArgs }} - "{{.Key}}": "{{.Value}}" + "{{ .Key }}": "{{ .Value }}" {{ end }} {{ end }} @@ -94,7 +94,7 @@ nodes: {{ if .ControllerManagerExtraArgs }} extraArgs: {{ range .ControllerManagerExtraArgs }} - "{{.Key}}": "{{.Value}}" + "{{ .Key }}": "{{ .Value }}" {{ end }} {{ end }} @@ -117,7 +117,7 @@ nodes: {{ if .SchedulerExtraArgs }} extraArgs: {{ range .SchedulerExtraArgs }} - "{{.Key}}": "{{.Value}}" + "{{ .Key }}": "{{ .Value }}" {{ end }} {{ end }} diff --git a/pkg/kwokctl/runtime/util.go b/pkg/kwokctl/runtime/util.go index 0b00a74a3..a9aa94aac 100644 --- a/pkg/kwokctl/runtime/util.go +++ b/pkg/kwokctl/runtime/util.go @@ -102,6 +102,26 @@ func GetComponentPatches(conf *internalversion.KwokctlConfiguration, componentNa return componentPatches } +// ApplyComponentPatches applies patches to a component. +func ApplyComponentPatches(component *internalversion.Component, patches []internalversion.ComponentPatches) { + for _, patch := range patches { + applyComponentPatch(component, patch) + } +} + +func applyComponentPatch(component *internalversion.Component, patch internalversion.ComponentPatches) { + if patch.Name != component.Name { + return + } + + component.Volumes = append(component.Volumes, patch.ExtraVolumes...) + component.Envs = append(component.Envs, patch.ExtraEnvs...) + + for _, a := range patch.ExtraArgs { + component.Args = append(component.Args, fmt.Sprintf("--%s=%s", a.Key, a.Value)) + } +} + // ExpandVolumesHostPaths expands relative paths specified in volumes to absolute paths func ExpandVolumesHostPaths(volumes []internalversion.Volume) ([]internalversion.Volume, error) { result := make([]internalversion.Volume, 0, len(volumes)) diff --git a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt index 7071e370c..3ede55566 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt @@ -119,15 +119,15 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml # Add context kwok- to ~/.kube/config -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 >/workdir/clusters//logs/etcd.log 2>&1 & +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 --log-level=debug >/workdir/clusters//logs/etcd.log 2>&1 & echo $! >/workdir/clusters//pids/etcd.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-apiserver --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local >/workdir/clusters//logs/kube-apiserver.log 2>&1 & +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --v=5 >/workdir/clusters//logs/kube-apiserver.log 2>&1 & echo $! >/workdir/clusters//pids/kube-apiserver.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-controller-manager --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & echo $! >/workdir/clusters//pids/kube-controller-manager.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-scheduler --v=5 --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kwok-controller --manage-all-nodes=true --v=-4 --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=40 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=40 --v=-4 >/workdir/clusters//logs/kwok-controller.log 2>&1 & echo $! >/workdir/clusters//pids/kwok-controller.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE prometheus --log.level=debug --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 >/workdir/clusters//logs/prometheus.log 2>&1 & +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE prometheus --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug >/workdir/clusters//logs/prometheus.log 2>&1 & echo $! >/workdir/clusters//pids/prometheus.pid diff --git a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt index 157807405..9b8a7c78f 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt @@ -139,12 +139,12 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml docker network create kwok- --label=com.docker.compose.project=kwok- -docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 -docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 -docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --v=5 +docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --volume=/extras/controller:/extras/tmp --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 --v=-4 +docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/prometheus:/extras/tmp --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config docker start kwok--etcd docker start kwok--kube-apiserver diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt index 916c79d89..263ff296f 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt @@ -91,7 +91,7 @@ nodes: - hostPath: /extras/controller containerPath: /var/components/controller/extras/tmp readOnly: false - - hostPath: ./extras/prometheus + - hostPath: /extras/prometheus containerPath: /var/components/prometheus/extras/tmp readOnly: false - hostPath: /workdir/clusters//prometheus.yaml @@ -110,7 +110,6 @@ spec: - args: - --manage-all-nodes=false - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake - - --v=-4 - --kubeconfig=~/.kube/config - --config=~/.kwok/kwok.yaml - --tls-cert-file=/etc/kubernetes/pki/admin.crt @@ -120,6 +119,7 @@ spec: - --node-port=10247 - --server-address=0.0.0.0:10247 - --node-lease-duration-seconds=40 + - --v=-4 command: - kwok env: @@ -134,48 +134,48 @@ spec: name: kwok-controller resources: {} volumeMounts: - - mountPath: /extras/tmp - name: tmp-controller - mountPath: ~/.kube/config - name: volume-1 + name: volume-0 readOnly: true - mountPath: /etc/kubernetes/pki/ca.crt - name: volume-2 + name: volume-1 readOnly: true - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-3 + name: volume-2 readOnly: true - mountPath: /etc/kubernetes/pki/admin.key - name: volume-4 + name: volume-3 readOnly: true - mountPath: ~/.kwok/kwok.yaml - name: volume-5 + name: volume-4 readOnly: true + - mountPath: /extras/tmp + name: tmp-controller hostNetwork: true restartPolicy: Always securityContext: runAsGroup: 0 runAsUser: 0 volumes: - - hostPath: - path: /extras/controller - type: DirectoryOrCreate - name: tmp-controller - hostPath: path: /etc/kubernetes/admin.conf - name: volume-1 + name: volume-0 - hostPath: path: /etc/kubernetes/pki/ca.crt - name: volume-2 + name: volume-1 - hostPath: path: /etc/kubernetes/pki/admin.crt - name: volume-3 + name: volume-2 - hostPath: path: /etc/kubernetes/pki/admin.key - name: volume-4 + name: volume-3 - hostPath: path: /etc/kwok/kwok.yaml - name: volume-5 + name: volume-4 + - hostPath: + path: /extras/controller + type: DirectoryOrCreate + name: tmp-controller status: {} EOF cat </workdir/clusters//manifests/prometheus.yaml @@ -188,9 +188,9 @@ metadata: spec: containers: - args: - - --log.level=debug - --config.file=/etc/prometheus/prometheus.yaml - --web.listen-address=0.0.0.0:9090 + - --log.level=debug command: - prometheus env: @@ -204,23 +204,23 @@ spec: hostPort: 9090 resources: {} volumeMounts: - - mountPath: /extras/tmp - name: tmp-prometheus - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + - mountPath: /etc/prometheus/prometheus.yaml + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt name: volume-1 readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + - mountPath: /etc/kubernetes/pki/admin.key name: volume-2 readOnly: true - - mountPath: /etc/prometheus/prometheus.yaml + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt name: volume-3 readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key name: volume-4 readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-5 - readOnly: true + - mountPath: /extras/tmp + name: tmp-prometheus hostNetwork: true restartPolicy: Always securityContext: @@ -228,24 +228,24 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /extras/prometheus - type: DirectoryOrCreate - name: tmp-prometheus + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: volume-0 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.crt + path: /etc/kubernetes/pki/admin.crt name: volume-1 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.key + path: /etc/kubernetes/pki/admin.key name: volume-2 - hostPath: - path: /var/components/prometheus/etc/prometheus/prometheus.yaml + path: /etc/kubernetes/pki/apiserver-etcd-client.crt name: volume-3 - hostPath: - path: /etc/kubernetes/pki/admin.crt + path: /etc/kubernetes/pki/apiserver-etcd-client.key name: volume-4 - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-5 + path: /extras/prometheus + type: DirectoryOrCreate + name: tmp-prometheus status: {} EOF cat </workdir/clusters//prometheus.yaml diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt index e0ba2265d..9342121a8 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt @@ -271,19 +271,19 @@ spec: hostPort: 9090 resources: {} volumeMounts: - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + - mountPath: /etc/prometheus/prometheus.yaml name: volume-0 readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + - mountPath: /etc/kubernetes/pki/admin.crt name: volume-1 readOnly: true - - mountPath: /etc/prometheus/prometheus.yaml + - mountPath: /etc/kubernetes/pki/admin.key name: volume-2 readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt name: volume-3 readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key name: volume-4 readOnly: true hostNetwork: true @@ -293,19 +293,19 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.crt + path: /var/components/prometheus/etc/prometheus/prometheus.yaml name: volume-0 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.key + path: /etc/kubernetes/pki/admin.crt name: volume-1 - hostPath: - path: /var/components/prometheus/etc/prometheus/prometheus.yaml + path: /etc/kubernetes/pki/admin.key name: volume-2 - hostPath: - path: /etc/kubernetes/pki/admin.crt + path: /etc/kubernetes/pki/apiserver-etcd-client.crt name: volume-3 - hostPath: - path: /etc/kubernetes/pki/admin.key + path: /etc/kubernetes/pki/apiserver-etcd-client.key name: volume-4 status: {} EOF diff --git a/test/kwokctl/testdata/kind/create_cluster_with_extra.txt b/test/kwokctl/testdata/kind/create_cluster_with_extra.txt index 4fef5532f..85b2a0304 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_extra.txt @@ -91,7 +91,7 @@ nodes: - hostPath: /extras/controller containerPath: /var/components/controller/extras/tmp readOnly: false - - hostPath: ./extras/prometheus + - hostPath: /extras/prometheus containerPath: /var/components/prometheus/extras/tmp readOnly: false - hostPath: /workdir/clusters//prometheus.yaml @@ -110,7 +110,6 @@ spec: - args: - --manage-all-nodes=false - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake - - --v=-4 - --kubeconfig=~/.kube/config - --config=~/.kwok/kwok.yaml - --tls-cert-file=/etc/kubernetes/pki/admin.crt @@ -120,6 +119,7 @@ spec: - --node-port=10247 - --server-address=0.0.0.0:10247 - --node-lease-duration-seconds=40 + - --v=-4 command: - kwok env: @@ -134,48 +134,48 @@ spec: name: kwok-controller resources: {} volumeMounts: - - mountPath: /extras/tmp - name: tmp-controller - mountPath: ~/.kube/config - name: volume-1 + name: volume-0 readOnly: true - mountPath: /etc/kubernetes/pki/ca.crt - name: volume-2 + name: volume-1 readOnly: true - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-3 + name: volume-2 readOnly: true - mountPath: /etc/kubernetes/pki/admin.key - name: volume-4 + name: volume-3 readOnly: true - mountPath: ~/.kwok/kwok.yaml - name: volume-5 + name: volume-4 readOnly: true + - mountPath: /extras/tmp + name: tmp-controller hostNetwork: true restartPolicy: Always securityContext: runAsGroup: 0 runAsUser: 0 volumes: - - hostPath: - path: /extras/controller - type: DirectoryOrCreate - name: tmp-controller - hostPath: path: /etc/kubernetes/admin.conf - name: volume-1 + name: volume-0 - hostPath: path: /etc/kubernetes/pki/ca.crt - name: volume-2 + name: volume-1 - hostPath: path: /etc/kubernetes/pki/admin.crt - name: volume-3 + name: volume-2 - hostPath: path: /etc/kubernetes/pki/admin.key - name: volume-4 + name: volume-3 - hostPath: path: /etc/kwok/kwok.yaml - name: volume-5 + name: volume-4 + - hostPath: + path: /extras/controller + type: DirectoryOrCreate + name: tmp-controller status: {} EOF cat </workdir/clusters//manifests/prometheus.yaml @@ -188,9 +188,9 @@ metadata: spec: containers: - args: - - --log.level=debug - --config.file=/etc/prometheus/prometheus.yaml - --web.listen-address=0.0.0.0:9090 + - --log.level=debug command: - prometheus env: @@ -204,23 +204,23 @@ spec: hostPort: 9090 resources: {} volumeMounts: - - mountPath: /extras/tmp - name: tmp-prometheus - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + - mountPath: /etc/prometheus/prometheus.yaml + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt name: volume-1 readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + - mountPath: /etc/kubernetes/pki/admin.key name: volume-2 readOnly: true - - mountPath: /etc/prometheus/prometheus.yaml + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt name: volume-3 readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key name: volume-4 readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-5 - readOnly: true + - mountPath: /extras/tmp + name: tmp-prometheus hostNetwork: true restartPolicy: Always securityContext: @@ -228,24 +228,24 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /extras/prometheus - type: DirectoryOrCreate - name: tmp-prometheus + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: volume-0 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.crt + path: /etc/kubernetes/pki/admin.crt name: volume-1 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.key + path: /etc/kubernetes/pki/admin.key name: volume-2 - hostPath: - path: /var/components/prometheus/etc/prometheus/prometheus.yaml + path: /etc/kubernetes/pki/apiserver-etcd-client.crt name: volume-3 - hostPath: - path: /etc/kubernetes/pki/admin.crt + path: /etc/kubernetes/pki/apiserver-etcd-client.key name: volume-4 - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-5 + path: /extras/prometheus + type: DirectoryOrCreate + name: tmp-prometheus status: {} EOF cat </workdir/clusters//prometheus.yaml diff --git a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt index fb7f04bff..2ec2f54f8 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt @@ -271,19 +271,19 @@ spec: hostPort: 9090 resources: {} volumeMounts: - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + - mountPath: /etc/prometheus/prometheus.yaml name: volume-0 readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + - mountPath: /etc/kubernetes/pki/admin.crt name: volume-1 readOnly: true - - mountPath: /etc/prometheus/prometheus.yaml + - mountPath: /etc/kubernetes/pki/admin.key name: volume-2 readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt name: volume-3 readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key name: volume-4 readOnly: true hostNetwork: true @@ -293,19 +293,19 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.crt + path: /var/components/prometheus/etc/prometheus/prometheus.yaml name: volume-0 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.key + path: /etc/kubernetes/pki/admin.crt name: volume-1 - hostPath: - path: /var/components/prometheus/etc/prometheus/prometheus.yaml + path: /etc/kubernetes/pki/admin.key name: volume-2 - hostPath: - path: /etc/kubernetes/pki/admin.crt + path: /etc/kubernetes/pki/apiserver-etcd-client.crt name: volume-3 - hostPath: - path: /etc/kubernetes/pki/admin.key + path: /etc/kubernetes/pki/apiserver-etcd-client.key name: volume-4 status: {} EOF diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt index 6834ed259..1573b6836 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt @@ -139,12 +139,12 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml nerdctl network create kwok- --label=com.docker.compose.project=kwok- -nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 -nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 -nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --v=5 +nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --volume=/extras/controller:/extras/tmp --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 --v=-4 +nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/prometheus:/extras/tmp --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config nerdctl start kwok--etcd nerdctl start kwok--kube-apiserver diff --git a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt index 868c8ed95..5ece10de8 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt @@ -139,12 +139,12 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- -podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 -podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 -podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --v=5 +podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --volume=/extras/controller:/extras/tmp --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 --v=-4 +podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/prometheus:/extras/tmp --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config podman start kwok--etcd podman start kwok--kube-apiserver