GoogleCloudPlatform · abheda-crest · Nov 8, 2024 · Nov 8, 2024 · Nov 12, 2024 · Dec 17, 2024
@@ -41,8 +41,11 @@ kubectl scale --replicas=0 deployment/test-load-one-secret
 Additional subcommands:
 
 * `./script/load-test.sh single` - a deployment where the pod references a `SecretProviderClass` with only 1 secret.
-* `./script/load-test.sh many` - a deployment where the pod references a `SecretProviderClass` with 50 secrets.
-* `./script/load-test.sh seed` - creates 50 secrets for use with `many`
+* `./script/load-test.sh many_50` - a deployment where the pod references a `SecretProviderClass` with 50 secrets.
+* `./script/load-test.sh many_100` - a deployment where the pod references a `SecretProviderClass` with 100 secrets.
+* `./script/load-test.sh seed_with_50` - creates 50 secrets for use with `many_50`
+* `./script/load-test.sh seed_with_100` - creates 100 secrets for use with `many_100`
+
 
 Metric of interest:
 

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"syscall"
 	"time"
 
@@ -61,8 +62,7 @@ var (
 	_                     = flag.Bool("write_secrets", false, "[unused]")
 	smConnectionPoolSize  = flag.Int("sm_connection_pool_size", 5, "size of the connection pool for the secret manager API client")
 	iamConnectionPoolSize = flag.Int("iam_connection_pool_size", 5, "size of the connection pool for the IAM API client")
-
-	version = "dev"
+	version               = "dev"
 )
 
 func main() {
@@ -103,6 +103,24 @@ func main() {
 		klog.Fatal("failed to read kubeconfig")
 	}
 	rc.ContentType = runtime.ContentTypeProtobuf
+	// If QPS and Burst are provided by environment variables, configure QPS and Burst capacity for SA token retrieval to support higher request rates.
+	if os.Getenv("K8S_CLIENT_QPS") != "" && os.Getenv("K8S_CLIENT_BURST") != "" {
+		qps, err := strconv.ParseFloat(os.Getenv("K8S_CLIENT_QPS"), 32)
+		if err != nil {
+			klog.ErrorS(err, "failed to convert qps from string to float")
+			return
+		}
+		rc.QPS = float32(qps)
+		klog.InfoS("Setting QPS value for k8s client configuration", "K8S_CLIENT_QPS", qps)
+
+		burst, err := strconv.Atoi(os.Getenv("K8S_CLIENT_BURST"))
+		if err != nil {
+			klog.ErrorS(err, "failed to convert burst from string to integer")
+			return
+		}
+		rc.Burst = burst
+		klog.InfoS("Setting Burst capacity for k8s client configuration", "K8S_CLIENT_BURST", burst)
+	}
 
 	clientset, err := kubernetes.NewForConfig(rc)
 	if err != nil {

@@ -22,17 +22,28 @@ set -x          # Print each command as it is run
 # Usage ./scripts/load-test.sh <COMMAND>
 #
 # See docs/testing-notes.md for full details
-if [ "$1" == "many"  ]; then
-    sed "s/\$PROJECT_ID/${PROJECT_ID}/g;s/\$TEST_SECRET_ID/${TEST_SECRET_ID}/g" test/e2e/templates/load-many-secrets.yaml.tmpl | kubectl apply -f -
+if [ "$1" == "many_50"  ]; then
+    sed "s/\$PROJECT_ID/${PROJECT_ID}/g;s/\$TEST_SECRET_ID/${TEST_SECRET_ID}/g" test/e2e/templates/load-50-secrets.yaml.tmpl | kubectl apply -f -
+elif [ "$1" == "many_100" ]; then
+    sed "s/\$PROJECT_ID/${PROJECT_ID}/g;s/\$TEST_SECRET_ID/${TEST_SECRET_ID}/g" test/e2e/templates/load-100-secrets.yaml.tmpl | kubectl apply -f -
 elif [ "$1" == "single" ]; then
     sed "s/\$PROJECT_ID/${PROJECT_ID}/g;s/\$TEST_SECRET_ID/${TEST_SECRET_ID}/g" test/e2e/templates/load-one-secret.yaml.tmpl | kubectl apply -f -
-elif [ "$1" == "seed" ]; then
+elif [ "$1" == "seed_with_50" ]; then
     for i in {1..50}; do
         printf "s3cr3t" | gcloud secrets create ${TEST_SECRET_ID}-${i} --data-file=- || true
         gcloud secrets add-iam-policy-binding ${TEST_SECRET_ID}-${i} \
            --member=serviceAccount:$PROJECT_ID.svc.id.goog[default/test-cluster-sa] \
            --role=roles/secretmanager.secretAccessor
         # give the API a rest between creates
         sleep 1
+    done    
+elif [ "$1" == "seed_with_100" ]; then
+    for i in {1..100}; do
+        printf "s3cr3t" | gcloud secrets create ${TEST_SECRET_ID}-${i} --data-file=- || true
+        gcloud secrets add-iam-policy-binding ${TEST_SECRET_ID}-${i} \
+           --member=serviceAccount:$PROJECT_ID.svc.id.goog[default/test-cluster-sa] \
+           --role=roles/secretmanager.secretAccessor
+        # give the API a rest between creates
+        sleep 1
     done
 fi
@@ -0,0 +1,264 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: test-cluster-sa
+---
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: gcp-test-load-100-secrets
+spec:
+  provider: gcp
+  parameters:
+    secrets: |
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-1/versions/latest"
+        path: "secret-1"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-2/versions/latest"
+        path: "secret-2"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-3/versions/latest"
+        path: "secret-3"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-4/versions/latest"
+        path: "secret-4"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-5/versions/latest"
+        path: "secret-5"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-6/versions/latest"
+        path: "secret-6"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-7/versions/latest"
+        path: "secret-7"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-8/versions/latest"
+        path: "secret-8"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-9/versions/latest"
+        path: "secret-9"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-10/versions/latest"
+        path: "secret-10"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-11/versions/latest"
+        path: "secret-11"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-12/versions/latest"
+        path: "secret-12"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-13/versions/latest"
+        path: "secret-13"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-14/versions/latest"
+        path: "secret-14"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-15/versions/latest"
+        path: "secret-15"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-16/versions/latest"
+        path: "secret-16"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-17/versions/latest"
+        path: "secret-17"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-18/versions/latest"
+        path: "secret-18"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-19/versions/latest"
+        path: "secret-19"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-20/versions/latest"
+        path: "secret-20"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-21/versions/latest"
+        path: "secret-21"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-22/versions/latest"
+        path: "secret-22"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-23/versions/latest"
+        path: "secret-23"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-24/versions/latest"
+        path: "secret-24"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-25/versions/latest"
+        path: "secret-25"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-26/versions/latest"
+        path: "secret-26"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-27/versions/latest"
+        path: "secret-27"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-28/versions/latest"
+        path: "secret-28"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-29/versions/latest"
+        path: "secret-29"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-30/versions/latest"
+        path: "secret-30"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-31/versions/latest"
+        path: "secret-31"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-32/versions/latest"
+        path: "secret-32"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-33/versions/latest"
+        path: "secret-33"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-34/versions/latest"
+        path: "secret-34"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-35/versions/latest"
+        path: "secret-35"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-36/versions/latest"
+        path: "secret-36"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-37/versions/latest"
+        path: "secret-37"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-38/versions/latest"
+        path: "secret-38"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-39/versions/latest"
+        path: "secret-39"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-40/versions/latest"
+        path: "secret-40"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-41/versions/latest"
+        path: "secret-41"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-42/versions/latest"
+        path: "secret-42"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-43/versions/latest"
+        path: "secret-43"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-44/versions/latest"
+        path: "secret-44"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-45/versions/latest"
+        path: "secret-45"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-46/versions/latest"
+        path: "secret-46"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-47/versions/latest"
+        path: "secret-47"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-48/versions/latest"
+        path: "secret-48"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-49/versions/latest"
+        path: "secret-49"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-50/versions/latest"
+        path: "secret-50"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-51/versions/latest"
+        path: "secret-51"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-52/versions/latest"
+        path: "secret-52"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-53/versions/latest"
+        path: "secret-53"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-54/versions/latest"
+        path: "secret-54"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-55/versions/latest"
+        path: "secret-55"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-56/versions/latest"
+        path: "secret-56"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-57/versions/latest"
+        path: "secret-57"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-58/versions/latest"
+        path: "secret-58"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-59/versions/latest"
+        path: "secret-59"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-60/versions/latest"
+        path: "secret-60"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-61/versions/latest"
+        path: "secret-61"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-62/versions/latest"
+        path: "secret-62"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-63/versions/latest"
+        path: "secret-63"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-64/versions/latest"
+        path: "secret-64"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-65/versions/latest"
+        path: "secret-65"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-66/versions/latest"
+        path: "secret-66"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-67/versions/latest"
+        path: "secret-67"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-68/versions/latest"
+        path: "secret-68"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-69/versions/latest"
+        path: "secret-69"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-70/versions/latest"
+        path: "secret-70"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-71/versions/latest"
+        path: "secret-71"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-72/versions/latest"
+        path: "secret-72"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-73/versions/latest"
+        path: "secret-73"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-74/versions/latest"
+        path: "secret-74"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-75/versions/latest"
+        path: "secret-75"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-76/versions/latest"
+        path: "secret-76"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-77/versions/latest"
+        path: "secret-77"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-78/versions/latest"
+        path: "secret-78"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-79/versions/latest"
+        path: "secret-79"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-80/versions/latest"
+        path: "secret-80"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-81/versions/latest"
+        path: "secret-81"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-82/versions/latest"
+        path: "secret-82"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-83/versions/latest"
+        path: "secret-83"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-84/versions/latest"
+        path: "secret-84"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-85/versions/latest"
+        path: "secret-85"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-86/versions/latest"
+        path: "secret-86"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-87/versions/latest"
+        path: "secret-87"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-88/versions/latest"
+        path: "secret-88"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-89/versions/latest"
+        path: "secret-88"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-90/versions/latest"
+        path: "secret-90"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-91/versions/latest"
+        path: "secret-91"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-92/versions/latest"
+        path: "secret-92"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-93/versions/latest"
+        path: "secret-93"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-94/versions/latest"
+        path: "secret-94"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-95/versions/latest"
+        path: "secret-95"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-96/versions/latest"
+        path: "secret-96"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-97/versions/latest"
+        path: "secret-97"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-98/versions/latest"
+        path: "secret-98"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-99/versions/latest"
+        path: "secret-99"
+      - resourceName: "projects/$PROJECT_ID/secrets/$TEST_SECRET_ID-100/versions/latest"
+        path: "secret-100"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-load-100-secrets
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: test-load-100-secrets
+  template:
+    metadata:
+      labels:
+        app: test-load-100-secrets
+    spec:
+      serviceAccountName: test-cluster-sa
+      containers:
+      - image: gcr.io/google.com/cloudsdktool/cloud-sdk:slim
+        imagePullPolicy: IfNotPresent
+        name: test-secret-mounter
+        resources:
+          requests:
+            cpu: 1m
+        stdin: true
+        stdinOnce: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        tty: true
+        volumeMounts:
+        - mountPath: "/var/gcp-test-secrets"
+          name: gcp-test-secrets
+      volumes:
+      - name: gcp-test-secrets
+        csi:
+          driver: secrets-store.csi.k8s.io
+          readOnly: true
+          volumeAttributes:
+              secretProviderClass: "gcp-test-load-100-secrets"
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,7 +19,7 @@ metadata:
 apiVersion: secrets-store.csi.x-k8s.io/v1
 kind: SecretProviderClass
 metadata:
-  name: gcp-test-load-many-secrets
+  name: gcp-test-load-50-secrets
 spec:
   provider: gcp
   parameters:
@@ -128,16 +128,16 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: test-load-many-secrets
+  name: test-load-50-secrets
 spec:
   replicas: 0
   selector:
     matchLabels:
-      app: test-load-many-secrets
+      app: test-load-50-secrets
   template:
     metadata:
       labels:
-        app: test-load-many-secrets
+        app: test-load-50-secrets
     spec:
       serviceAccountName: test-cluster-sa
       containers:
@@ -161,4 +161,4 @@ spec:
           driver: secrets-store.csi.k8s.io
           readOnly: true
           volumeAttributes:
-              secretProviderClass: "gcp-test-load-many-secrets"
+              secretProviderClass: "gcp-test-load-50-secrets"