Make throttling params QPS and Burst configurable (#323)

* Make throttling params of the underlying k8s client, QPS and Burst, configurable to users

---------

Co-authored-by: Kai Zhu <kaizuu@amazon.com>
Co-authored-by: Simon Marty <simon.marty0@gmail.com>

README.md

@@ -225,6 +225,12 @@ helm repo add aws-secrets-manager https://aws.github.io/secrets-store-csi-driver
 helm install -n kube-system secrets-provider-aws aws-secrets-manager/secrets-store-csi-driver-provider-aws --set useFipsEndpoint=true
 ```
 
+### Client-Side Rate-Limitting to Kubernetes API server
+
+To mount each secret on each pod, the AWS CSI provider lookups the region of the pod and the role ARN associated with the service account by calling the Kubernetes APIs. You can increase the value of qps and burst if you notice the provider is throttled by client-side limit to the API server.
+
+If you use Helm chart to install the provider, append the `--set-json 'k8sThrottlingParams={"qps": "<custom qps>", "burst": "<custom qps>"}'` flag in the install step.
+
 ### Security Considerations
 
 The AWS Secrets Manager and Config Provider provides compatibility for legacy applications that access secrets as mounted files in the pod. Security conscious applications should use the native AWS APIs to fetch secrets and optionally cache them in memory rather than storing them in the file system.

charts/secrets-store-csi-driver-provider-aws/templates/daemonset.yaml

@@ -31,6 +31,14 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
             - --provider-volume={{ .Values.providerVolume }}
+            {{- if .Values.k8sThrottlingParams }}
+            {{- if .Values.k8sThrottlingParams.qps }}
+            - --qps={{ .Values.k8sThrottlingParams.qps }}
+            {{- end }}
+            {{- if .Values.k8sThrottlingParams.burst }}
+            - --burst={{ .Values.k8sThrottlingParams.burst }}
+            {{- end }}
+            {{- end }}
           resources:
 {{ toYaml .Values.resources | indent 12 }}
           securityContext:

main.go

@@ -22,6 +22,8 @@ import (
 var (
 	endpointDir        = flag.String("provider-volume", "/etc/kubernetes/secrets-store-csi-providers", "Rendezvous directory for provider socket")
 	driverWriteSecrets = flag.Bool("driver-writes-secrets", false, "The driver will do the write instead of the plugin")
+	qps                = flag.Int("qps", 5, "Maximum query per second to the Kubernetes API server. To mount the requested secret on the pod, the AWS CSI provider lookups the region of the pod and the role ARN associated with the service account by calling the K8s APIs. Increase the value if the provider is throttled by client-side limit to the API server.")
+	burst              = flag.Int("burst", 10, "Maximum burst for throttle. To mount the requested secret on the pod, the AWS CSI provider lookups the region of the pod and the role ARN associated with the service account by calling the K8s APIs. Increase the value if the provider is throttled by client-side limit to the API server.")
 )
 
 // Main entry point for the Secret Store CSI driver AWS provider. This main
@@ -58,6 +60,9 @@ func main() {
 		klog.Fatalf("Can not get cluster config. error: %v", err)
 	}
 
+	cfg.QPS = float32(*qps)
+	cfg.Burst = *burst
+
 	clientset, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		klog.Fatalf("Can not initialize kubernetes client. error: %v", err)