diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index 85cba014..c5a185a4 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -26,7 +26,544 @@ instance_groups: kubernetes: ((tls-kubernetes)) kubernetes-dashboard: ((tls-kubernetes-dashboard)) metrics-server: ((tls-metrics-server)) + specs: + coredns: + - name: service-account + value: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + - name: cluster-role + value: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - name: cluster-role-binding + value: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + - name: config-map + value: + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: | + .:53 { + errors + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + upstream + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + proxy . /etc/resolv.conf { + policy sequential # needed for workloads to be able to use BOSH-DNS + } + cache 30 + loop + reload + loadbalance + } + - name: deployment + value: + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" + spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + spec: + priorityClassName: system-cluster-critical # Added for Guaranteed Scheduling + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: coredns + image: coredns/coredns:1.3.1 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + # NOTE: Security Context is denied unless privileged containers + # are enabled. Once security context can be separated from + # allow-privileged in the release, then this should become + # conditional. + # securityContext: + # allowPrivilegeEscalation: false + # capabilities: + # add: + # - NET_BIND_SERVICE + # drop: + # - all + # readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - name: service + value: + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + spec: + selector: + k8s-app: kube-dns + clusterIP: 10.100.200.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + kubernetes-dashboard: | + # Copyright 2017 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # Configuration to deploy release version of the Dashboard UI compatible with + # Kubernetes 1.8. + # + # Example usage: kubectl create -f + + --- + # ------------------- Dashboard Service Account ------------------- # + + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + + --- + # ------------------- Dashboard Role & Role Binding ------------------- # + + kind: Role + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-minimal + namespace: kube-system + rules: + # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret. Added separately from Addons Spec + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: kubernetes-dashboard-minimal + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard-minimal + subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system + + --- + # ------------------- Dashboard Config ------------------- # + + apiVersion: v1 + kind: ConfigMap + metadata: + name: kubernetes-dashboard-settings + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + --- + # ------------------- Dashboard Deployment ------------------- # + + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + spec: + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + spec: + priorityClassName: system-cluster-critical + containers: + - name: kubernetes-dashboard + image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1 + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 50m + memory: 100Mi + ports: + - containerPort: 8443 + protocol: TCP + args: + - --default-cert-dir=/certs + - --tls-cert-file=kubernetes-dashboard-cert + - --tls-key-file=kubernetes-dashboard-key + # Uncomment the following line + # Create on-disk volume to store exec logs + volumeMounts: + - mountPath: /tmp + name: tmp-volume + - mountPath: /certs + name: kubernetes-dashboard-certs + readOnly: true + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + # Comment the following tolerations if Dashboard must not be deployed on master + # tolerations: + # - key: node-role.kubernetes.io/master + # effect: NoSchedule + + --- + # ------------------- Dashboard Service ------------------- # + + apiVersion: v1 + kind: Service + metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + spec: + selector: + k8s-app: kubernetes-dashboard + ports: + - port: 443 + targetPort: 8443 + type: NodePort + metrics-server: | + --- + # ------------------- Auth Delegator ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + + --- + # ------------------- Auth Reader ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + + --- + # ------------------- Metrics APIService ------------------- # + + apiVersion: apiregistration.k8s.io/v1beta1 + kind: APIService + metadata: + name: v1beta1.metrics.k8s.io + spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 + + --- + # ------------------- Metrics Server Deployment ------------------- # + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: metrics-server + namespace: kube-system + --- + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server + version: v0.3.1 + spec: + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + - name: metrics-server-secrets + secret: + secretName: metrics-server-certs + containers: + - name: metrics-server + image: k8s.gcr.io/metrics-server-amd64:v0.3.3 + imagePullPolicy: IfNotPresent + command: + - /metrics-server + - --kubelet-preferred-address-types=InternalIP + - --kubelet-insecure-tls + - --client-ca-file=/var/run/kubernetes/client-ca.crt + - --requestheader-client-ca-file=/var/run/kubernetes/requestheader-client-ca.crt + - --tls-cert-file=/var/run/kubernetes/client.crt + - --tls-private-key-file=/var/run/kubernetes/client.key + ports: + - containerPort: 443 + name: https + protocol: TCP + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: metrics-server-secrets + mountPath: /var/run/kubernetes + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + + --- + # ------------------- Metrics Server Service ------------------- # + + apiVersion: v1 + kind: Service + metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" + spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: https + + --- + # ------------------- Resource Reader ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - create + - get + - apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - deployments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system release: kubo + - name: kubernetes-dependencies + release: kubernetes lifecycle: errand name: apply-addons networks: @@ -49,6 +586,8 @@ instance_groups: certificate: ((tls-etcdctl-flanneld.certificate)) private_key: ((tls-etcdctl-flanneld.private_key)) release: kubo + - name: kubernetes-dependencies + release: kubernetes - name: kube-apiserver properties: admin-password: ((kubo-admin-password)) @@ -293,9 +832,174 @@ instance_groups: - name: kubernetes-roles properties: admin-password: ((kubo-admin-password)) - admin-username: admin tls: kubernetes: ((tls-kubernetes)) + post-start-policies: + - name: cluster-admin + value: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:admin + subjects: + - kind: User + name: admin + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io + - name: kube-proxy + value: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kube-proxy + subjects: + - kind: User + name: kube-proxy + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: system:node-proxier + apiGroup: rbac.authorization.k8s.io + - name: kube-system-podsecuritypolicy + value: | + --- + apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: kube-system-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + allowedCapabilities: + - '*' + # Allow core volume types. + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + hostIPC: true + hostPID: true + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + # rule: 'MustRunAs' + # ranges: + # # Forbid adding the root group. + # - min: 1 + # max: 65535 + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: psp:kube-system-psp + namespace: kube-system + rules: + - apiGroups: + - extensions + resourceNames: + - kube-system-psp + resources: + - podsecuritypolicies + verbs: + - use + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: psp:kube-system-psp + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: psp:kube-system-psp + subjects: + - kind: ServiceAccount + name: coredns + - kind: ServiceAccount + name: metrics-server + - kind: ServiceAccount + name: kubernetes-dashboard + - name: kubelet-drain + value: | + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet-drain + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "get", "patch", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list", "delete"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: ["apps"] + resources: ["statefulsets", "daemonsets"] + verbs: ["get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get"] + - apiGroups: [""] + resources: ["replicationcontrollers"] + verbs: ["get"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet-drain + subjects: + - kind: User + name: kubelet-drain + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: kubo:internal:kubelet-drain + apiGroup: rbac.authorization.k8s.io + - name: kubelet + value: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet + subjects: + - kind: User + name: kubelet + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: system:node + apiGroup: rbac.authorization.k8s.io release: kubo - name: etcd properties: @@ -373,7 +1077,7 @@ instance_groups: store_dir: /var/vcap/data release: docker - name: kubernetes-dependencies - release: kubo + release: kubernetes - name: kubelet properties: api-token: ((kubelet-password)) @@ -454,6 +1158,8 @@ releases: sha1: c3376e70d6a080054012afa45fae1e9249b2a6d9 url: https://storage.googleapis.com/kubo-precompiled-releases/bpm-1.0.4-ubuntu-xenial-315.64-20190703-011222-636424609.tgz version: 1.0.4 +- name: kubernetes + version: latest stemcells: - alias: default os: ubuntu-xenial diff --git a/manifests/ops-files/addons-spec.yml b/manifests/ops-files/addons-spec.yml index 1e767ae2..a62d3eea 100644 --- a/manifests/ops-files/addons-spec.yml +++ b/manifests/ops-files/addons-spec.yml @@ -1,3 +1,3 @@ - type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/addons-spec? + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs?/addons? value: ((addons-spec)) diff --git a/manifests/ops-files/change-cidrs.yml b/manifests/ops-files/change-cidrs.yml index c8bbcffc..422695a8 100644 --- a/manifests/ops-files/change-cidrs.yml +++ b/manifests/ops-files/change-cidrs.yml @@ -1,8 +1,4 @@ --- -- type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/kubedns-service-ip? - value: ((kubedns_service_ip)) - - type: replace path: /instance_groups/name=master/jobs/name=kube-apiserver/properties/k8s-args/service-cluster-ip-range? value: ((service_cluster_cidr)) @@ -11,6 +7,10 @@ path: /instance_groups/name=worker/jobs/name=kubelet/properties/kubelet-configuration/clusterDNS? value: [((kubedns_service_ip))] +- type: replace + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs/coredns/name=service/value/spec/clusterIP + value: ((kubedns_service_ip)) + - type: replace path: /instance_groups/name=worker/jobs/name=flanneld/properties?/pod-network-cidr? value: ((pod_network_cidr)) diff --git a/manifests/ops-files/iaas/azure/cloud-provider.yml b/manifests/ops-files/iaas/azure/cloud-provider.yml index 644ddc07..a39c36a8 100644 --- a/manifests/ops-files/iaas/azure/cloud-provider.yml +++ b/manifests/ops-files/iaas/azure/cloud-provider.yml @@ -80,3 +80,29 @@ - type: replace path: /instance_groups/name=worker/jobs/name=kube-proxy/properties/cloud-provider? value: azure + +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/cloud-provider? + value: | + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:azure-cloud-provider + rules: + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] + --- + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: ClusterRoleBinding + metadata: + name: system:azure-cloud-provider + roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider + subjects: + - kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system diff --git a/manifests/ops-files/iaas/vsphere/cloud-provider.yml b/manifests/ops-files/iaas/vsphere/cloud-provider.yml index 3e5fa472..2e2818f6 100644 --- a/manifests/ops-files/iaas/vsphere/cloud-provider.yml +++ b/manifests/ops-files/iaas/vsphere/cloud-provider.yml @@ -62,3 +62,20 @@ path: /instance_groups/name=apply-addons/jobs/name=apply-specs/consumes? value: cloud-provider: {from: master-cloud-provider} + +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/cloud-provider? + value: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:vsphere-cloud-provider + subjects: + - kind: ServiceAccount + name: vsphere-cloud-provider + namespace: kube-system + roleRef: + kind: ClusterRole + name: system:node + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/ops-files/misc/dev.yml b/manifests/ops-files/misc/dev.yml index f8ebb609..c3a280bf 100644 --- a/manifests/ops-files/misc/dev.yml +++ b/manifests/ops-files/misc/dev.yml @@ -3,3 +3,9 @@ value: name: kubo version: latest + +- type: replace + path: /releases/name=kubernetes? + value: + name: kubernetes + version: latest diff --git a/manifests/ops-files/non-precompiled-releases.yml b/manifests/ops-files/non-precompiled-releases.yml index fe3d434b..c6c30bc8 100644 --- a/manifests/ops-files/non-precompiled-releases.yml +++ b/manifests/ops-files/non-precompiled-releases.yml @@ -17,3 +17,6 @@ sha1: 41df19697d6a69d2552bc2c132928157fa91abe0 url: https://bosh.io/d/github.com/cloudfoundry-incubator/bpm-release?v=1.0.4 version: 1.0.4 + - name: kubernetes + url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-release.tar.gz + version: 0.0.0-dev.3 diff --git a/manifests/ops-files/windows/add-worker.yml b/manifests/ops-files/windows/add-worker.yml index b15193c1..e3b4789e 100644 --- a/manifests/ops-files/windows/add-worker.yml +++ b/manifests/ops-files/windows/add-worker.yml @@ -21,6 +21,13 @@ url: "https://storage.googleapis.com/kubo-precompiled-releases/kubo-windows-0.31.0-windows2019-2019.2-20190325-131732-878123.tgz" sha1: "05ead5f098611e25a6fc6e5cfb33825cf1c9b8ae" +- type: replace + path: /releases/- + value: + name: "kubernetes-windows" + url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-windows-release.tar.gz + version: 0.0.0-dev.7 + - type: replace path: /addons/- value: