feat: meta-trigger support of multiple GCP projects

.cicd/apply-triggers.cloudbuild.yaml → .cicd/jobs/apply-triggers.cloudbuild.yaml

@@ -1,3 +1,6 @@
+tags:
+  - "meta-triggers"
+
 steps:
   - name: gcr.io/$PROJECT_ID/cancelot
     id: "Cancelot"
@@ -13,4 +16,3 @@ steps:
       - "Cancelot"
     args:
       - .cicd
-tags: ["meta-triggers"]

.cicd/build-custom-builders.cloudbuild.yaml → .cicd/jobs/build-custom-builders.cloudbuild.yaml

@@ -1,3 +1,6 @@
+tags:
+  - "cloud-builders"
+
 steps:
   - name: gcr.io/$PROJECT_ID/cancelot
     id: "Cancelot"
@@ -15,4 +18,3 @@ steps:
       - ".cicd/builders/custom-builders.yaml"
       - "--timeout=900"
       - "--async"
-tags: ["cloud-builders"]

.cicd/jobs/lint.cloudbuild.yaml

@@ -0,0 +1,30 @@
+tags:
+  - "lint"
+
+steps:
+  - name: gcr.io/$PROJECT_ID/cancelot
+    id: "Cancelot"
+    waitFor:
+      - "-"
+    args:
+      - "--current_build_id"
+      - "$BUILD_ID"
+      - "--branch_name"
+      - "$BRANCH_NAME"
+      - "--same_trigger_only"
+
+  - name: gcr.io/$PROJECT_ID/shellcheck
+    id: "Lint: Shellcheck"
+    waitFor:
+      - "Cancelot"
+    args:
+      - ./meta-cloud-builder/run.bash
+      - ./trigger-sync/run.bash
+
+  - name: mvdan/shfmt:latest-alpine
+    id: "Lint: shell format"
+    waitFor:
+      - "Cancelot"
+    args:
+      - "-d"
+      - "."

.cicd/triggers/build-trigger-sync.trigger.yaml

@@ -0,0 +1,10 @@
+name: build-trigger-sync
+description: "Build trigger-sync builder on change"
+github:
+  owner: jthegedus
+  name: meta-cloud-builders
+  pullRequest:
+    branch: .*
+filename: trigger-sync/cloudbuild.yaml
+includedFiles:
+  - trigger-sync/**

.cicd/triggers/lint.trigger.yaml

@@ -5,4 +5,4 @@ github:
   name: meta-cloud-builders
   pullRequest:
     branch: .*
-filename: .cicd/lint.cloudbuild.yaml
+filename: .cicd/jobs/lint.cloudbuild.yaml

.cicd/triggers/run-meta-builder.trigger.yaml

@@ -5,7 +5,7 @@ github:
   name: meta-cloud-builders
   push:
     branch: master
-filename: .cicd/build-custom-builders.cloudbuild.yaml
+filename: .cicd/jobs/build-custom-builders.cloudbuild.yaml
 includedFiles:
-  - .cicd/build-custom-builders.cloudbuild.yaml # the Cloud Build job file
+  - .cicd/jobs/build-custom-builders.cloudbuild.yaml # the Cloud Build job file
   - .cicd/builders/custom-builders.yaml # the config file

.cicd/triggers/run-meta-triggers.trigger.yaml

@@ -5,6 +5,6 @@ github:
   name: meta-cloud-builders
   push:
     branch: master
-filename: .cicd/apply-triggers.cloudbuild.yaml
+filename: .cicd/jobs/apply-triggers.cloudbuild.yaml
 includedFiles:
   - .cicd/**

.editorconfig

@@ -0,0 +1,11 @@
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+
+[*.bash]
+indent_style = tab
+indent_size = 4
+end_of_line = lf
+insert_final_newline = true

README.md

@@ -75,7 +75,7 @@ github:
     branch: master
 filename: .cicd/builders.cloudbuild.yaml
 includedFiles:
-  - .cicd/builders.cloudbuild.yaml      # the Cloud Build job file
+  - .cicd/builders.cloudbuild.yaml # the Cloud Build job file
   - .cicd/builders/custom-builders.yaml # the config file
 ```
 
@@ -108,6 +108,7 @@ gcloud scheduler jobs create http \
 Just fill in `PROJECTID`, `TRIGGERID` and create a Service Account and fill in the `EMAIL_ADDRESS` accordingly.
 
 Suggested schedule intervals:
+
 - daily: `0 0 * * *`
 - every sunday: `0 0 * * SUN`
 

meta-cloud-builder/Dockerfile

@@ -1,12 +1,24 @@
-FROM gcr.io/cloud-builders/git
+# hadolint ignore=DL3007
+FROM gcr.io/cloud-builders/git:latest
 
+ENV JQ_VERION="1.5+dfsg-1ubuntu0.1"
+ENV YQ_VERSION="3.3.0"
+ENV WGET_VERSION="1.17.1-1ubuntu1.5"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# setup jq
 RUN apt-get -qq -y update \
-    && apt-get -qq -y install jq
+	&& apt-get -qq -y --no-install-recommends install "jq=${JQ_VERION}" "wget=${WGET_VERSION}" \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
 
-ADD yq /usr/bin
+# setup yq
+RUN wget -qO- "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" > /usr/bin/yq
 RUN chmod +x /usr/bin/yq
 
-ADD run.sh /usr/bin
-RUN chmod +x /usr/bin/run.sh
+# setup entrypoint script
+COPY run.bash /usr/bin
+RUN chmod +x /usr/bin/run.bash
 
-ENTRYPOINT [ "/usr/bin/run.sh" ]
+ENTRYPOINT [ "/usr/bin/run.bash" ]

meta-cloud-builder/cloudbuild.yaml

@@ -1,9 +1,17 @@
+tags:
+  - "cloud-builders-community"
+  - "meta-cloud-builder"
+
 steps:
   - name: gcr.io/cloud-builders/docker
     id: "Build: meta builder image"
     waitFor:
       - "-"
-    args: ["build", "-t", "gcr.io/$PROJECT_ID/meta-cloud-builder", "."]
+    args:
+      - "build"
+      - "-t"
+      - "gcr.io/$PROJECT_ID/meta-cloud-builder"
+      - "meta-cloud-builder"
+
 images:
   - gcr.io/$PROJECT_ID/meta-cloud-builder
-tags: ["cloud-builders-community"]

meta-cloud-builder/run.bash

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+config_file=""
+workspace_dir="/workspace/builder_repos"
+
+if [ ! -f "$1" ]; then
+	printf "[error] first arg should be a config file. File %s does NOT exist\n" "$1"
+	exit 1
+fi
+
+if [[ "$1" == *.yaml ]]; then
+	printf "[info] converting .yaml file to .json\n"
+	yq r -j "$1" >"${1%.yaml}.json"
+	config_file="${1%.yaml}.json"
+else
+	config_file="$1"
+fi
+
+if [[ "$config_file" != *.json ]]; then
+	printf "[error] invalid config file %s File must be .json or .yaml\n" "$config_file"
+	exit 1
+fi
+
+printf "[info] loading from config file: %s\n" "$config_file"
+
+# for each repo, clone to workspace
+jq -r '.[] | .repo' "$config_file" |
+	while read -r url; do
+		printf "[info] cloning repo %s\n" "$url"
+		final_path="${url//https:\/\/github.com\//}"
+		git clone "$url" "$workspace_dir/$final_path"
+
+		# for each builder in the config for this repo, build with gcloud
+		jq -r --arg repository "$url" '.[] | select(.repo == $repository) | .builders | .[]' "$config_file" |
+			while read -r builder; do
+				printf "\n[info] building %s\n" "$builder"
+				set -x
+				gcloud builds submit \
+					"${@:2}" \
+					--config "$workspace_dir/$final_path/$builder/cloudbuild.yaml" \
+					"$workspace_dir/$final_path/$builder"
+				set +x
+			done
+	done

meta-cloud-builder/test/cloudbuild.yaml

@@ -1,3 +1,7 @@
+tags:
+  - "cloud-builders-community"
+  - "test"
+
 steps:
   # sanity check
   - name: gcr.io/$PROJECT_ID/meta-cloud-builder
@@ -38,4 +42,3 @@ steps:
       - list
       - --filter
       - cache
-tags: ["cloud-builders-community", "test"]

meta-cloud-builder/test/test-config.json

@@ -1,9 +1,9 @@
 [
-    {
-        "builders": [
-            "cancelot",
-            "cache"
-        ],
-        "repo": "https://github.com/GoogleCloudPlatform/cloud-builders-community"
-    }
-]
+	{
+		"builders": [
+			"cancelot",
+			"cache"
+		],
+		"repo": "https://github.com/GoogleCloudPlatform/cloud-builders-community"
+	}
+]