-
Notifications
You must be signed in to change notification settings - Fork 17
247 lines (241 loc) · 13.1 KB
/
build-deploy-acc.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
name: Build and Deploy Acceptance
on:
workflow_dispatch:
push:
branches: [ "main" ]
jobs:
build-deeplynx-acc:
runs-on: [ self-hosted ]
environment: acceptance
steps:
- name: Checkout
uses: actions/checkout@v3
with:
path: deeplynx
ref: main
- name: Checkout tools repo
uses: actions/checkout@v3
with:
repository: Digital-Engineering/kubernetes
ref: main
path: kubernetes
token: ${{ secrets.GH_TOKEN }}
- shell: bash
name: ACR build
env:
ACR_SP_USER: ${{ secrets.CI_SP_USER }}
ACR_SP_PASSWORD: ${{ secrets.CI_SP_PASSWORD }}
ACR_REGISTRY: ${{ secrets.CI_REGISTRY }}
ACR_PATH: ${{ secrets.CI_REGISTRY_PATH }}
ACR_SP_TENANT: ${{ secrets.CI_SP_TENANT }}
ACR_SUBSCRIPTION: ${{ secrets.CI_ACR_SUBSCRIPTION }}
NODE_ENV: ${{ secrets.NODE_ENV }}
SHORT_SHA: ${{ steps.vars.outputs.sha_short }}
run: |
cd $GITHUB_WORKSPACE/deeplynx
az cloud set --name AzureUSGovernment
az login --service-principal -u $ACR_SP_USER -p $ACR_SP_PASSWORD --tenant $ACR_SP_TENANT
az account set --subscription $ACR_SUBSCRIPTION
az acr build -r $ACR_REGISTRY -f $GITHUB_WORKSPACE/deeplynx/Dockerfile -t $ACR_PATH:$GITHUB_SHA-latest .
# scan-deeplynx-acc:
# runs-on: [ self-hosted ]
# environment: acceptance
# needs: build-deeplynx-acc
# steps:
# - name: Checkout
# uses: actions/checkout@v3
# with:
# path: deeplynx
# ref: main
# - shell: bash
# name: ACR Get Scan
# env:
# ACR_REGISTRY: ${{ secrets.CI_REGISTRY }}
# ACR_PATH: ${{ secrets.CI_REGISTRY_PATH }}
# SHORT_SHA: ${{ steps.vars.outputs.sha_short }}
# run: |
# imageDigest=$(az acr repository show -n $ACR_REGISTRY -t $ACR_PATH:$GITHUB_SHA-latest | jq --raw-output '.digest')
# healthquery="securityresources
# | where type == 'microsoft.security/assessments/subassessments'
# | where id matches regex '(.+?)/providers/Microsoft.ContainerRegistry/registries/(.+)/providers/Microsoft.Security/assessments/dbd0cb49-b563-45e7-9724-889e799fa648/'
# | extend registryResourceId = tostring(split(id, '/providers/Microsoft.Security/assessments/')[0])
# | extend registryResourceName = tostring(split(registryResourceId, '/providers/Microsoft.ContainerRegistry/registries/')[1])
# | extend imageDigest = tostring(properties.additionalData.imageDigest)
# | extend repository = tostring(properties.additionalData.repositoryName)
# | extend scanFindingSeverity = tostring(properties.status.severity), scanStatus = tostring(properties.status.code)
# | summarize scanFindingSeverityCount = count() by scanFindingSeverity, scanStatus, registryResourceId, registryResourceName, repository, imageDigest
# | summarize severitySummary = make_bag(pack(scanFindingSeverity, scanFindingSeverityCount)) by registryResourceId, registryResourceName, repository, imageDigest, scanStatus
# | where imageDigest contains '$imageDigest'"
# query="SecurityResources
# | where type == 'microsoft.security/assessments'
# | where properties.displayName contains 'Container registry images should have vulnerability findings resolved'
# | summarize by assessmentKey=name //the ID of the assessment
# | join kind=inner (securityresources | where type == 'microsoft.security/assessments/subassessments' | extend assessmentKey = extract('.*assessments/(.+?)/.*',1, id)) on assessmentKey
# | project parse_json(properties)
# | extend description = properties.description,displayName = properties.displayName,resourceId = properties.resourceDetails.id,resourceSource = properties.resourceDetails.source,category = properties.category,severity = properties.status.severity,code = properties.status.code,timeGenerated = properties.timeGenerated,remediation = properties.remediation,impact = properties.impact,vulnId = properties.id,additionalData = properties.additionalData
# | where resourceId contains '$imageDigest'"
# az config set extension.use_dynamic_install=yes_without_prompt
# count=1
# querycount=1
# until false; do
# scanhealth=$(az graph query -q "$healthquery" | jq --raw-output '.data[] | .scanStatus')
# if [[ $scanhealth = 'Healthy' ]]; then
# echo 'Scan returned health'
# break
# elif [[ $scanhealth = 'Unhealthy' ]]; then
# echo "Building report with findings"
# rm -f scanreport.tsv
# echo -e 'severity\tid\tpatchable\tpublished\tregistryhost\treponame\tos\tdisplayname\tdescription\timpact\tcvetitle\tcvelink\tvendorrefrencetitle\tvendorerefrencelink\tscanner\ttype\timagedigest' >>scanreport.tsv
# az graph query -q "$query" | jq --raw-output '.data[] | [.severity, .properties.id, .properties.additionalData.patchable, .properties.additionalData.publishedTime, .properties.additionalData.registryHost, .properties.additionalData.repositoryName, .additionalData.imageDetails.osDetails, .displayName, '.description', .impact, .properties.additionalData.cve[].title, .properties.additionalData.cve[].link,.properties.additionalData.vendorReferences[].title, .properties.additionalData.vendorReferences[].link, .properties.additionalData.scanner,.properties.additionalData.type , .additionalData.imageDigest] | @tsv' >>scanreport.tsv
# break
# elif [[ $count -eq 10 ]]; then
# echo "Image scan not found exiting"
# break
# else
# echo "Scan not complete... waiting $count"
# sleep 30
# count="$((count + 1))"
# fi
# done
# echo -e "run:\nreg add "HKEY_LOCAL_MACHINE\SOFTWARE\Clients\Spreadsheet\Microsoft Excel\Capabilities\FileAssociations" /v ".tsv" /t REG_SZ /d "Excel.SLK" /f\nto associate .tsv with excel. You should only need to do this once." >> SCAN_READ_ME.txt
# - uses: actions/upload-artifact@v3
# with:
# name: Azure_Container_Scan_Result
# path: |
# scanreport.tsv
# SCAN_READ_ME.txt
deploy-deeplynx-acc:
runs-on: [ self-hosted ]
environment: acceptance
needs: build-deeplynx-acc
steps:
- name: Checkout
uses: actions/checkout@v3
with:
path: deeplynx
ref: main
- name: Checkout tools repo
uses: actions/checkout@v3
with:
repository: Digital-Engineering/kubernetes
ref: main
path: kubernetes
token: ${{ secrets.GH_TOKEN }}
- shell: bash
name: Manifest env substitute
env:
CI_REGISTRY: ${{ secrets.CI_REGISTRY}}
CI_REGISTRY_PATH: ${{ secrets.CI_REGISTRY_PATH}}
DB_NAME: ${{ secrets.DB_NAME }}
AZURE_BLOB_CONTAINER_NAME: ${{ secrets.AZURE_BLOB_CONTAINER_NAME }}
CONTAINER_INVITE_URL: ${{ secrets.CONTAINER_INVITE_URL }}
FILE_STORAGE_METHOD: ${{ secrets.FILE_STORAGE_METHOD}}
SMTP_HOST: ${{ secrets.SMTP_HOST}}
SMTP_TLS: ${{ secrets.SMTP_TLS}}
EMAIL_ADDRESS: ${{ secrets.EMAIL_ADDRESS}}
EMAIL_ENABLED: ${{ secrets.EMAIL_ENABLED}}
EMAIL_VALIDATION_ENFORCED: ${{ secrets.EMAIL_VALIDATION_ENFORCED}}
ROOT_ADDRESS: ${{ secrets.ROOT_ADDRESS }}
ENCRYPTION_KEY_PATH: ${{ secrets.ENCRYPTION_KEY_PATH }}
NODE_ENV: ${{ secrets.NODE_ENV }}
AUTH_STRATEGY: ${{ secrets.AUTH_STRATEGY }}
BASE_URL: ${{ secrets.BASE_URL }}
SAML_ENABLED: ${{ secrets.SAML_ENABLED }}
SAML_ADFS_ENTRY_POINT: ${{ secrets.SAML_ADFS_ENTRY_POINT }}
SAML_ADFS_ISSUER: ${{ secrets.SAML_ADFS_ISSUER }}
SAML_ADFS_CALLBACK: ${{ secrets.SAML_ADFS_CALLBACK }}
SAML_ADFS_PRIVATE_CERT_PATH: ${{ secrets.SAML_ADFS_PRIVATE_CERT_PATH }}
SAML_ADFS_PUBLIC_CERT_PATH: ${{ secrets.SAML_ADFS_PUBLIC_CERT_PATH }}
SAML_ADFS_CLAIMS_EMAIL: ${{ secrets.SAML_ADFS_CLAIMS_EMAIL }}
SAML_ADFS_CLAIMS_NAME: ${{ secrets.SAML_ADFS_CLAIMS_NAME }}
SERVAL_URL: ${{secrets.SERVAL_URL}}
PROCESS_QUEUE_NAME: ${{ secrets.PROCESS_QUEUE_NAME }}
EVENTS_QUEUE_NAME: ${{ secrets.EVENTS_QUEUE_NAME }}
DATA_SOURCES_QUEUE_NAME: ${{ secrets.DATA_SOURCES_QUEUE_NAME }}
CACHE_PROVIDER: ${{ secrets.CACHE_PROVIDER }}
CACHE_REDIS_CONNECTION_STRING: ${{ secrets.CACHE_REDIS_CONNECTION_STRING }}
EDGE_INSERTION_QUEUE_NAME: ${{ secrets.EDGE_INSERTION_QUEUE_NAME }}
QUEUE_SYSTEM: ${{ secrets.QUEUE_SYSTEM }}
TIMESCALEDB_ENABLED: ${{ secrets.TIMESCALEDB_ENABLED }}
TZ: ${{ secrets.TZ }}
CI_COMMIT_SHA: $GITHUB_SHA
NODE_EXTRA_CA_CERTS: ${{ secrets.NODE_EXTRA_CA_CERTS }}
RUN_JOBS: ${{ secrets.RUN_JOBS }}
ELM_HOST: ${{ secrets.ELM_HOST }}
ELM_IP: ${{ secrets.ELM_IP }}
NETWORKING_HOST: ${{ secrets.NETWORKING_HOST }}
run: |
cd $GITHUB_WORKSPACE/kubernetes/deeplynx/manifests
envsubst < acceptance.yml > acceptance_final.yml
- uses: azure/setup-kubectl@v3
- uses: azure/k8s-set-context@v3
with:
method: kubeconfig
kubeconfig: ${{ secrets.KUBE_CONFIG }}
context: deploy-service-account
- uses: Azure/k8s-deploy@v4
with:
resource-group: ${{ secrets.CI_RESOURCE_GROUP }}
name: ${{ secrets.CLUSTER_NAME }}
namespace: deeplynx-acc
action: deploy
force: true
strategy: basic
manifests: |
kubernetes/deeplynx/manifests/acceptance_final.yml
mirror-repository:
runs-on: [ self-hosted ]
environment: production
needs: build-deeplynx-acc
steps:
- uses: actions/checkout@v3
with:
ref: main
- shell: bash
name: mirror-deep-lynx-repo
run: |
git fetch --unshallow origin
git remote add github-pub ${{ secrets.PUBLIC_GITHUB_URL }} || git remote set-url github-pub ${{ secrets.PUBLIC_GITHUB_URL }}
git branch -m master
git push github-pub master --force
git push github-pub --tags
git branch -m main
update-sdks:
runs-on: [ self-hosted ]
environment: production
needs: mirror-repository
steps:
- uses: actions/checkout@v3
with:
ref: main
- uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '8'
- run: |
sudo mkdir /opt/swagger && cd /opt/swagger
sudo wget https://repo1.maven.org/maven2/io/swagger/codegen/v3/swagger-codegen-cli/3.0.35/swagger-codegen-cli-3.0.35.jar -O swagger-codegen-cli.jar
git clone ${{ secrets.DL_JS_SDK }} ~/deeplynx-js-sdk && sudo chmod -R 777 ~/deeplynx-js-sdk
git clone ${{ secrets.DL_PY_SDK }} ~/deeplynx-py-sdk && sudo chmod -R 777 ~/deeplynx-py-sdk
git clone ${{ secrets.DL_CSHARP_SDK }} ~/deeplynx-csharp-sdk && sudo chmod -R 777 ~/deeplynx-csharp-sdk
git clone ${{ secrets.DL_R_SDK }} ~/deeplynx-r-sdk && sudo chmod -R 777 ~/deeplynx-r-sdk
java -jar swagger-codegen-cli.jar generate -i $GITHUB_WORKSPACE/documentation/Core.swagger_collection.yaml -l typescript-axios -o ~/deeplynx-js-sdk
java -jar swagger-codegen-cli.jar generate -i $GITHUB_WORKSPACE/documentation/Core.swagger_collection.yaml -l csharp -o ~/deeplynx-csharp-sdk
java -jar swagger-codegen-cli.jar generate -i $GITHUB_WORKSPACE/documentation/Core.swagger_collection.yaml -l r -o ~/deeplynx-r-sdk
java -jar swagger-codegen-cli.jar generate -i $GITHUB_WORKSPACE/documentation/Core.swagger_collection.yaml -l python -o ~/deeplynx-py-sdk --additional-properties packageName=deep_lynx projectName=deep_lynx
cd ~/deeplynx-js-sdk
git config --global user.email "${{ secrets.GH_ENTERPRISE_EMAIL }}" && git config --global user.name "${{ secrets.GH_ENTERPRISE_USERNAME }}"
git add . && git commit -m $GITHUB_SHA || true
git push ${{ secrets.DL_JS_SDK }} main
cd ~/deeplynx-py-sdk
git config --global user.email "${{ secrets.GH_ENTERPRISE_EMAIL }}" && git config --global user.name "${{ secrets.GH_ENTERPRISE_USERNAME }}"
git add . && git commit -m $GITHUB_SHA || true
git push ${{ secrets.DL_PY_SDK }} main
cd ~/deeplynx-csharp-sdk
git config --global user.email "${{ secrets.GH_ENTERPRISE_EMAIL }}" && git config --global user.name "${{ secrets.GH_ENTERPRISE_USERNAME }}"
git add . && git commit -m $GITHUB_SHA || true
git push ${{ secrets.DL_CSHARP_SDK }} main
cd ~/deeplynx-r-sdk
git config --global user.email "${{ secrets.GH_ENTERPRISE_EMAIL }}" && git config --global user.name "${{ secrets.GH_ENTERPRISE_USERNAME }}"
git add . && git commit -m $GITHUB_SHA || true
git push ${{ secrets.DL_R_SDK }} main