Merge pull request #496 from idaholab/fix/helm-datahub #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Deploy Development | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: [ "development" ] | |
| jobs: | |
| build-deeplynx-dev: | |
| runs-on: [ self-hosted ] | |
| environment: Development | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| path: deeplynx | |
| ref: development | |
| - name: Checkout tools repo | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: Digital-Engineering/kubernetes | |
| ref: development | |
| path: kubernetes | |
| token: ${{ secrets.GH_TOKEN }} | |
| - shell: bash | |
| name: ACR build deeplynx | |
| env: | |
| ACR_SP_USER: ${{ secrets.CI_SP_USER }} | |
| ACR_SP_PASSWORD: ${{ secrets.CI_SP_PASSWORD }} | |
| ACR_REGISTRY: ${{ secrets.CI_REGISTRY }} | |
| ACR_PATH: ${{ secrets.CI_REGISTRY_PATH }} | |
| ACR_SP_TENANT: ${{ secrets.CI_SP_TENANT }} | |
| ACR_SUBSCRIPTION: ${{ secrets.CI_ACR_SUBSCRIPTION }} | |
| NODE_ENV: ${{ secrets.NODE_ENV }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/deeplynx | |
| az cloud set --name AzureUSGovernment | |
| az login --service-principal -u $ACR_SP_USER -p $ACR_SP_PASSWORD --tenant $ACR_SP_TENANT | |
| az account set --subscription $ACR_SUBSCRIPTION | |
| az acr build -r $ACR_REGISTRY -f $GITHUB_WORKSPACE/deeplynx/Dockerfile -t $ACR_PATH:$GITHUB_SHA-dev . | |
| # scan-deeplynx-dev: | |
| # runs-on: [ self-hosted ] | |
| # environment: Development | |
| # needs: build-deeplynx-dev | |
| # steps: | |
| # - name: Checkout | |
| # uses: actions/checkout@v3 | |
| # with: | |
| # path: deeplynx | |
| # ref: development | |
| # - shell: bash | |
| # name: ACR Get Scan | |
| # env: | |
| # ACR_REGISTRY: ${{ secrets.CI_REGISTRY }} | |
| # ACR_PATH: ${{ secrets.CI_REGISTRY_PATH }} | |
| # SHORT_SHA: ${{ steps.vars.outputs.sha_short }} | |
| # run: | | |
| # imageDigest=$(az acr repository show -n $ACR_REGISTRY -t $ACR_PATH:$GITHUB_SHA-dev | jq --raw-output '.digest') | |
| # healthquery="securityresources | |
| # | where type == 'microsoft.security/assessments/subassessments' | |
| # | where id matches regex '(.+?)/providers/Microsoft.ContainerRegistry/registries/(.+)/providers/Microsoft.Security/assessments/dbd0cb49-b563-45e7-9724-889e799fa648/' | |
| # | extend registryResourceId = tostring(split(id, '/providers/Microsoft.Security/assessments/')[0]) | |
| # | extend registryResourceName = tostring(split(registryResourceId, '/providers/Microsoft.ContainerRegistry/registries/')[1]) | |
| # | extend imageDigest = tostring(properties.additionalData.imageDigest) | |
| # | extend repository = tostring(properties.additionalData.repositoryName) | |
| # | extend scanFindingSeverity = tostring(properties.status.severity), scanStatus = tostring(properties.status.code) | |
| # | summarize scanFindingSeverityCount = count() by scanFindingSeverity, scanStatus, registryResourceId, registryResourceName, repository, imageDigest | |
| # | summarize severitySummary = make_bag(pack(scanFindingSeverity, scanFindingSeverityCount)) by registryResourceId, registryResourceName, repository, imageDigest, scanStatus | |
| # | where imageDigest contains '$imageDigest'" | |
| # query="SecurityResources | |
| # | where type == 'microsoft.security/assessments' | |
| # | where properties.displayName contains 'Container registry images should have vulnerability findings resolved' | |
| # | summarize by assessmentKey=name //the ID of the assessment | |
| # | join kind=inner (securityresources | where type == 'microsoft.security/assessments/subassessments' | extend assessmentKey = extract('.*assessments/(.+?)/.*',1, id)) on assessmentKey | |
| # | project parse_json(properties) | |
| # | extend description = properties.description,displayName = properties.displayName,resourceId = properties.resourceDetails.id,resourceSource = properties.resourceDetails.source,category = properties.category,severity = properties.status.severity,code = properties.status.code,timeGenerated = properties.timeGenerated,remediation = properties.remediation,impact = properties.impact,vulnId = properties.id,additionalData = properties.additionalData | |
| # | where resourceId contains '$imageDigest'" | |
| # az config set extension.use_dynamic_install=yes_without_prompt | |
| # count=1 | |
| # querycount=1 | |
| # until false; do | |
| # scanhealth=$(az graph query -q "$healthquery" | jq --raw-output '.data[] | .scanStatus') | |
| # if [[ $scanhealth = 'Healthy' ]]; then | |
| # echo 'Scan returned health' | |
| # break | |
| # elif [[ $scanhealth = 'Unhealthy' ]]; then | |
| # echo "Building report with findings" | |
| # rm -f scanreport.tsv | |
| # echo -e 'severity\tid\tpatchable\tpublished\tregistryhost\treponame\tos\tdisplayname\tdescription\timpact\tcvetitle\tcvelink\tvendorrefrencetitle\tvendorerefrencelink\tscanner\ttype\timagedigest' >>scanreport.tsv | |
| # az graph query -q "$query" | jq --raw-output '.data[] | [.severity, .properties.id, .properties.additionalData.patchable, .properties.additionalData.publishedTime, .properties.additionalData.registryHost, .properties.additionalData.repositoryName, .additionalData.imageDetails.osDetails, .displayName, '.description', .impact, .properties.additionalData.cve[].title, .properties.additionalData.cve[].link,.properties.additionalData.vendorReferences[].title, .properties.additionalData.vendorReferences[].link, .properties.additionalData.scanner,.properties.additionalData.type , .additionalData.imageDigest] | @tsv' >>scanreport.tsv | |
| # break | |
| # elif [[ $count -eq 10 ]]; then | |
| # echo "Image scan not found exiting" | |
| # break | |
| # else | |
| # echo "Scan not complete... waiting $count" | |
| # sleep 30 | |
| # count="$((count + 1))" | |
| # fi | |
| # done | |
| # echo -e "run:\nreg add "HKEY_LOCAL_MACHINE\SOFTWARE\Clients\Spreadsheet\Microsoft Excel\Capabilities\FileAssociations" /v ".tsv" /t REG_SZ /d "Excel.SLK" /f\nto associate .tsv with excel. You should only need to do this once." >> SCAN_READ_ME.txt | |
| # - uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: Azure_Container_Scan_Result | |
| # path: | | |
| # scanreport.tsv | |
| # SCAN_READ_ME.txt | |
| deploy-deeplynx-dev: | |
| runs-on: [ self-hosted ] | |
| environment: development | |
| needs: build-deeplynx-dev | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| path: deeplynx | |
| ref: development | |
| - name: Checkout tools repo | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: Digital-Engineering/kubernetes | |
| ref: development | |
| path: kubernetes | |
| token: ${{ secrets.GH_TOKEN }} | |
| - shell: bash | |
| name: Manifest env substitute | |
| env: | |
| CI_REGISTRY: ${{ secrets.CI_REGISTRY }} | |
| CI_REGISTRY_PATH: ${{ secrets.CI_REGISTRY_PATH }} | |
| DB_NAME: ${{ secrets.DB_NAME }} | |
| AZURE_BLOB_CONTAINER_NAME: ${{ secrets.AZURE_BLOB_CONTAINER_NAME }} | |
| CONTAINER_INVITE_URL: ${{ secrets.CONTAINER_INVITE_URL }} | |
| FILE_STORAGE_METHOD: ${{ secrets.FILE_STORAGE_METHOD}} | |
| SMTP_HOST: ${{ secrets.SMTP_HOST }} | |
| SMTP_TLS: ${{ secrets.SMTP_TLS }} | |
| EMAIL_ADDRESS: ${{ secrets.EMAIL_ADDRESS }} | |
| EMAIL_ENABLED: ${{ secrets.EMAIL_ENABLED }} | |
| EMAIL_VALIDATION_ENFORCED: ${{ secrets.EMAIL_VALIDATION_ENFORCED }} | |
| ROOT_ADDRESS: ${{ secrets.ROOT_ADDRESS }} | |
| ENCRYPTION_KEY_PATH: ${{ secrets.ENCRYPTION_KEY_PATH }} | |
| NODE_ENV: ${{ secrets.NODE_ENV }} | |
| AUTH_STRATEGY: ${{ secrets.AUTH_STRATEGY }} | |
| BASE_URL: ${{ secrets.BASE_URL }} | |
| SAML_ENABLED: ${{ secrets.SAML_ENABLED }} | |
| SAML_ADFS_ENTRY_POINT: ${{ secrets.SAML_ADFS_ENTRY_POINT }} | |
| SAML_ADFS_ISSUER: ${{ secrets.SAML_ADFS_ISSUER }} | |
| SAML_ADFS_CALLBACK: ${{ secrets.SAML_ADFS_CALLBACK }} | |
| SAML_ADFS_PRIVATE_CERT_PATH: ${{ secrets.SAML_ADFS_PRIVATE_CERT_PATH }} | |
| SAML_ADFS_PUBLIC_CERT_PATH: ${{ secrets.SAML_ADFS_PUBLIC_CERT_PATH }} | |
| SAML_ADFS_CLAIMS_EMAIL: ${{ secrets.SAML_ADFS_CLAIMS_EMAIL }} | |
| SAML_ADFS_CLAIMS_NAME: ${{ secrets.SAML_ADFS_CLAIMS_NAME }} | |
| SERVAL_URL: ${{secrets.SERVAL_URL}} | |
| PROCESS_QUEUE_NAME: ${{ secrets.PROCESS_QUEUE_NAME }} | |
| EVENTS_QUEUE_NAME: ${{ secrets.EVENTS_QUEUE_NAME }} | |
| DATA_SOURCES_QUEUE_NAME: ${{ secrets.DATA_SOURCES_QUEUE_NAME }} | |
| CACHE_PROVIDER: ${{ secrets.CACHE_PROVIDER }} | |
| CACHE_REDIS_CONNECTION_STRING: ${{ secrets.CACHE_REDIS_CONNECTION_STRING }} | |
| EDGE_INSERTION_QUEUE_NAME: ${{ secrets.EDGE_INSERTION_QUEUE_NAME }} | |
| QUEUE_SYSTEM: ${{ secrets.QUEUE_SYSTEM }} | |
| TIMESCALEDB_ENABLED: ${{ secrets.TIMESCALEDB_ENABLED }} | |
| TZ: ${{ secrets.TZ }} | |
| CI_COMMIT_SHA: $GITHUB_SHA | |
| NODE_EXTRA_CA_CERTS: ${{ secrets.NODE_EXTRA_CA_CERTS }} | |
| RUN_JOBS: ${{ secrets.RUN_JOBS }} | |
| ELM_HOST: ${{ secrets.ELM_HOST }} | |
| ELM_IP: ${{ secrets.ELM_IP }} | |
| NETWORKING_HOST: ${{ secrets.NETWORKING_HOST }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/kubernetes/deeplynx/manifests | |
| envsubst < development.yml > development_final.yml | |
| - uses: Azure/setup-kubectl@v3.1 | |
| - uses: Azure/k8s-set-context@v3.0 | |
| with: | |
| method: kubeconfig | |
| kubeconfig: ${{ secrets.KUBE_CONFIG }} | |
| context: deploy-service-account | |
| - uses: Azure/k8s-deploy@v4 | |
| with: | |
| resource-group: ${{ secrets.CI_RESOURCE_GROUP }} | |
| name: ${{ secrets.CLUSTER_NAME }} | |
| namespace: deeplynx-dev | |
| action: deploy | |
| force: true | |
| strategy: basic | |
| manifests: | | |
| kubernetes/deeplynx/manifests/development_final.yml | |
| mirror-repository: | |
| runs-on: [ self-hosted ] | |
| environment: development | |
| needs: deploy-deeplynx-dev | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: development | |
| - shell: bash | |
| name: mirror-deep-lynx-repo | |
| run: | | |
| git fetch --unshallow origin | |
| git remote add github-pub ${{ secrets.PUBLIC_GITHUB_URL }} || git remote set-url github-pub ${{ secrets.PUBLIC_GITHUB_URL }} | |
| git push github-pub development --force | |
| git push github-pub --tags |