Skip to content

Commit

Permalink
Add .gitlab-ci.yml and pre-cache script (#14)
Browse files Browse the repository at this point in the history
* add .gitlab-ci.yml and pre-cache script

* fix fmt and clippy

* ci: fix deny

* ci: fix deny

* add unleash

* Update .gitlab-ci.yml

Co-authored-by: Squirrel <gilescope@gmail.com>

* unleash on k8s runner

* switch all jobs to k8s runner

* removed pre_cache script

* add image to k8s template

* returned docker executor back

Co-authored-by: Squirrel <gilescope@gmail.com>
  • Loading branch information
alvicsam and gilescope authored Oct 7, 2021
1 parent d2ea7c9 commit 48bce25
Show file tree
Hide file tree
Showing 3 changed files with 360 additions and 0 deletions.
138 changes: 138 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
stages:
- lint
- test
- build
- publish

variables: &default-vars
GIT_STRATEGY: fetch
GIT_DEPTH: 100
CARGO_INCREMENTAL: 0
CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
CI_IMAGE: "paritytech/ci-linux:production"
CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.12"
CARGO_UNLEASH_PKG_DEF: ""
VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io"
VAULT_AUTH_PATH: "gitlab-parity-io-jwt"
VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}"

#### Vault secrets
.vault-secrets: &vault-secrets
secrets:
CRATES_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/CRATES_TOKEN@kv
file: false

.build-refs: &build-refs
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1


.rust-info-script: &rust-info-script
- rustup show
- cargo --version
- rustup +nightly show
- cargo +nightly --version
- bash --version
- sccache -s

.docker-env: &docker-env
image: "${CI_IMAGE}"
before_script:
- *rust-info-script
- ./scripts/ci/pre_cache.sh
- sccache -s
retry:
max: 2
when:
- runner_system_failure
- unknown_failure
- api_failure
interruptible: true
tags:
- linux-docker

.kubernetes-env: &kubernetes-env
image: "${CI_IMAGE}"
retry:
max: 2
when:
- runner_system_failure
- unknown_failure
- api_failure
interruptible: true
tags:
- kubernetes-parity-build

.collect-artifacts: &collect-artifacts
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: on_success
expire_in: 28 days
paths:
- ./artifacts/

#### stage: lint

fmt:
stage: lint
<<: *kubernetes-env
script:
- cargo +nightly fmt --all -- --check

clippy:
stage: lint
<<: *docker-env
script:
- cargo +nightly clippy --all-targets --verbose

#### stage: test

deny:
stage: test
<<: *docker-env
<<: *collect-artifacts
script:
- cargo deny check --hide-inclusion-graph -c .maintain/deny.toml
after_script:
- mkdir -p ./artifacts
- echo "___Complete logs can be found in the artifacts___"
- cargo deny check --hide-inclusion-graph 2> cargo_deny.log

test:
stage: test
<<: *docker-env
script:
- cargo test --all --verbose

check:
stage: test
<<: *docker-env
script:
- cargo check --all --verbose

#### stage: build

build:
stage: build
<<: *build-refs
<<: *docker-env
script:
- cargo build --no-default-features --target wasm32-unknown-unknown --verbose

#### stage: publish

unleash-to-crates-io:
stage: publish
<<: *docker-env
<<: *vault-secrets
rules:
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
script:
- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
- cargo unleash em-dragons --no-check --owner github:paritytech:core-devs ${CARGO_UNLEASH_PKG_DEF}

193 changes: 193 additions & 0 deletions .maintain/deny.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
# This template contains all of the possible sections and their default values

# Note that all fields that take a lint level have these possible values:
# * deny - An error will be produced and the check will fail
# * warn - A warning will be produced, but the check will not fail
# * allow - No warning or error will be produced, though in some cases a note
# will be

# The values provided in this template are the default values that will be used
# when any section or field is not specified in your own configuration

# If 1 or more target triples (and optionally, target_features) are specified,
# only the specified targets will be checked when running `cargo deny check`.
# This means, if a particular package is only ever used as a target specific
# dependency, such as, for example, the `nix` crate only being used via the
# `target_family = "unix"` configuration, that only having windows targets in
# this list would mean the nix crate, as well as any of its exclusive
# dependencies not shared by any other crates, would be ignored, as the target
# list here is effectively saying which targets you are building for.
targets = [
# The triple can be any string, but only the target triples built in to
# rustc (as of 1.40) can be checked against actual config expressions
#{ triple = "x86_64-unknown-linux-musl" },
# You can also specify which target_features you promise are enabled for a
# particular target. target_features are currently not validated against
# the actual valid features supported by the target architecture.
#{ triple = "wasm32-unknown-unknown", features = ["atomics"] },
]

# This section is considered when running `cargo deny check advisories`
# More documentation for the advisories section can be found here:
# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html
[advisories]
# The path where the advisory database is cloned/fetched into
db-path = "~/.cargo/advisory-db"
# The url of the advisory database to use
db-url = "https://github.com/rustsec/advisory-db"
# The lint level for security vulnerabilities
vulnerability = "deny"
# The lint level for unmaintained crates
unmaintained = "warn"
# The lint level for crates that have been yanked from their source registry
yanked = "warn"
# The lint level for crates with security notices. Note that as of
# 2019-12-17 there are no security notice advisories in
# https://github.com/rustsec/advisory-db
notice = "warn"
# A list of advisory IDs to ignore. Note that ignored advisories will still
# output a note when they are encountered.
ignore = [
#"RUSTSEC-0000-0000",
]
# Threshold for security vulnerabilities, any vulnerability with a CVSS score
# lower than the range specified will be ignored. Note that ignored advisories
# will still output a note when they are encountered.
# * None - CVSS Score 0.0
# * Low - CVSS Score 0.1 - 3.9
# * Medium - CVSS Score 4.0 - 6.9
# * High - CVSS Score 7.0 - 8.9
# * Critical - CVSS Score 9.0 - 10.0
#severity-threshold =

# This section is considered when running `cargo deny check licenses`
# More documentation for the licenses section can be found here:
# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html
[licenses]
# The lint level for crates which do not have a detectable license
unlicensed = "deny"
# List of explictly allowed licenses
# See https://spdx.org/licenses/ for list of possible licenses
# [possible values: any SPDX 3.7 short identifier (+ optional exception)].
allow = [
#"MIT",
#"Apache-2.0",
#"Apache-2.0 WITH LLVM-exception",
]
# List of explictly disallowed licenses
# See https://spdx.org/licenses/ for list of possible licenses
# [possible values: any SPDX 3.7 short identifier (+ optional exception)].
deny = [
#"Nokia",
]
# Lint level for licenses considered copyleft
copyleft = "allow"
# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses
# * both - The license will be approved if it is both OSI-approved *AND* FSF
# * either - The license will be approved if it is either OSI-approved *OR* FSF
# * osi-only - The license will be approved if is OSI-approved *AND NOT* FSF
# * fsf-only - The license will be approved if is FSF *AND NOT* OSI-approved
# * neither - This predicate is ignored and the default lint level is used
allow-osi-fsf-free = "either"
# Lint level used when no other predicates are matched
# 1. License isn't in the allow or deny lists
# 2. License isn't copyleft
# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither"
default = "deny"
# The confidence threshold for detecting a license from license text.
# The higher the value, the more closely the license text must be to the
# canonical license text of a valid SPDX license file.
# [possible values: any between 0.0 and 1.0].
confidence-threshold = 0.9
# Allow 1 or more licenses on a per-crate basis, so that particular licenses
# aren't accepted for every possible crate as with the normal allow list
exceptions = [
# Each entry is the crate and version constraint, and its specific allow
# list
#{ allow = ["Zlib"], name = "adler32", version = "*" },
]

# Some crates don't have (easily) machine readable licensing information,
# adding a clarification entry for it allows you to manually specify the
# licensing information
[[licenses.clarify]]
# The name of the crate the clarification applies to
name = "ring"
# THe optional version constraint for the crate
#version = "*"
# The SPDX expression for the license requirements of the crate
expression = "OpenSSL"
# One or more files in the crate's source used as the "source of truth" for
# the license expression. If the contents match, the clarification will be used
# when running the license check, otherwise the clarification will be ignored
# and the crate will be checked normally, which may produce warnings or errors
# depending on the rest of your configuration
license-files = [
# Each entry is a crate relative path, and the (opaque) hash of its contents
{ path = "LICENSE", hash = 0xbd0eed23 }
]
[[licenses.clarify]]
name = "webpki"
expression = "ISC"
license-files = [{ path = "LICENSE", hash = 0x001c7e6c }]

[licenses.private]
# If true, ignores workspace crates that aren't published, or are only
# published to private registries
ignore = false
# One or more private registries that you might publish crates to, if a crate
# is only published to private registries, and ignore is true, the crate will
# not have its license(s) checked
registries = [
#"https://sekretz.com/registry
]

# This section is considered when running `cargo deny check bans`.
# More documentation about the 'bans' section can be found here:
# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html
[bans]
# Lint level for when multiple versions of the same crate are detected
multiple-versions = "warn"
# The graph highlighting used when creating dotgraphs for crates
# with multiple versions
# * lowest-version - The path to the lowest versioned duplicate is highlighted
# * simplest-path - The path to the version with the fewest edges is highlighted
# * all - Both lowest-version and simplest-path are used
highlight = "lowest-version"
# List of crates that are allowed. Use with care!
allow = [
#{ name = "ansi_term", version = "=0.11.0" },
]
# List of crates to deny
deny = [
{ name = "parity-util-mem", version = "<0.6" }
# Each entry the name of a crate and a version range. If version is
# not specified, all versions will be matched.
]
# Certain crates/versions that will be skipped when doing duplicate detection.
skip = [
#{ name = "ansi_term", version = "=0.11.0" },
]
# Similarly to `skip` allows you to skip certain crates during duplicate
# detection. Unlike skip, it also includes the entire tree of transitive
# dependencies starting at the specified crate, up to a certain depth, which is
# by default infinite
skip-tree = [
#{ name = "ansi_term", version = "=0.11.0", depth = 20 },
]

# This section is considered when running `cargo deny check sources`.
# More documentation about the 'sources' section can be found here:
# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html
[sources]
# Lint level for what to happen when a crate from a crate registry that is not
# in the allow list is encountered
unknown-registry = "deny"
# Lint level for what to happen when a crate from a git repository that is not
# in the allow list is encountered
unknown-git = "warn"
# List of URLs for allowed crate registries. Defaults to the crates.io index
# if not specified. If it is specified but empty, no registries are allowed.
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
# List of URLs for allowed Git repositories
allow-git = []
29 changes: 29 additions & 0 deletions scripts/ci/pre_cache.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/bin/bash

set -u

# if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME
# create such directory and
# copy recursively all the files from the newest dir which has $CI_JOB_NAME, if it exists

# cache lives in /ci-cache/${CI_PROJECT_NAME}/${2}/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}

function prepopulate {
if [[ ! -d $1 ]]; then
mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME";
FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \
-type d -name "$CI_JOB_NAME" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2);
if [[ -d $FRESH_CACHE ]]; then
echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____";
time cp -r "$FRESH_CACHE" "$1";
else
echo "_____No such $2 dir, proceeding from scratch_____";
fi
else
echo "____No need to prepopulate $2 cache____";
fi
}

# CARGO_HOME cache is still broken so would be handled some other way.
prepopulate "$CARGO_TARGET_DIR" targets

0 comments on commit 48bce25

Please sign in to comment.