Skip to content

Commit

Permalink
Build images for mainnet full nodes (backport #2032) (#2055)
Browse files Browse the repository at this point in the history
Co-authored-by: roy-dydx <133032749+roy-dydx@users.noreply.github.com>
mergify[bot] and roy-dydx authored Aug 7, 2024
1 parent e4a86c2 commit 52e7d2f
Showing 9 changed files with 4,094 additions and 1 deletion.
48 changes: 48 additions & 0 deletions .github/workflows/protocol-build-and-push-mainnet.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: Protocol Build & Push Image to AWS ECR

on: # yamllint disable-line rule:truthy
pull_request:
branches:
- 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x
- 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x
push:
branches:
- 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x
- 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x

jobs:
build-and-push-mainnet:
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./protocol
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: '0' # without this, ignite fails.

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_MAINNET }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_MAINNET }}
aws-region: ap-northeast-1

- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1

- name: Build, Tag, and Push the Image to Amazon ECR
id: build-image
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: mainnet-full-node
run: |
make localnet-build-amd64
commit_hash=$(git rev-parse --short=7 HEAD)
docker build \
--platform amd64 \
-t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \
-f testing/mainnet/Dockerfile .
docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags
1 change: 0 additions & 1 deletion protocol/.gitignore
Original file line number Diff line number Diff line change
@@ -10,7 +10,6 @@
*.code-workspace

# Build
bin
build
vendor
.vendor-new
13 changes: 13 additions & 0 deletions protocol/testing/mainnet/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
FROM dydxprotocol-base

RUN apk add --no-cache bash jq aws-cli
RUN go install cosmossdk.io/tools/cosmovisor/cmd/cosmovisor@v1.5.0

COPY ./testing/mainnet/. /dydxprotocol/

ENV HOME /dydxprotocol
WORKDIR $HOME

RUN /dydxprotocol/mainnet.sh

ENTRYPOINT ["/dydxprotocol/start.sh"]
Binary file not shown.
3,829 changes: 3,829 additions & 0 deletions protocol/testing/mainnet/genesis.json

Large diffs are not rendered by default.

83 changes: 83 additions & 0 deletions protocol/testing/mainnet/mainnet.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/bin/bash
set -eo pipefail

# This file initializes a non-validating full node for mainnet

source "./vars.sh"

CHAIN_ID="dydx-mainnet-1"

# Define the mapping from version to URL
declare -A version_to_url
# version_to_url["v5.1.0"]="https://github.com/dydxprotocol/v4-chain/releases/download/protocol%2Fv5.1.0-dev4/dydxprotocold-v5.1.0-dev4-linux-amd64.tar.gz"

# Define dependencies for this script.
# `jq` and `dasel` are used to manipulate json and yaml files respectively.
install_prerequisites() {
apk add dasel jq
}

set_cosmovisor_binary_permissions() {
# The genesis binary should always exist.
for f in $HOME/cosmovisor/genesis/bin/* ; do
chmod 755 $f
done
# Set up upgrade binaries.
for version in "${!version_to_url[@]}"; do
echo "Setting up version ${version}..."
version_dir="$HOME/cosmovisor/upgrades/$version"
mkdir -p "$version_dir/bin"
url=${version_to_url[$version]}
tar_file=$(basename $url)

echo "Downloading tar file from ${url}..."
wget -O $tar_file $url
tar -xzf $tar_file -C "$version_dir"
rm $tar_file
binary_file="${tar_file%.tar.gz}"
mv "$version_dir/build/$binary_file" "$version_dir/bin/dydxprotocold"
chmod 755 "$version_dir/bin/dydxprotocold"
echo "Successfully set up $version_dir/bin/dydxprotocold"
done
current_version_path="$HOME/cosmovisor/upgrades/$CURRENT_VERSION_DIR/bin"
mkdir -p $current_version_path
cp /bin/dydxprotocold $current_version_path
}

create_full_nodes() {
# Create directories for full-nodes to use.
for i in $(seq 1 $LAST_FULL_NODE_INDEX); do
FULL_NODE_HOME_DIR="$HOME/chain/.full-node-$i"
FULL_NODE_CONFIG_DIR="$FULL_NODE_HOME_DIR/config"
dydxprotocold init "full-node" -o --chain-id=$CHAIN_ID --home "$FULL_NODE_HOME_DIR"
done

# Copy the genesis file to the full-node directories.
for i in $(seq 1 $LAST_FULL_NODE_INDEX); do
FULL_NODE_HOME_DIR="$HOME/chain/.full-node-$i"
FULL_NODE_CONFIG_DIR="$FULL_NODE_HOME_DIR/config"

cp "$HOME/genesis.json" "$FULL_NODE_CONFIG_DIR/genesis.json"
done

# Set up CosmosVisor for full-nodes.
for i in $(seq 1 $LAST_FULL_NODE_INDEX); do
FULL_NODE_HOME_DIR="$HOME/chain/.full-node-$i"
# DAEMON_NAME is the name of the binary.
export DAEMON_NAME=dydxprotocold

# DAEMON_HOME is the location where the cosmovisor/ directory is kept
# that contains the genesis binary, the upgrade binaries, and any additional
# auxiliary files associated with each binary
export DAEMON_HOME="$HOME/chain/.full-node-$i"

# Create the folder structure required for using cosmovisor.
cosmovisor init /bin/dydxprotocold

cp -r "$HOME/cosmovisor" "$FULL_NODE_HOME_DIR/"
done
}

install_prerequisites
set_cosmovisor_binary_permissions
create_full_nodes
95 changes: 95 additions & 0 deletions protocol/testing/mainnet/snapshot.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#!/bin/bash
set -xeo pipefail

# This script spins up a non-validating full node that periodically is halted
# and uploads snapshots of the data directory (contains all blocks) to S3.
# Snapshots can be used to start a new full-node from the height the snapshot
# was taken at.

# Example usage: ./snapshot.sh --s3_snapshot_bucket dev4fullnodesnapshots \
# --genesis_file_rpc_address http://18.178.88.89:26657 \
# --p2p_seeds dfa67970296bbecce14daba6cb0da516ed60458a@3.129.102.24:26656 \
# --upload_period 300

# Display usage information
function display_usage() {
echo "Usage: ./snapshot.sh [options]"
echo "Options:"
echo " --genesis_file_rpc_address RPC address of a validator node to retrieve the genesis file from, e.g. http://18.178.88.89:26657"
echo " --p2p_seeds List of seed nodes to peer with, e.g. dfa67970296bbecce14daba6cb0da516ed60458a@3.129.102.24:26656"
echo " --upload_period Upload frequency in seconds, e.g. 300"
echo " --s3_snapshot_bucket Name of the S3 bucket to upload snapshots to, e.g. dev4fullnodesnapshots"
echo " --dd_agent_host Datadog agent host"
}

if [[ "$1" == "-h" || "$1" == "--help" ]]; then
display_usage
exit 0
fi

# Copy the correct binaries into the full node home directories.
source "./vars.sh"

# Set up CosmosVisor for full-nodes.
for i in $(seq 1 $LAST_FULL_NODE_INDEX); do
FULL_NODE_HOME_DIR="$HOME/chain/.full-node-$i"
# Copy binaries for `cosmovisor` from the docker image into the home directory.
# Work-around to ensure docker volume contains the same binaries as the git repo.
cp -r "$HOME/cosmovisor" "$FULL_NODE_HOME_DIR/"
done

install_prerequisites() {
apk add dasel jq curl
}

install_prerequisites


# local path to temporary snapshots. snapshots are deleted after uploading to S3.
SNAP_PATH="/dydxprotocol/chain/.full-node-2/snapshots/dydxprotocol/"
# logfile containing snapshot timestamps
LOG_PATH="/dydxprotocol/chain/.full-node-2/snapshots/dydxprotocol/dydxprotocol_log.txt"
# data directory to snapshot. this contains the blockchain state.
DATA_PATH="/dydxprotocol/chain/.full-node-2/data/"
RPC_ADDRESS="http://127.0.0.1:26657"

while [ $# -gt 0 ]; do

if [[ $1 == *"--"* ]]; then
v="${1/--/}"
export $v="$2"
fi

shift
done


# initialize snapshot path and genesis file
mkdir -p $SNAP_PATH
touch $LOG_PATH
sleep 10
CHAIN_ID="dydx-mainnet-1"

# Prune snapshots to prevent them from getting too big. We make 3 changes:
# Prune all app state except last 2 blocks
sed -i 's/pruning = "default"/pruning = "everything"/' /dydxprotocol/chain/.full-node-2/config/app.toml
# Tendermint pruning is decided by picking the most restrictive of multiple factors.
# Make the custom config setting as permissive as possible.
sed -i 's/min-retain-blocks = 0/min-retain-blocks = 2/' /dydxprotocol/chain/.full-node-2/config/app.toml
# Do not index tx_index.db
sed -i 's/indexer = "kv"/indexer = "null"/' /dydxprotocol/chain/.full-node-2/config/config.toml

# TODO: add metrics around snapshot upload latency/frequency/success rate
while true; do
# p2p.seeds taken from --p2p.persistent_peers flag of full node
cosmovisor run start --log_level info --home /dydxprotocol/chain/.full-node-2 --p2p.seeds "${p2p_seeds}" --non-validating-full-node=true --dd-agent-host=${dd_agent_host} &

sleep ${upload_period}
kill -TERM $(pidof cosmovisor)

SNAP_NAME=$(echo "${CHAIN_ID}_$(date '+%Y-%m-%d-%H-%M').tar.gz")
tar cvzf ${SNAP_PATH}/${SNAP_NAME} ${DATA_PATH}
aws s3 cp ${SNAP_PATH}/${SNAP_NAME} s3://${s3_snapshot_bucket}/ --region ap-northeast-1 --debug || true
rm -rf ${SNAP_PATH}/${SNAP_NAME}

done
19 changes: 19 additions & 0 deletions protocol/testing/mainnet/start.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/bin/bash
set -eo pipefail

# This file is the startup script for the full-nodes. Copies the correct binaries into
# the full-node home directories, and starts the node using `cosmovisor` to run `dydxprotocold`.
# Any arguments passed into this script is forwarded to `cosmovisor`.
# Example usage: ./start.sh run start --home chain/.full-node-1

source "./vars.sh"

# Set up CosmosVisor for full-nodes.
for i in $(seq 1 $LAST_FULL_NODE_INDEX); do
FULL_NODE_HOME_DIR="$HOME/chain/.full-node-$i"
# Copy binaries for `cosmovisor` from the docker image into the home directory.
# Work-around to ensure docker volume contains the same binaries as the git repo.
cp -r "$HOME/cosmovisor" "$FULL_NODE_HOME_DIR/"
done

cosmovisor "$@"
7 changes: 7 additions & 0 deletions protocol/testing/mainnet/vars.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash
set -eo pipefail

CURRENT_VERSION_DIR="v5.1.0"

# Full node home directories will be set up for indices 0 to LAST_FULL_NODE_INDEX
LAST_FULL_NODE_INDEX=5

0 comments on commit 52e7d2f

Please sign in to comment.