From 49376ec25802fb3f2bf6d105e8d6f914eb316579 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Sun, 29 Sep 2024 23:28:07 +0200 Subject: [PATCH] TRD ignore provider error - do not send alert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TRD chart runs tezos reward distribution software for delegations. We support sending slack alerts when reward distribution fail. Here, we address a common false positive: Provider errors are usually because tzkt is rate limited or busy. example: │ 2024-09-29 21:02:16,534 - MainThread - INFO - -------------------------------------------- │ │ 2024-09-29 21:02:16,535 - MainThread - INFO - BAKING ADDRESS is │ 2024-09-29 21:02:16,535 - MainThread - INFO - PAYMENT ADDRESS is │ 2024-09-29 21:02:16,535 - MainThread - INFO - -------------------------------------------- │ │ 2024-09-29 21:02:16,537 - MainThread - INFO - [Plugins] No plugins enabled │ │ 2024-09-29 21:02:16,539 - MainThread - INFO - Initial cycle set to -1 │ │ 2024-09-29 21:02:16,542 - MainThread - INFO - Application is READY! │ │ 2024-09-29 21:02:16,544 - producer - INFO - No failed payment files found under directory '/trd/reports/xxx/payments/failed' on or after cycle '-1' │ │ 2024-09-29 21:02:16,545 - MainThread - INFO - -------------------------------------------- │ │ 2024-09-29 21:02:16,624 - producer - ERROR - Unable to fetch current cycle from provider tzkt, Not synced. Exiting. │ │ 2024-09-29 21:02:16,626 - consumer0 - WARNING - Exit signal received. Terminating... │ │ 2024-09-29 21:02:16,626 - MainThread - INFO - Application stop handler called: 12 │ │ 2024-09-29 21:02:16,628 - producer - INFO - TRD Exit triggered by producer, exit code: 8 │ │ 2024-09-29 21:02:16,629 - MainThread - INFO - TRD is shutting down... │ │ 2024-09-29 21:02:16,630 - MainThread - INFO - -------------------------------------------------------- │ │ 2024-09-29 21:02:16,631 - MainThread - INFO - Sensitive operations are in progress! │ │ 2024-09-29 21:02:16,631 - MainThread - INFO - Please wait while the application is being shut down! │ │ 2024-09-29 21:02:16,632 - MainThread - INFO - -------------------------------------------------------- │ │ 2024-09-29 21:02:16,632 - MainThread - INFO - Lock file removed! │ │ 2024-09-29 21:02:16,633 - MainThread - INFO - Shutdown due to error!, exit code: 1 │ │ Tezos Reward Distributor (TRD) is Starting We also modify TRD to add a specific error code for this specific benign case: https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/pull/713 --- charts/tezos-reward-distributor/scripts/run.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/charts/tezos-reward-distributor/scripts/run.sh b/charts/tezos-reward-distributor/scripts/run.sh index 1ab32f5..db7fca3 100644 --- a/charts/tezos-reward-distributor/scripts/run.sh +++ b/charts/tezos-reward-distributor/scripts/run.sh @@ -22,21 +22,31 @@ python src/main.py \ ${EXTRA_TRD_ARGS} \ ${dry_run_arg} -# if TRD fails, send a slack alert -if [ $? -ne 0 ]; then +# if TRD fails or returns exit code 9, send a slack alert +exit_code=$? +if [ $exit_code -ne 0 ]; then # check if bot token and channel are set if [ -z "${SLACK_BOT_TOKEN}" ] || [ -z "${SLACK_CHANNEL}" ]; then echo "TRD failed, but SLACK_BOT_TOKEN or SLACK_CHANNEL is not set, failing job" exit 1 fi - python -c " + echo "TRD exited in error, exit code is ${exit_code}, maybe send slack alert" + EXIT_CODE=${exit_code} python -c " import os +import sys import requests import json slack_bot_token = os.getenv('SLACK_BOT_TOKEN') slack_channel = os.getenv('SLACK_CHANNEL') baker_alias = os.getenv('BAKER_ALIAS') +exit_code = os.getenv('EXIT_CODE') + +if exit_code == '9': + print(f'TRD returned exit code 9 (PROVIDER_BUSY) for Tezos baker {baker_alias}. Not alerting.') + sys.exit(0) +else: + message = f'TRD Payout failed for Tezos baker {baker_alias}, exit code {exit_code}.' response = requests.post( 'https://slack.com/api/chat.postMessage', @@ -46,7 +56,7 @@ response = requests.post( }, data=json.dumps({ 'channel': slack_channel, - 'text': f'TRD Payout failed for Tezos baker {baker_alias}' + 'text': message }) )