daily-archive-transfers.yaml

# This file defines automated storage transfers from pusher-* to archive-*
# and, for production, from archive-mlab-oti to archive-measurement-lab
# The overall schedule is (all times UTC):
#     pusher ->         archive: 02:30, 14:30
#     oti    -> measurement-lab: 06:30, 18:30
#
# NOTE: four hour delays are based on typical transfer times for recent GCS ST
# jobs (2020-07-15).
options:
  env:
  - PROJECT_ID=$PROJECT_ID

steps:
# Create the gcp-config image for later steps.
- name: gcr.io/cloud-builders/docker
  args: [
    'build', '-t', 'gcp-config-cbif', '.'
  ]

# 02:00:00 Nodes upload to the pusher-* bucket. 2hrs is the maximum upload delay.
# 02:10:00 Last upload from utilization/switch archives.
# 02:30:00 Configure daily pusher to local archive transfer.
- name: gcp-config-cbif
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-oti
  args: [
    'stctl', '-gcs.source=pusher-$PROJECT_ID',
             '-gcs.target=archive-$PROJECT_ID',
             '-time=02:30:00',
             '-maxFileAge=36h',
             '-include=host',
             '-include=ndt',
             '-include=neubot',
             '-include=revtr',
             '-include=utilization',
             '-include=wehe',
             '-include=msak',
             '-deleteAfterTransfer=true',
             'sync'
  ]

# Nodes upload to the pusher-* bucket every 2 hours.
# Configure hourly pusher to local archive transfer.
- name: gcp-config-cbif
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-oti
  args: [
    'stctl', '-gcs.source=pusher-$PROJECT_ID',
             '-gcs.target=archive-$PROJECT_ID',
             '-time=00:00:00',
             '-interval=3600s',
             '-include=autoload',
             '-deleteAfterTransfer=true',
             'sync'
  ]

# Configure hourly transfer from thirdparty-revtr to public archive.
- name: gcp-config-cbif
  env:
  - PROJECT_IN=measurement-lab
  args: [
    'stctl', '-gcs.source=thirdparty-revtr-mlab-oti',
             '-gcs.target=archive-measurement-lab',
             '-include=revtr',
             '-include=autoload',
             '-time=02:30:00',
             '-interval=3600s',
             '-maxFileAge=27h',
             'sync'
  ]

# 06:30:00 Configure daily local archive to public archive transfer.
# NOTE: this transfer is 4 hours after the previous transfer based on recent
# GCS ST jobs (2020-07-15).
- name: gcp-config-cbif
  env:
  - PROJECT_IN=measurement-lab
  args: [
    'stctl', '-gcs.source=archive-mlab-oti',
             '-gcs.target=archive-measurement-lab',
             '-time=06:30:00',
             '-maxFileAge=27h',
             '-deleteAfterTransfer=true',
             'sync'
  ]

# Repeat transfers every 12 hours, so that the final transfer
# has less data to move, and can complete more quickly.

# 14:30:00 Configure daily pusher to local archive transfer.
# For this one, maxFileAge is set to 7 days, to catch stragglers.
# This likely means the calculating phase will take much longer, but this
# transfer is 12 hours offset from the last transfer of the day
# preceeding the daily parsing kickoff.
- name: gcp-config-cbif
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-oti
  args: [
    'stctl', '-gcs.source=pusher-$PROJECT_ID',
             '-gcs.target=archive-$PROJECT_ID',
             '-time=14:30:00',
             '-maxFileAge=168h',  # 7 days
             '-include=host',
             '-include=ndt',
             '-include=neubot',
             '-include=revtr',
             '-include=utilization',
             '-include=wehe',
             '-include=msak',
             '-deleteAfterTransfer=true',
             'sync'
  ]

# 18:30:00 Configure daily local archive to public archive transfer.
# For this one, maxFileAge is set to 8 days, to catch stragglers.
# This likely means the calculating phase will take much longer.
- name: gcp-config-cbif
  env:
  - PROJECT_IN=measurement-lab
  args: [
    'stctl', '-gcs.source=archive-mlab-oti',
             '-gcs.target=archive-measurement-lab',
             '-time=18:30:00',
             '-deleteAfterTransfer=true',
             '-maxFileAge=192h',  # 8 days
             'sync'
  ]

# Hourly local archive to public archive transfer for autoloaded data.
- name: gcp-config-cbif
  env:
  - PROJECT_IN=measurement-lab
  args: [
    'stctl', '-gcs.source=archive-mlab-oti',
             '-gcs.target=archive-measurement-lab',
             '-time=00:00:00',
             '-interval=3600s',
             '-include=autoload',
             '-deleteAfterTransfer=true',
             'sync'
  ]

# TODO(soltesz): once ST schedules above stabilize, bring backup transfer
# schedule in sync. Until then, complete daily backups will be delayed by a day.

# 04:30:00 Gardener or other jobs that depend on the public archive being up to date may run.
# 04:30:00 Configure daily public archive to backup transfer.
# NOTE: mlab-backups intentionally restricts access. This configuration is documentation.
#- name: gcp-config-cbif
#  env:
#  - PROJECT_IN=mlab-backups
#  args: [
#    'stctl', '-gcs.source=archive-measurement-lab',
#             '-gcs.target=mlab-cold-storage-backup',
#             '-time=04:30:00',
#             'sync'
#  ]