informalsystems · greg-szabo · Jun 20, 2024 · Jun 18, 2024 · Jun 18, 2024 · Jun 18, 2024
@@ -4,7 +4,7 @@ variable "ssh_keys" {
 
 variable "instance_tags" {
   type    = list(string)
-  default = ["malachite"]
+  default = ["Malachite"]
 }
 
 resource "digitalocean_droplet" "cc" {
@@ -17,7 +17,9 @@ resource "digitalocean_droplet" "cc" {
   # Build takes about 2.5 minutes on an 8-core Digital Ocean server
   #size      = "s-8vcpu-16gb"
   ssh_keys  = var.ssh_keys
-  user_data = file("user-data/cc-data.txt")
+  user_data = templatefile("user-data/cc-data.txt", {
+    grafana_dashboard = filebase64("../viewer/config-grafana/provisioning/dashboards-data/main.json")
+  })
 }
 
 resource "digitalocean_droplet" "small" {

@@ -6,7 +6,6 @@
 # PSSH_H - space-separated list of all the node server IP addresses for pssh input
 # PSSH_P - the number of parallel processes to run with pssh
 # MALACHITE_DIR - the path to the malachite repository directory
-# MALACHITE_DIR - the path to the malachite repository directory
 # IS_CC - 1 means we are on the CC server, 0 we are not. (Used to determine the docker -H parameter.)
 ##
 # Aliases for easy manual access to the servers (don't use these in scripts)
@@ -63,20 +62,16 @@ get_ip() {
 
 ok_cc() {
   _keyscan_cc 2> /dev/null
-  PSSH_P=1 PSSH_H=$CANDC xssh "cat /etc/done" && \
-    echo "Updating cc server..." && \
-    scp -q "$${1:-$${MALACHITE_DIR}/qa/terraform/hosts}" root@$${CANDC}:/etc/hosts && \
-    ssh root@$${CANDC} "systemctl restart dnsmasq" && \
-    scp -q "$${1:-$${MALACHITE_DIR}/qa/terraform/commands.sh}" root@$${CANDC}:/etc/profile.d/commands.sh && \
-    ssh root@$${CANDC} \
-      "sed -i 's,^export MALACHITE_DIR=.*,export MALACHITE_DIR=/root/malachite,' /etc/profile.d/commands.sh && \
-       sed -i 's,^export IS_CC=.*,export IS_CC=1,' /etc/profile.d/commands.sh && \
-       source /etc/profile.d/commands.sh && \
-       _keyscan_all_servers 2> /dev/null"
+  PSSH_P=1 PSSH_H=$CANDC xssh "cat /etc/done"
+  sftp -C -q root@$${CANDC} <<EOF
+      put $${1:-$${MALACHITE_DIR}/qa/terraform/hosts} /etc/hosts
+      put $${1:-$${MALACHITE_DIR}/qa/terraform/commands.sh} /etc/profile.d/commands.sh
+EOF
+  ssh root@$${CANDC} "sed -i -e 's,^export MALACHITE_DIR=.*,export MALACHITE_DIR=/root/malachite,' -e 's,^export IS_CC=.*,export IS_CC=1,' /etc/profile.d/commands.sh && systemctl reload dnsmasq"
 }
 
 ok_all() {
-  _keyscan_all_servers 2> /dev/null
+  _keyscan_servers 2> /dev/null
   xssh "cat /etc/done && mount /data" # Mount /data in case a QA node came online earlier than CC
 }
 
@@ -180,7 +175,7 @@ _keyscan_cc() {
   ssh-keyscan -t ed25519 $CANDC >> $HOME/.ssh/known_hosts
 }
 
-_keyscan_all_servers() {
+_keyscan_servers() {
   _keyscan_cc 2> /dev/null
 %{~ for n in concat(small, large) }
   ssh-keygen -R ${n.ip} > /dev/null

@@ -1,3 +1,4 @@
+127.0.0.1       localhost
 ${cc.ip}        g-${cc.name}
 %{~ for n in small }
 ${n.ip}        g-${n.name}

@@ -95,7 +95,7 @@ write_files:
       {
         "insecure-registries" : ["0.0.0.0/0"]
       }
-  - path: /etc/systemd/resolved.conf.d/DigitalOcean.conf
+  - path: /etc/systemd/resolved.conf.d/1-dnsmasq.conf
     content: |
       [Resolve]
       DNS=127.0.0.1
@@ -321,19 +321,109 @@ write_files:
             'large96:9000',
             'large97:9000',
             'large98:9000',
-            'large99:9000'
+            'large99:9000',
+            'node0:9000',
+            'node1:9000',
+            'node2:9000',
+            'node3:9000',
+            'node4:9000',
+            'node5:9000',
+            'node6:9000',
+            'node7:9000',
+            'node8:9000',
+            'node9:9000',
+            'node10:9000',
+            'node11:9000',
+            'node12:9000',
+            'node13:9000',
+            'node14:9000',
+            'node15:9000',
+            'node16:9000',
+            'node17:9000',
+            'node18:9000',
+            'node19:9000',
+            'node20:9000',
+            'node21:9000',
+            'node22:9000',
+            'node23:9000',
+            'node24:9000',
+            'node25:9000',
+            'node26:9000',
+            'node27:9000',
+            'node28:9000',
+            'node29:9000',
+            'node30:9000',
+            'node31:9000',
+            'node32:9000',
+            'node33:9000',
+            'node34:9000',
+            'node35:9000',
+            'node36:9000',
+            'node37:9000',
+            'node38:9000',
+            'node39:9000',
+            'node40:9000',
+            'node41:9000',
+            'node42:9000',
+            'node43:9000',
+            'node44:9000',
+            'node45:9000',
+            'node46:9000',
+            'node47:9000',
+            'node48:9000',
+            'node49:9000',
+            'node50:9000',
+            'node51:9000',
+            'node52:9000',
+            'node53:9000',
+            'node54:9000',
+            'node55:9000',
+            'node56:9000',
+            'node57:9000',
+            'node58:9000',
+            'node59:9000',
+            'node60:9000',
+            'node61:9000',
+            'node62:9000',
+            'node63:9000',
+            'node64:9000',
+            'node65:9000',
+            'node66:9000',
+            'node67:9000',
+            'node68:9000',
+            'node69:9000',
+            'node70:9000',
+            'node71:9000',
+            'node72:9000',
+            'node73:9000',
+            'node74:9000',
+            'node75:9000',
+            'node76:9000',
+            'node77:9000',
+            'node78:9000',
+            'node79:9000',
+            'node80:9000',
+            'node81:9000',
+            'node82:9000',
+            'node83:9000',
+            'node84:9000',
+            'node85:9000',
+            'node86:9000',
+            'node87:9000',
+            'node88:9000',
+            'node89:9000',
+            'node90:9000',
+            'node91:9000',
+            'node92:9000',
+            'node93:9000',
+            'node94:9000',
+            'node95:9000',
+            'node96:9000',
+            'node97:9000',
+            'node98:9000',
+            'node99:9000'
             ]
           scrape_interval: 1s
-  - path: /root/grafana.yml
-    content: |
-      apiVersion: 1
-      datasources:
-        - name: prometheus
-          uid: prometheus
-          type: prometheus
-          url: http://host.docker.internal:9090
-          is_default: true
-          editable: true
   - path: /etc/nsswitch.conf
     content: |
       passwd:         files
@@ -347,7 +437,34 @@ write_files:
       ethers:         db files
       rpc:            db files
       netgroup:       nis
-  - path: /root/compose.yml
+  - path: /root/docker/grafana.yml
+    content: |
+      apiVersion: 1
+      datasources:
+        - name: prometheus
+          uid: prometheus
+          type: prometheus
+          url: http://host.docker.internal:9090
+          is_default: true
+          editable: true
+  - path: /root/docker/malachite.yaml
+    content: |
+      apiVersion: 1
+      providers:
+        - name: 'default'
+          orgId: 1
+          folder: ''
+          type: file
+          disableDeletion: false
+          updateIntervalSeconds: 10
+          allowUiUpdates: true
+          options:
+            path: '/dashboards'
+            foldersFromFilesStructure: true
+  - path: /root/dashboards/main.json
+    encoding: b64
+    content: ${grafana_dashboard}
+  - path: /root/docker/compose.yml
     content: |
       services:
         registry:
@@ -364,7 +481,9 @@ write_files:
           container_name: grafana
           image: grafana/grafana-oss
           volumes:
-            - /root/grafana.yml:/etc/grafana/provisioning/datasources/prometheus.yml
+            - /root/docker/grafana.yml:/etc/grafana/provisioning/datasources/prometheus.yml
+            - /root/docker/malachite.yml:/etc/grafana/provisioning/dashboards/malachite.yml
+            - /root/dashboards:/dashboards
             - grafana:/var/lib/grafana
           ports:
             - 0.0.0.0:3000:3000
@@ -409,7 +528,8 @@ runcmd:
   - systemctl enable nfs-kernel-server
   - systemctl start nfs-kernel-server
   - systemctl restart systemd-journald
-  - docker compose -f /root/compose.yml up -d
+  - systemctl restart dnsmasq
+  - docker compose -f /root/docker/compose.yml up -d
   - curl -s -o /usr/bin/sconfig -L https://github.com/freshautomations/sconfig/releases/download/v0.2.0/sconfig_linux_amd64
   - chmod 755 /usr/bin/sconfig
   - date > /etc/done

@@ -206,5 +206,105 @@ scrape_configs:
         'large96:9000',
         'large97:9000',
         'large98:9000',
-        'large99:9000'
+        'large99:9000',
+        'node0:9000',
+        'node1:9000',
+        'node2:9000',
+        'node3:9000',
+        'node4:9000',
+        'node5:9000',
+        'node6:9000',
+        'node7:9000',
+        'node8:9000',
+        'node9:9000',
+        'node10:9000',
+        'node11:9000',
+        'node12:9000',
+        'node13:9000',
+        'node14:9000',
+        'node15:9000',
+        'node16:9000',
+        'node17:9000',
+        'node18:9000',
+        'node19:9000',
+        'node20:9000',
+        'node21:9000',
+        'node22:9000',
+        'node23:9000',
+        'node24:9000',
+        'node25:9000',
+        'node26:9000',
+        'node27:9000',
+        'node28:9000',
+        'node29:9000',
+        'node30:9000',
+        'node31:9000',
+        'node32:9000',
+        'node33:9000',
+        'node34:9000',
+        'node35:9000',
+        'node36:9000',
+        'node37:9000',
+        'node38:9000',
+        'node39:9000',
+        'node40:9000',
+        'node41:9000',
+        'node42:9000',
+        'node43:9000',
+        'node44:9000',
+        'node45:9000',
+        'node46:9000',
+        'node47:9000',
+        'node48:9000',
+        'node49:9000',
+        'node50:9000',
+        'node51:9000',
+        'node52:9000',
+        'node53:9000',
+        'node54:9000',
+        'node55:9000',
+        'node56:9000',
+        'node57:9000',
+        'node58:9000',
+        'node59:9000',
+        'node60:9000',
+        'node61:9000',
+        'node62:9000',
+        'node63:9000',
+        'node64:9000',
+        'node65:9000',
+        'node66:9000',
+        'node67:9000',
+        'node68:9000',
+        'node69:9000',
+        'node70:9000',
+        'node71:9000',
+        'node72:9000',
+        'node73:9000',
+        'node74:9000',
+        'node75:9000',
+        'node76:9000',
+        'node77:9000',
+        'node78:9000',
+        'node79:9000',
+        'node80:9000',
+        'node81:9000',
+        'node82:9000',
+        'node83:9000',
+        'node84:9000',
+        'node85:9000',
+        'node86:9000',
+        'node87:9000',
+        'node88:9000',
+        'node89:9000',
+        'node90:9000',
+        'node91:9000',
+        'node92:9000',
+        'node93:9000',
+        'node94:9000',
+        'node95:9000',
+        'node96:9000',
+        'node97:9000',
+        'node98:9000',
+        'node99:9000'
       ]