Add monitoring for upload requests

group_vars/upload.yml

@@ -79,3 +79,22 @@ software_groups_to_install:
 
 ## kernel_5
 kernel_5_package: kernel-ml
+rustus_monitor_requests_path: /usr/local/bin/monitor-rustus-requests
+rustus_monitor_requests: |
+  #!/bin/bash
+  successful=$(journalctl -u 'rustus-main_uploads.service' --since '1 minute ago' | \
+              grep "PATCH /api/upload/resumable_upload/"| \
+              grep "204" | sort -u | wc -l)
+  errors=$(journalctl -u 'rustus-main_uploads.service' --since '1 minute ago' | \
+              grep "PATCH /api/upload/resumable_upload/"| \
+              grep -v "204" | sort -u | wc -l)
+  echo rustus,timespan=last_1_min successful_requests="${successful:-0}i",erroring_requests="${errors:-0}i"
+
+telegraf_plugins_extra:
+  rustus_requests:
+    plugin: "exec"
+    config:
+      - commands = ["{{ rustus_monitor_requests_path }}"]
+      - timeout = "15s"
+      - interval = "1m"
+      - data_format = "influx"

upload.yml

@@ -4,6 +4,7 @@
   become: true
   become_user: root
   vars_files:
+    - secret_group_vars/all.yml
     - group_vars/all.yml
     - secret_group_vars/sentry.yml
     - mounts/mountpoints.yml
@@ -14,7 +15,13 @@
         port: 1080-1081/tcp
         permanent: true
         state: enabled
-
+    - name: Copy Rustus request monitoring script
+      ansible.builtin.copy:
+        content: "{{ rustus_monitor_requests }}"
+        dest: "{{ rustus_monitor_requests_path }}"
+        mode: "0755"
+        owner: root
+        group: root
   roles:
     - role: usegalaxy_eu.handy.os_setup
       vars:
@@ -23,7 +30,15 @@
         enable_kernel_5: true
         enable_install_software: true
     - influxdata.chrony
+    - dj-wasabi.telegraf
     ## Filesystems
     - usegalaxy-eu.autofs     # Setup the mount points which will be needed later
 
     - usegalaxy_eu.rustus
+  post_tasks:
+    - name: add telegraf to systemd-journal
+      ansible.builtin.user:
+        name: telegraf
+        groups:
+          - systemd-journal
+        append: true