Adding support for IBM autopilot GPU metrics, alerts, dashboards (#57)

We wish to use the IBM Autopilot application. A Kubernetes-native daemon
that continuously monitors and evaluates GPUs, network and storage
health, designed to detect and report infrastructure-level issues during
the lifetime of AI workloads. We will deploy autopilot to the test and
prod cluster, and the grafana dashboard to the obs cluster.

clusters/lib/autopilot/application.yaml

@@ -0,0 +1,15 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: autopilot
+  labels:
+    nerc.mghpcc.org/sync-policy: common
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/ocp-on-nerc/nerc-ocp-config.git
+    targetRevision: HEAD
+    path: REPLACEME
+  destination:
+    name: REPLACEME
+    namespace: autopilot

clusters/lib/autopilot/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- application.yaml

clusters/nerc-ocp-obs/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 resources:
 - ../lib/cluster-scope
 - ../lib/logging
+- ../lib/autopilot
 - dex
 - loki
 - fake-metrics-server
@@ -47,3 +48,11 @@ patches:
       - op: replace
         path: /spec/source/path
         value: logging/overlays/nerc-ocp-obs
+
+  - target:
+      kind: Application
+      name: autopilot
+    patch: |
+      - op: replace
+        path: /spec/source/path
+        value: autopilot/overlays/nerc-ocp-obs

clusters/nerc-ocp-prod/kustomization.yaml

@@ -9,6 +9,7 @@ resources:
 - ../lib/nfd-operator
 - ../lib/nvidia-gpu-operator
 - ../lib/nerc-images
+- ../lib/autopilot
 - gatekeeper-policy
 - acct-mgt
 - curator
@@ -97,3 +98,11 @@ patches:
       - op: replace
         path: /spec/source/path
         value: nvidia-gpu-operator/overlays/nerc-ocp-prod
+
+  - target:
+      kind: Application
+      name: autopilot
+    patch: |
+      - op: replace
+        path: /spec/source/path
+        value: autopilot/overlays/nerc-ocp-prod

clusters/nerc-ocp-test/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
   - ../lib/virt
   - ../lib/nfs
   - ../lib/csi-driver-nfs
+  - ../lib/autopilot
 
 nameSuffix: -test
 
@@ -86,3 +87,11 @@ patches:
       - op: replace
         path: /spec/source/path
         value: csi-driver-nfs/overlays/nerc-ocp-test
+
+  - target:
+      kind: Application
+      name: autopilot
+    patch: |
+      - op: replace
+        path: /spec/source/path
+        value: autopilot/overlays/nerc-ocp-test