historical_uptime: push precomputed chain height differences

Signed-off-by: bingyuyap <bingyu.yap.21@gmail.com>

fly/cmd/historical_uptime/main.go

@@ -59,6 +59,14 @@ var (
 		[]string{"guardian", "chain"},
 	)
 
+	guardianChainHeightDifferences = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "guardian_chain_height_differences",
+			Help: "Current height difference of each guardian from max height on each chain over time",
+		},
+		[]string{"guardian", "chain"},
+	)
+
 	guardianHeartbeats = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "guardian_heartbeats",
@@ -78,6 +86,11 @@ var (
 
 const PYTHNET_CHAIN_ID = int(vaa.ChainIDPythNet)
 
+var (
+	// guardianChainHeights indexes current chain height by chain id and guardian name
+	guardianChainHeights = make(common.GuardianChainHeights)
+)
+
 func loadEnvVars() {
 	err := godotenv.Load() // By default loads .env
 	if err != nil {
@@ -105,6 +118,18 @@ func verifyEnvVar(key string) string {
 	return value
 }
 
+func recordGuardianHeightDifferences() {
+	guardianHeightDifferencesByChain := historical_uptime.GetGuardianHeightDifferencesByChain(guardianChainHeights)
+
+	for chainId, guardianHeightDifferences := range guardianHeightDifferencesByChain {
+		chainName := vaa.ChainID(chainId).String()
+
+		for guardian, heightDifference := range guardianHeightDifferences {
+			guardianChainHeightDifferences.WithLabelValues(guardian, chainName).Set(float64(heightDifference))
+		}
+	}
+}
+
 func initPromScraper(promRemoteURL string, logger *zap.Logger, errC chan error) {
 	usingPromRemoteWrite := promRemoteURL != ""
 	if usingPromRemoteWrite {
@@ -125,6 +150,7 @@ func initPromScraper(promRemoteURL string, logger *zap.Logger, errC chan error)
 				case <-ctx.Done():
 					return nil
 				case <-t.C:
+					recordGuardianHeightDifferences()
 					for i := 1; i < 36; i++ {
 						if i == PYTHNET_CHAIN_ID {
 							continue
@@ -324,6 +350,11 @@ func main() {
 				}
 
 				for _, network := range hb.Networks {
+					if guardianChainHeights[network.Id] == nil {
+						guardianChainHeights[network.Id] = make(common.GuardianHeight)
+					}
+
+					guardianChainHeights[network.Id][guardianName] = uint64(network.Height)
 					guardianChainHeight.With(
 						prometheus.Labels{
 							"guardian": guardianName,
@@ -338,7 +369,6 @@ func main() {
 					},
 				).Set(float64(hb.Counter))
 
-
 			}
 		}
 	}()

fly/common/types.go

@@ -0,0 +1,7 @@
+package common
+
+type GuardianHeight map[string]uint64 // map of guardian addr to chain height
+
+type GuardianChainHeights map[uint32]GuardianHeight // map of chainIds to guardian heights
+
+type ChainHeights map[uint32]uint64 // map of chainIds to heights

fly/pkg/historical_uptime/helpers.go

@@ -69,3 +69,45 @@ func UpdateMetrics(guardianMissedObservations *prometheus.CounterVec, guardianMi
 		}
 	}
 }
+
+func computeMaxChainHeights(guardianChainHeights common.GuardianChainHeights) common.ChainHeights {
+	maxChainHeights := make(common.ChainHeights)
+
+	for chainId, guardianHeights  := range guardianChainHeights {
+		highest := uint64(0)
+
+		for _, guardianHeight := range guardianHeights {
+			if highest < guardianHeight {
+				highest = guardianHeight
+			}
+		}
+
+		maxChainHeights[chainId] = highest
+	}
+
+	return maxChainHeights
+}
+
+func computeGuardianChainHeightDifferences(guardianChainHeights common.GuardianChainHeights, maxChainHeights common.ChainHeights) common.GuardianChainHeights {
+	heightDifferences := make(common.GuardianChainHeights)
+
+	for chainId, guardianHeights := range guardianChainHeights {
+		for guardian, height := range guardianHeights {
+			if heightDifferences[chainId] == nil {
+				heightDifferences[chainId] = make(common.GuardianHeight)
+			}
+
+			// maxChainHeights[chain] always guaranteed to be at least height since it's computed in `computeMaxChainHeights`
+			heightDifferences[chainId][guardian] = maxChainHeights[chainId] - height
+		}
+	}
+
+	return heightDifferences
+}
+
+func GetGuardianHeightDifferencesByChain(guardianChainHeights common.GuardianChainHeights) common.GuardianChainHeights {
+	maxChainHeights := computeMaxChainHeights(guardianChainHeights)
+	return computeGuardianChainHeightDifferences(guardianChainHeights, maxChainHeights)
+}
+
+