refactor: added alert suppression

errsole · Sep 18, 2024 · 5810ab2 · 5810ab2
1 parent 591b9fa
commit 5810ab2
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 35 deletions.
diff --git a/lib/main/server/utils/alerts.js b/lib/main/server/utils/alerts.js
@@ -2,49 +2,81 @@ const { getStorageConnection } = require('../storageConnection');
 const axios = require('axios');
 const nodemailer = require('nodemailer');
 
-// Improved handling by adding return statements and catching exceptions
-exports.customLoggerAlert = async function (message, messageExtraInfo, errsoleLogId) {
-  try {
-    await SlackService.sendAlert(message, 'Alert', messageExtraInfo, errsoleLogId);
-    await EmailService.sendAlert(message, 'Alert', messageExtraInfo, errsoleLogId);
-    return true; // Successfully sent alerts
-  } catch (error) {
-    console.error('Error in customLoggerAlert:', error);
-    return false; // Indicate failure
+const ALERT_COOLDOWN_MS = 60 * 1000; // 1 minutes
+const MAX_ALERT_ENTRIES = 100;
+
+const TOTAL_ALERT_LIMIT = 1000; // Max alerts per timeframe
+const ALERT_TIMEFRAME_MS = 10 * 60 * 1000; // 10 minutes
+
+const lastAlertTime = new Map();
+let alertCounter = 0;
+const alertQueue = [];
+
+setInterval(() => {
+  const now = Date.now();
+  for (const [message, timestamp] of lastAlertTime.entries()) {
+    if (now - timestamp > ALERT_COOLDOWN_MS) {
+      lastAlertTime.delete(message);
+    }
+  }
+}, ALERT_COOLDOWN_MS);
+
+setInterval(() => {
+  alertCounter = 0;
+  while (alertQueue.length > 0 && alertCounter < TOTAL_ALERT_LIMIT) {
+    const alert = alertQueue.shift();
+    sendAlert(alert);
+  }
+}, ALERT_TIMEFRAME_MS);
+
+function updateLastAlertTime (message) {
+  const now = Date.now();
+  lastAlertTime.set(message, now);
+  if (lastAlertTime.size > MAX_ALERT_ENTRIES) {
+    const oldestKey = lastAlertTime.keys().next().value;
+    lastAlertTime.delete(oldestKey);
+  }
+}
+
+async function sendAlert ({ message, type, messageExtraInfo, errsoleLogId }) {
+  const now = Date.now();
+  const lastSent = lastAlertTime.get(message) || 0;
+
+  if (now - lastSent > ALERT_COOLDOWN_MS && alertCounter < TOTAL_ALERT_LIMIT) {
+    const successSlack = await SlackService.sendAlert(message, type, messageExtraInfo, errsoleLogId);
+    const successEmail = await EmailService.sendAlert(message, type, messageExtraInfo, errsoleLogId);
+
+    if (successSlack || successEmail) {
+      updateLastAlertTime(message);
+      alertCounter++;
+    }
   }
+}
+
+exports.customLoggerAlert = async function (message, messageExtraInfo, errsoleLogId) {
+  return enqueueAlert({ message, type: 'Alert', messageExtraInfo, errsoleLogId });
 };
 
 exports.handleUncaughtExceptions = async function (message, messageExtraInfo, errsoleLogId) {
-  try {
-    await SlackService.sendAlert(message, 'Uncaught Exception', messageExtraInfo, errsoleLogId);
-    await EmailService.sendAlert(message, 'Uncaught Exception', messageExtraInfo, errsoleLogId);
-    return true; // Successfully handled exception
-  } catch (error) {
-    console.error('Error in handleUncaughtExceptions:', error);
-    return false; // Indicate failure
-  }
+  return enqueueAlert({ message, type: 'Uncaught Exception', messageExtraInfo, errsoleLogId });
 };
 
 exports.testSlackAlert = async function (message, messageExtraInfo) {
-  try {
-    const result = await SlackService.sendAlert(message, 'Test', messageExtraInfo);
-    return result; // Successfully sent alerts
-  } catch (error) {
-    console.error('Error in testSlackAlert:', error);
-    return false; // Indicate failure
-  }
+  return SlackService.sendAlert(message, 'Test', messageExtraInfo);
 };
 
 exports.testEmailAlert = async function (message, messageExtraInfo) {
-  try {
-    const result = await EmailService.sendAlert(message, 'Test', messageExtraInfo);
-    return result; // Successfully sent alerts
-  } catch (error) {
-    console.error('Error in testEmailAlert:', error);
-    return false; // Indicate failure
-  }
+  return EmailService.sendAlert(message, 'Test', messageExtraInfo);
 };
 
+function enqueueAlert (alert) {
+  if (alertCounter < TOTAL_ALERT_LIMIT) {
+    sendAlert(alert).catch(err => console.error('Failed to send alert:', err));
+  } else {
+    alertQueue.push(alert);
+  }
+}
+
 const SlackService = {};
 
 SlackService.sendAlert = async function (message, type, messageExtraInfo, errsoleLogId) {
@@ -54,8 +86,7 @@ SlackService.sendAlert = async function (message, type, messageExtraInfo, errsol
     if (data && data.item) {
       const parsedValue = JSON.parse(data.item.value);
       if (parsedValue.status === false) {
-        console.log('Slack integration is disabled.');
-        return false; // Slack integration is disabled
+        return false;
       }
       // create alert url
       const alertUrlData = await storageConnection.getConfig('alertUrl');
@@ -154,7 +185,6 @@ EmailService.sendAlert = async function (message, type, messageExtraInfo, errsol
       if (data && data.item) {
         const parsedValue = JSON.parse(data.item.value);
         if (parsedValue.status === false) {
-          console.log('Email integration is disabled.');
           return false; // Email integration is disabled
         }
         // create alert url

diff --git a/lib/web/src/actions/userActions.js b/lib/web/src/actions/userActions.js
@@ -80,7 +80,7 @@ export function getUserProfile (callback) {
       })
       .catch((error) => {
         if (error.response) {
-          if (error.response.status === 401) {
+          if (error.response.status === 401 || error.response.status === 403) {
             window.localStorage.removeItem('errsole-jwt-token');
             const { protocol, hostname, port, pathname } = window.location;
             const path = `${protocol}//${hostname}${port ? ':' + port : ''}${pathname}`;