From f2c1382b3473bfff3fad385fa5b5602d6b4f6dff Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Sat, 5 Oct 2024 02:42:26 +0530 Subject: [PATCH 01/14] system logs ui changes --- web-server/src/content/Service/SystemLogs.tsx | 106 +++++++++++++---- web-server/src/utils/logFormatter.ts | 109 ++++++++++++++++++ 2 files changed, 195 insertions(+), 20 deletions(-) create mode 100644 web-server/src/utils/logFormatter.ts diff --git a/web-server/src/content/Service/SystemLogs.tsx b/web-server/src/content/Service/SystemLogs.tsx index ccbe4d38..aafe1901 100644 --- a/web-server/src/content/Service/SystemLogs.tsx +++ b/web-server/src/content/Service/SystemLogs.tsx @@ -1,46 +1,112 @@ -import CircularProgress from '@mui/material/CircularProgress'; +import { CircularProgress, useTheme } from '@mui/material'; import { useEffect, useRef, useMemo } from 'react'; import { FlexBox } from '@/components/FlexBox'; import { Line } from '@/components/Text'; import { ServiceNames } from '@/constants/service'; import { useSelector } from '@/store'; +import { parseLogLine } from '@/utils/logFormater'; export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { + const theme = useTheme(); const services = useSelector((state) => state.service.services); const loading = useSelector((state) => state.service.loading); - const logs = useMemo(() => { - return services[serviceName].logs || []; - }, [serviceName, services]); + const logs = useMemo( + () => services[serviceName]?.logs || [], + [serviceName, services] + ); const containerRef = useRef(null); useEffect(() => { - if (containerRef.current) { - containerRef.current.scrollTop = containerRef.current.scrollHeight; + const container = containerRef.current; + if (container) { + container.scrollTop = container.scrollHeight; } }, [logs]); + const renderLogLine = (logLine: string, index: number) => { + const parsedLog = parseLogLine(logLine); + if (!parsedLog) return null; + + const { timestamp, ip, logLevel, message, metadata } = parsedLog; + + return ( + + + {timestamp} + {' '} + {ip && ( + + {ip}{' '} + + )} + + [{logLevel}] + {' '} + {message} + {metadata && ( + + {Object.entries(metadata).map(([key, value]) => ( + + {key}: {JSON.stringify(value)}{' '} + + ))} + + )} + + ); + }; + return ( - + {loading ? ( - - + + Loading... ) : ( - services && - logs.map((log, index) => ( - - {log} - - )) + logs.map(renderLogLine) )} ); }; + +const getLevelColor = (level: string, theme: any): string => { + const colors: { [key: string]: string } = { + INFO: theme.colors.success.main, + MAIN_INFO: theme.colors.success.main, + CHILD_INFO: theme.colors.success.main, + SENTINEL_INFO: theme.colors.success.main, + WARN: theme.colors.warning.main, + WARNING: theme.colors.warning.main, + NOTICE: theme.colors.warning.main, + ERROR: theme.colors.error.main, + FATAL: theme.colors.error.main, + PANIC: theme.colors.error.main, + DEBUG: theme.colors.info.main, + MAIN_SYSTEM: theme.colors.primary.main, + CHILD_SYSTEM: theme.colors.primary.main, + SENTINEL_SYSTEM: theme.colors.primary.main, + LOG: theme.colors.info.main, + CRITICAL: theme.colors.error.main + }; + + return colors[level.toUpperCase()] || theme.colors.info.main; +}; \ No newline at end of file diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts new file mode 100644 index 00000000..c446082a --- /dev/null +++ b/web-server/src/utils/logFormatter.ts @@ -0,0 +1,109 @@ +interface ParsedLog { + timestamp: string; + logLevel: string; + message: string; + ip?: string; + metadata?: Record; + } + + export const parseLogLine = (rawLogLine: string): ParsedLog | null => { + const generalLogRegex = + /^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; + const generalLogMatch = rawLogLine.match(generalLogRegex); + + if (generalLogMatch) { + const [, timestamp, id, logLevel, message] = generalLogMatch; + const metadata: Record = {}; + if (id) metadata.id = id; + return { + timestamp, + logLevel, + message, + ...(Object.keys(metadata).length > 0 && { metadata }) + }; + } + + const httpLogRegex = + /(\d+\.\d+\.\d+\.\d+) - - \[(.*?)\] "(GET|POST|PUT|DELETE) (.+?) (HTTP\/\d\.\d)" (\d+) (\d+)/; + const httpLogMatch = rawLogLine.match(httpLogRegex); + + if (httpLogMatch) { + const [, ip, timestamp, method, path, , status, size] = httpLogMatch; + const metadata: Record = {}; + if (size) metadata.size = size; + return { + timestamp, + logLevel: 'INFO', // Assuming all HTTP logs are INFO level + message: `${method} ${path} ${status}`, + ip, + ...(Object.keys(metadata).length > 0 && { metadata }) + }; + } + + const redisLogRegex = + /^(\d+):([CMS]) (\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2})\.\d{3} ([#*]) (.+)$/; + const redisLogMatch = rawLogLine.match(redisLogRegex); + + if (redisLogMatch) { + const [, pid, role, timestamp, logType, message] = redisLogMatch; + const metadata: Record = {}; + if (pid) metadata.pid = pid; + + let logLevel: string; + switch (role) { + case 'M': + logLevel = 'MAIN'; + break; + case 'C': + logLevel = 'CHILD'; + break; + case 'S': + logLevel = 'SENTINEL'; + break; + default: + logLevel = 'UNKNOWN'; + } + + logLevel += logType === '#' ? '_INFO' : '_SYSTEM'; + + return { + timestamp, + logLevel, + message, + ...(Object.keys(metadata).length > 0 && { metadata }) + }; + } + + const postgresLogRegex = + /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[(\d+)\] (\w+): (.+)$/; + const postgresLogMatch = rawLogLine.match(postgresLogRegex); + + if (postgresLogMatch) { + const [, timestamp, pid, logLevel, message] = postgresLogMatch; + const metadata: Record = {}; + if (pid) metadata.pid = pid; + + const validLogLevels = [ + 'DEBUG', + 'INFO', + 'NOTICE', + 'WARNING', + 'ERROR', + 'LOG', + 'FATAL', + 'PANIC' + ]; + const normalizedLogLevel = logLevel.toUpperCase(); + + return { + timestamp, + logLevel: validLogLevels.includes(normalizedLogLevel) + ? normalizedLogLevel + : 'INFO', + message, + ...(Object.keys(metadata).length > 0 && { metadata }) + }; + } + + return null; + }; \ No newline at end of file From d1d83cd1ed3ace0a8a7b645db4efcf024289d9c8 Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Sat, 5 Oct 2024 03:24:26 +0530 Subject: [PATCH 02/14] refact required codes --- web-server/src/content/Service/SystemLogs.tsx | 117 +++++++++--------- 1 file changed, 56 insertions(+), 61 deletions(-) diff --git a/web-server/src/content/Service/SystemLogs.tsx b/web-server/src/content/Service/SystemLogs.tsx index aafe1901..db3f93e5 100644 --- a/web-server/src/content/Service/SystemLogs.tsx +++ b/web-server/src/content/Service/SystemLogs.tsx @@ -8,81 +8,76 @@ import { useSelector } from '@/store'; import { parseLogLine } from '@/utils/logFormater'; export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { - const theme = useTheme(); const services = useSelector((state) => state.service.services); const loading = useSelector((state) => state.service.loading); - const logs = useMemo( - () => services[serviceName]?.logs || [], - [serviceName, services] - ); + const theme = useTheme(); + const logs = useMemo(() => { + return services[serviceName].logs || []; + }, [serviceName, services]); const containerRef = useRef(null); useEffect(() => { - const container = containerRef.current; - if (container) { - container.scrollTop = container.scrollHeight; + if (containerRef.current) { + containerRef.current.scrollTop = containerRef.current.scrollHeight; } }, [logs]); - const renderLogLine = (logLine: string, index: number) => { - const parsedLog = parseLogLine(logLine); - if (!parsedLog) return null; - - const { timestamp, ip, logLevel, message, metadata } = parsedLog; - - return ( - - - {timestamp} - {' '} - {ip && ( - - {ip}{' '} - - )} - - [{logLevel}] - {' '} - {message} - {metadata && ( - - {Object.entries(metadata).map(([key, value]) => ( - - {key}: {JSON.stringify(value)}{' '} - - ))} - - )} - - ); - }; - return ( - + {loading ? ( - - + + Loading... ) : ( - logs.map(renderLogLine) + services && + logs.map((log, index) => { + const parsedLog = parseLogLine(log); + if (!parsedLog) { + return ( + + {log} + + ); + } + const { timestamp, ip, logLevel, message, metadata } = parsedLog; + return ( + + + {timestamp} + {' '} + {ip && ( + + {ip}{' '} + + )} + + [{logLevel}] + {' '} + {message} + {metadata && ( + + {Object.entries(metadata).map(([key, value]) => ( + + {key}: {JSON.stringify(value)}{' '} + + ))} + + )} + + ); + }) )} ); From e531aa787a78d08f3df7effa56bf38efae8a51d7 Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Sat, 5 Oct 2024 11:58:46 +0530 Subject: [PATCH 03/14] removed metadata & fixed linting --- web-server/src/content/Service/SystemLogs.tsx | 66 +++--- web-server/src/utils/logFormatter.ts | 199 ++++++++---------- 2 files changed, 123 insertions(+), 142 deletions(-) diff --git a/web-server/src/content/Service/SystemLogs.tsx b/web-server/src/content/Service/SystemLogs.tsx index db3f93e5..6002f166 100644 --- a/web-server/src/content/Service/SystemLogs.tsx +++ b/web-server/src/content/Service/SystemLogs.tsx @@ -1,11 +1,11 @@ import { CircularProgress, useTheme } from '@mui/material'; -import { useEffect, useRef, useMemo } from 'react'; +import { useEffect, useRef, useMemo, useCallback } from 'react'; import { FlexBox } from '@/components/FlexBox'; import { Line } from '@/components/Text'; import { ServiceNames } from '@/constants/service'; import { useSelector } from '@/store'; -import { parseLogLine } from '@/utils/logFormater'; +import { parseLogLine } from '@/utils/logFormatter'; export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { const services = useSelector((state) => state.service.services); @@ -17,6 +17,32 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { const containerRef = useRef(null); + const getLevelColor = useCallback( + (level: string) => { + const colors: { [key: string]: string } = { + INFO: theme.colors.success.main, + MAIN_INFO: theme.colors.success.main, + CHILD_INFO: theme.colors.success.main, + SENTINEL_INFO: theme.colors.success.main, + WARN: theme.colors.warning.main, + WARNING: theme.colors.warning.main, + NOTICE: theme.colors.warning.main, + ERROR: theme.colors.error.main, + FATAL: theme.colors.error.main, + PANIC: theme.colors.error.main, + DEBUG: theme.colors.info.main, + MAIN_SYSTEM: theme.colors.primary.main, + CHILD_SYSTEM: theme.colors.primary.main, + SENTINEL_SYSTEM: theme.colors.primary.main, + LOG: theme.colors.info.main, + CRITICAL: theme.colors.error.main + }; + + return colors[level.toUpperCase()] || theme.colors.info.main; + }, + [theme] + ); + useEffect(() => { if (containerRef.current) { containerRef.current.scrollTop = containerRef.current.scrollHeight; @@ -46,7 +72,7 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { ); } - const { timestamp, ip, logLevel, message, metadata } = parsedLog; + const { timestamp, ip, logLevel, message } = parsedLog; return ( { {ip}{' '} )} - + [{logLevel}] {' '} {message} - {metadata && ( - - {Object.entries(metadata).map(([key, value]) => ( - - {key}: {JSON.stringify(value)}{' '} - - ))} - - )} ); }) )} ); -}; - -const getLevelColor = (level: string, theme: any): string => { - const colors: { [key: string]: string } = { - INFO: theme.colors.success.main, - MAIN_INFO: theme.colors.success.main, - CHILD_INFO: theme.colors.success.main, - SENTINEL_INFO: theme.colors.success.main, - WARN: theme.colors.warning.main, - WARNING: theme.colors.warning.main, - NOTICE: theme.colors.warning.main, - ERROR: theme.colors.error.main, - FATAL: theme.colors.error.main, - PANIC: theme.colors.error.main, - DEBUG: theme.colors.info.main, - MAIN_SYSTEM: theme.colors.primary.main, - CHILD_SYSTEM: theme.colors.primary.main, - SENTINEL_SYSTEM: theme.colors.primary.main, - LOG: theme.colors.info.main, - CRITICAL: theme.colors.error.main - }; - - return colors[level.toUpperCase()] || theme.colors.info.main; }; \ No newline at end of file diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts index c446082a..7711d741 100644 --- a/web-server/src/utils/logFormatter.ts +++ b/web-server/src/utils/logFormatter.ts @@ -1,109 +1,96 @@ interface ParsedLog { - timestamp: string; - logLevel: string; - message: string; - ip?: string; - metadata?: Record; + timestamp: string; + logLevel: string; + message: string; + ip?: string; +} + +export const parseLogLine = (rawLogLine: string): ParsedLog | null => { + const generalLogRegex = + /^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; + const generalLogMatch = rawLogLine.match(generalLogRegex); + + if (generalLogMatch) { + const [, timestamp, , logLevel, message] = generalLogMatch; + return { + timestamp, + logLevel, + message + }; } - - export const parseLogLine = (rawLogLine: string): ParsedLog | null => { - const generalLogRegex = - /^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; - const generalLogMatch = rawLogLine.match(generalLogRegex); - - if (generalLogMatch) { - const [, timestamp, id, logLevel, message] = generalLogMatch; - const metadata: Record = {}; - if (id) metadata.id = id; - return { - timestamp, - logLevel, - message, - ...(Object.keys(metadata).length > 0 && { metadata }) - }; - } - - const httpLogRegex = - /(\d+\.\d+\.\d+\.\d+) - - \[(.*?)\] "(GET|POST|PUT|DELETE) (.+?) (HTTP\/\d\.\d)" (\d+) (\d+)/; - const httpLogMatch = rawLogLine.match(httpLogRegex); - - if (httpLogMatch) { - const [, ip, timestamp, method, path, , status, size] = httpLogMatch; - const metadata: Record = {}; - if (size) metadata.size = size; - return { - timestamp, - logLevel: 'INFO', // Assuming all HTTP logs are INFO level - message: `${method} ${path} ${status}`, - ip, - ...(Object.keys(metadata).length > 0 && { metadata }) - }; - } - - const redisLogRegex = - /^(\d+):([CMS]) (\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2})\.\d{3} ([#*]) (.+)$/; - const redisLogMatch = rawLogLine.match(redisLogRegex); - - if (redisLogMatch) { - const [, pid, role, timestamp, logType, message] = redisLogMatch; - const metadata: Record = {}; - if (pid) metadata.pid = pid; - - let logLevel: string; - switch (role) { - case 'M': - logLevel = 'MAIN'; - break; - case 'C': - logLevel = 'CHILD'; - break; - case 'S': - logLevel = 'SENTINEL'; - break; - default: - logLevel = 'UNKNOWN'; - } - - logLevel += logType === '#' ? '_INFO' : '_SYSTEM'; - - return { - timestamp, - logLevel, - message, - ...(Object.keys(metadata).length > 0 && { metadata }) - }; - } - - const postgresLogRegex = - /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[(\d+)\] (\w+): (.+)$/; - const postgresLogMatch = rawLogLine.match(postgresLogRegex); - - if (postgresLogMatch) { - const [, timestamp, pid, logLevel, message] = postgresLogMatch; - const metadata: Record = {}; - if (pid) metadata.pid = pid; - - const validLogLevels = [ - 'DEBUG', - 'INFO', - 'NOTICE', - 'WARNING', - 'ERROR', - 'LOG', - 'FATAL', - 'PANIC' - ]; - const normalizedLogLevel = logLevel.toUpperCase(); - - return { - timestamp, - logLevel: validLogLevels.includes(normalizedLogLevel) - ? normalizedLogLevel - : 'INFO', - message, - ...(Object.keys(metadata).length > 0 && { metadata }) - }; + + const httpLogRegex = + /(\d+\.\d+\.\d+\.\d+) - - \[(.*?)\] "(GET|POST|PUT|DELETE) (.+?) (HTTP\/\d\.\d)" (\d+) (\d+)/; + const httpLogMatch = rawLogLine.match(httpLogRegex); + + if (httpLogMatch) { + const [, ip, timestamp, method, path, , status] = httpLogMatch; + return { + timestamp, + logLevel: 'INFO', // Assuming all HTTP logs are INFO level + message: `${method} ${path} ${status}`, + ip + }; + } + + const redisLogRegex = + /^(\d+):([CMS]) (\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2})\.\d{3} ([#*]) (.+)$/; + const redisLogMatch = rawLogLine.match(redisLogRegex); + + if (redisLogMatch) { + const [, , role, timestamp, logType, message] = redisLogMatch; + + let logLevel: string; + switch (role) { + case 'M': + logLevel = 'MAIN'; + break; + case 'C': + logLevel = 'CHILD'; + break; + case 'S': + logLevel = 'SENTINEL'; + break; + default: + logLevel = 'UNKNOWN'; } - - return null; - }; \ No newline at end of file + + logLevel += logType === '#' ? '_INFO' : '_SYSTEM'; + + return { + timestamp, + logLevel, + message + }; + } + + const postgresLogRegex = + /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[(\d+)\] (\w+): (.+)$/; + const postgresLogMatch = rawLogLine.match(postgresLogRegex); + + if (postgresLogMatch) { + const [, timestamp, , logLevel, message] = postgresLogMatch; + const validLogLevels = new Set([ + 'DEBUG', + 'INFO', + 'NOTICE', + 'WARNING', + 'ERROR', + 'LOG', + 'FATAL', + 'PANIC' + ]); + + const normalizedLogLevel = logLevel.toUpperCase(); + + return { + timestamp, + logLevel: validLogLevels.has(normalizedLogLevel) + ? normalizedLogLevel + : 'INFO', + message + }; + } + + return null; +}; \ No newline at end of file From 98fd7000a71f12b2bfc575077579b3a3de6f5b15 Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Tue, 8 Oct 2024 03:58:54 +0530 Subject: [PATCH 04/14] Refactor logFormatter.ts to improve log parsing and add support for data synchronization logs --- web-server/src/utils/logFormatter.ts | 41 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts index 7711d741..e09e4463 100644 --- a/web-server/src/utils/logFormatter.ts +++ b/web-server/src/utils/logFormatter.ts @@ -5,11 +5,15 @@ interface ParsedLog { ip?: string; } +const generalLogRegex =/^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; +const httpLogRegex =/^(\S+) (\S+) (\S+) \[([^\]]+)\] "([^"]*)" (\d+) (\d+) "([^"]*)" "([^"]*)"$/; +const redisLogRegex =/^(\d+):([CMS]) (\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2})\.\d{3} ([#*]) (.+)$/; +const postgresLogRegex =/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[(\d+)\] (\w+): (.+)$/; +const dataSyncLogRegex = /\[(\w+)\]\s(.+?)\sfor\s(\w+)\s(.+)/; + export const parseLogLine = (rawLogLine: string): ParsedLog | null => { - const generalLogRegex = - /^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; - const generalLogMatch = rawLogLine.match(generalLogRegex); + const generalLogMatch = rawLogLine.match(generalLogRegex); if (generalLogMatch) { const [, timestamp, , logLevel, message] = generalLogMatch; return { @@ -19,24 +23,20 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { }; } - const httpLogRegex = - /(\d+\.\d+\.\d+\.\d+) - - \[(.*?)\] "(GET|POST|PUT|DELETE) (.+?) (HTTP\/\d\.\d)" (\d+) (\d+)/; const httpLogMatch = rawLogLine.match(httpLogRegex); - if (httpLogMatch) { - const [, ip, timestamp, method, path, , status] = httpLogMatch; + const [, ip, , , timestamp, request, status, bytes, referer, userAgent] = httpLogMatch; + const [method, path] = request.split(' '); return { - timestamp, + timestamp: timestamp.replace(/(\d{2})\/(\w{3})\/(\d{4}):(\d{2}:\d{2}:\d{2})/, '$3-$2-$1 $4'), logLevel: 'INFO', // Assuming all HTTP logs are INFO level - message: `${method} ${path} ${status}`, + message: `${method} ${path} ${status} ${bytes} "${referer}" "${userAgent}"`, ip }; } - const redisLogRegex = - /^(\d+):([CMS]) (\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2})\.\d{3} ([#*]) (.+)$/; - const redisLogMatch = rawLogLine.match(redisLogRegex); + const redisLogMatch = rawLogLine.match(redisLogRegex); if (redisLogMatch) { const [, , role, timestamp, logType, message] = redisLogMatch; @@ -64,10 +64,8 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { }; } - const postgresLogRegex = - /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[(\d+)\] (\w+): (.+)$/; - const postgresLogMatch = rawLogLine.match(postgresLogRegex); + const postgresLogMatch = rawLogLine.match(postgresLogRegex); if (postgresLogMatch) { const [, timestamp, , logLevel, message] = postgresLogMatch; const validLogLevels = new Set([ @@ -82,7 +80,6 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { ]); const normalizedLogLevel = logLevel.toUpperCase(); - return { timestamp, logLevel: validLogLevels.has(normalizedLogLevel) @@ -92,5 +89,17 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { }; } + + const dataSyncLogMatch = rawLogLine.match(dataSyncLogRegex); + if (dataSyncLogMatch) { + const [, logLevel, action, service, message] = dataSyncLogMatch; + return { + timestamp: '', + logLevel: logLevel.toUpperCase(), + message: `${action} for ${service} ${message}` + }; + } + + return null; }; \ No newline at end of file From 8c7e282958d19dd0784e9aec9507a7d1e823e92b Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Tue, 8 Oct 2024 17:05:09 +0530 Subject: [PATCH 05/14] improved redis parser --- web-server/src/content/Service/SystemLogs.tsx | 6 +- web-server/src/utils/logFormatter.ts | 80 +++++++++++-------- 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/web-server/src/content/Service/SystemLogs.tsx b/web-server/src/content/Service/SystemLogs.tsx index 6002f166..dcecdfca 100644 --- a/web-server/src/content/Service/SystemLogs.tsx +++ b/web-server/src/content/Service/SystemLogs.tsx @@ -26,11 +26,11 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { SENTINEL_INFO: theme.colors.success.main, WARN: theme.colors.warning.main, WARNING: theme.colors.warning.main, - NOTICE: theme.colors.warning.main, + NOTICE: theme.colors.warning.dark, ERROR: theme.colors.error.main, FATAL: theme.colors.error.main, PANIC: theme.colors.error.main, - DEBUG: theme.colors.info.main, + DEBUG: theme.colors.info.light, MAIN_SYSTEM: theme.colors.primary.main, CHILD_SYSTEM: theme.colors.primary.main, SENTINEL_SYSTEM: theme.colors.primary.main, @@ -98,4 +98,4 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { )} ); -}; \ No newline at end of file +}; diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts index e09e4463..5da0db55 100644 --- a/web-server/src/utils/logFormatter.ts +++ b/web-server/src/utils/logFormatter.ts @@ -2,14 +2,28 @@ interface ParsedLog { timestamp: string; logLevel: string; message: string; + role?: string; ip?: string; } const generalLogRegex =/^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; const httpLogRegex =/^(\S+) (\S+) (\S+) \[([^\]]+)\] "([^"]*)" (\d+) (\d+) "([^"]*)" "([^"]*)"$/; -const redisLogRegex =/^(\d+):([CMS]) (\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2})\.\d{3} ([#*]) (.+)$/; -const postgresLogRegex =/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[(\d+)\] (\w+): (.+)$/; +const redisLogRegex = /^(?\d+:[XCMS]) (?\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2}\.\d{3}) (?[\.\-\#\*]) (?.*)$/; +const postgresLogRegex =/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \w+) \[(\d+)\] (\w+):(.+)(?:\n(?:\s+.*)?)*$/m; +const postgresMultiLineLogRegex = /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[\d+\] (?[A-Z]+):\s*(?(.|\n)+?)$/; const dataSyncLogRegex = /\[(\w+)\]\s(.+?)\sfor\s(\w+)\s(.+)/; +const validLogLevels = new Set([ + 'DEBUG', + 'INFO', + 'NOTICE', + 'WARNING', + 'ERROR', + 'LOG', + 'FATAL', + 'PANIC', + 'STATEMENT', + 'DETAIL' +]); export const parseLogLine = (rawLogLine: string): ParsedLog | null => { @@ -28,56 +42,58 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { const [, ip, , , timestamp, request, status, bytes, referer, userAgent] = httpLogMatch; const [method, path] = request.split(' '); return { - timestamp: timestamp.replace(/(\d{2})\/(\w{3})\/(\d{4}):(\d{2}:\d{2}:\d{2})/, '$3-$2-$1 $4'), + timestamp, logLevel: 'INFO', // Assuming all HTTP logs are INFO level message: `${method} ${path} ${status} ${bytes} "${referer}" "${userAgent}"`, ip }; } - - const redisLogMatch = rawLogLine.match(redisLogRegex); - if (redisLogMatch) { - const [, , role, timestamp, logType, message] = redisLogMatch; - + const redisMatch = rawLogLine.match(redisLogRegex); + if (redisMatch) { + const { role, timestamp, loglevel, message } = redisMatch.groups; let logLevel: string; - switch (role) { - case 'M': - logLevel = 'MAIN'; + switch (loglevel) { + case '.': + logLevel = 'DEBUG'; break; - case 'C': - logLevel = 'CHILD'; + case '-': + logLevel = 'INFO'; break; - case 'S': - logLevel = 'SENTINEL'; + case '*': + logLevel = 'NOTICE'; + break; + case '#': + logLevel = 'WARNING'; break; default: - logLevel = 'UNKNOWN'; + logLevel = 'INFO'; } - - logLevel += logType === '#' ? '_INFO' : '_SYSTEM'; - return { + role, timestamp, logLevel, - message + message: message.trim() }; } + const postgresMultiLineLogMatch = rawLogLine.match(postgresMultiLineLogRegex); + if (postgresMultiLineLogMatch) { + const { timestamp, loglevel, message } = postgresMultiLineLogMatch.groups; + const normalizedLogLevel = loglevel.toUpperCase(); + + return { + timestamp: timestamp, + logLevel: validLogLevels.has(normalizedLogLevel) + ? normalizedLogLevel + : 'INFO', + message: message.trim() + }; + } const postgresLogMatch = rawLogLine.match(postgresLogRegex); if (postgresLogMatch) { const [, timestamp, , logLevel, message] = postgresLogMatch; - const validLogLevels = new Set([ - 'DEBUG', - 'INFO', - 'NOTICE', - 'WARNING', - 'ERROR', - 'LOG', - 'FATAL', - 'PANIC' - ]); const normalizedLogLevel = logLevel.toUpperCase(); return { @@ -94,7 +110,7 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { if (dataSyncLogMatch) { const [, logLevel, action, service, message] = dataSyncLogMatch; return { - timestamp: '', + timestamp: '', logLevel: logLevel.toUpperCase(), message: `${action} for ${service} ${message}` }; @@ -102,4 +118,4 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { return null; -}; \ No newline at end of file +}; From 591602dd27b3b2302f3135021a32ab2f7dac3c77 Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Tue, 8 Oct 2024 17:09:45 +0530 Subject: [PATCH 06/14] Add tests for logFormatter --- .../src/utils/__tests__/logFormatter.test.ts | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 web-server/src/utils/__tests__/logFormatter.test.ts diff --git a/web-server/src/utils/__tests__/logFormatter.test.ts b/web-server/src/utils/__tests__/logFormatter.test.ts new file mode 100644 index 00000000..4b1975f9 --- /dev/null +++ b/web-server/src/utils/__tests__/logFormatter.test.ts @@ -0,0 +1,236 @@ +import { parseLogLine } from '../logFormatter'; + +describe('Should correctly parse logs', () => { + it('Should correctly parse a log line', () => { + const rawLog = + '[2024-10-08 04:21:42 +0000] [164] [INFO] Booting worker with pid: 164'; + const expected = { + timestamp: '2024-10-08 04:21:42 +0000', + logLevel: 'INFO', + message: 'Booting worker with pid: 164' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('Should correctly parse a log line with different log level', () => { + const rawLog = + '[2024-10-08 04:21:42 +0000] [164] [ERROR] Booting worker with pid: 164'; + const expected = { + timestamp: '2024-10-08 04:21:42 +0000', + logLevel: 'ERROR', + message: 'Booting worker with pid: 164' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('Should correctly parse a with without log level', () => { + const rawLog = + '127.0.0.1 - - [08/Oct/2024:04:25:26 +0000] "GET / HTTP/1.1" 200 26 "-" "axios/0.26.1"'; + const expected = { + timestamp: '08/Oct/2024:04:25:26 +0000', + logLevel: 'INFO', + message: 'GET / 200 26 "-" "axios/0.26.1"', + ip: '127.0.0.1' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('Should correctly parse a log line without timestamp', () => { + const rawLog = + '[INFO] Data sync for sync_org_incidents completed successfully'; + const expected = { + timestamp: '', + logLevel: 'INFO', + message: 'Data sync for sync_org_incidents completed successfully' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('Should correctly parse a log line with different date format : [08/Oct/2024:04:25:16 +0000]', () => { + const rawLog = + '127.0.0.1 - - [08/Oct/2024:04:25:16 +0000] "GET / HTTP/1.1" 200 26 "-" "axios/0.26.1"'; + const expected = { + ip: '127.0.0.1', + logLevel: 'INFO', + message: 'GET / 200 26 "-" "axios/0.26.1"', + timestamp: '08/Oct/2024:04:25:16 +0000' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('Should correctly parse a log line with different date format : 2024-10-08 04:21:40.288 UTC', () => { + const rawLog = + '2024-10-08 04:21:40.288 UTC [49] LOG: starting PostgreSQL 15.8 (Debian 15.8-0+deb12u1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit'; + const expected = { + timestamp: '2024-10-08 04:21:40.288 UTC', + logLevel: 'LOG', + message: + 'starting PostgreSQL 15.8 (Debian 15.8-0+deb12u1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('Should correctly parse a log line with different date format : 08 Oct 2024 04:21:40.149', () => { + const rawLog = `51:M 08 Oct 2024 04:21:40.149 # Server initialized`; + const expected = { + role: '51:M', + timestamp: '08 Oct 2024 04:21:40.149', + logLevel: 'WARNING', + message: 'Server initialized' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse postgres log line', () => { + const rawLog = `2024-09-27 12:14:45.280 UTC [56] LOG: checkpoint starting: time`; + const expected = { + timestamp: '2024-09-27 12:14:45.280 UTC', + logLevel: 'LOG', + message: 'checkpoint starting: time' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse postgres log line with different log level : ERROR', () => { + const rawLog = `2024-09-24 07:25:08.658 UTC [160] ERROR: duplicate key value violates unique constraint "languages_pkey"`; + const expected = { + timestamp: '2024-09-24 07:25:08.658 UTC', + logLevel: 'ERROR', + message: 'duplicate key value violates unique constraint "languages_pkey"' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse postgres log line with different log level : STATEMENT', () => { + const rawLog = `2024-09-24 07:25:07.487 UTC [159] STATEMENT: INSERT INTO "public"."Language" ("createdAt","id","judge0Id","updatedAt","name") VALUES ($1,$2,$3,$4,$5), ($6,$7,$8,$9,$10)`; + const expected = { + timestamp: '2024-09-24 07:25:07.487 UTC', + logLevel: 'STATEMENT', + message: + 'INSERT INTO "public"."Language" ("createdAt","id","judge0Id","updatedAt","name") VALUES ($1,$2,$3,$4,$5), ($6,$7,$8,$9,$10)' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse postgres log line with different log level : ERROR', () => { + const rawLog = `2024-09-24 07:25:08.658 UTC [160] ERROR: duplicate key value violates unique constraint "languages_pkey"`; + const expected = { + timestamp: '2024-09-24 07:25:08.658 UTC', + logLevel: 'ERROR', + message: 'duplicate key value violates unique constraint "languages_pkey"' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse postgres log line with different log level : FATAL', () => { + const rawLog = `2024-10-05 12:00:43.377 UTC [48661] FATAL: unsupported frontend protocol 65363.19778: server supports 3.0 to 3.0`; + const expected = { + timestamp: '2024-10-05 12:00:43.377 UTC', + logLevel: 'FATAL', + message: + 'unsupported frontend protocol 65363.19778: server supports 3.0 to 3.0' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse postgres log line with different log level : DETAIL', () => { + const rawLog = `2024-09-24 07:25:08.658 UTC [160] DETAIL: Key (id)=(43) already exists.`; + const expected = { + timestamp: '2024-09-24 07:25:08.658 UTC', + logLevel: 'DETAIL', + message: 'Key (id)=(43) already exists.' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse postgres log line with different log level : DETAIL', () => { + const rawLog = `2024-09-24 07:25:08.658 UTC [160] DETAIL: Key (id)=(43) already exists.`; + const expected = { + timestamp: '2024-09-24 07:25:08.658 UTC', + logLevel: 'DETAIL', + message: 'Key (id)=(43) already exists.' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse large postgres log line ', () => { + const rawLog = `2024-09-24 07:25:08.658 UTC [160] STATEMENT: INSERT INTO "public"."languages" ("compile_cmd","source_file","id","name","is_archived","run_cmd") VALUES (null,$1,$2,$3,$4,$5), (null,$6,$7,$8,$9,$10), ($11,$12,$13,$14,$15,$16), (null,$17,$18,$19,$20,$21), ($22,$23,$24,$25,$26,$27), ($28,$29,$30,$31,$32,$33), ($34,$35,$36,$37,$38,$39), ($40,$41,$42,$43,$44,$45), ($46,$47,$48,$49,$50,$51), ($52,$53,$54,$55,$56,$57), ($58,$59,$60,$61,$62,$63), ($64,$65,$66,$67,$68,$69), (null,$70,$71,$72,$73,$74), ($75,$76,$77,$78,$79,$80), (null,$81,$82,$83,$84,$85), (null,$86,$87,$88,$89,$90), ($91,$92,$93,$94,$95,$96), ($97,$98,$99,$100,$101,$102), ($103,$104,$105,$106,$107,$108), ($109,$110,$111,$112,$113,$114), (null,$115,$116,$117,$118,$119), ($120,$121,$122,$123,$124,$125), ($126,$127,$128,$129,$130,$131), (null,$132,$133,$134,$135,$136), ($137,$138,$139,$140,$141,$142), (null,$143,$144,$145,$146,$147), ($148,$149,$150,$151,$152,$153), (null,$154,$155,$156,$157,$158), (null,$159,$160,$161,$162,$163), (null,$164,$165,$166,$167,$168), ($169,$170,$171,$172,$173,$174), ($175,$176,$177,$178,$179,$180), ($181,$182,$183,$184,$185,$186), ($187,$188,$189,$190,$191,$192), ($193,$194,$195,$196,$197,$198), ($199,$200,$201,$202,$203,$204), ($205,$206,$207,$208,$209,$210), (null,$211,$212,$213,$214,$215), ($216,$217,$218,$219,$220,$221), (null,$222,$223,$224,$225,$226), ($227,$228,$229,$230,$231,$232), ($233,$234,$235,$236,$237,$238), (null,$239,$240,$241,$242,$243), (null,$244,$245,$246,$247,$248), (null,$249,$250,$251,$252,$253), ($254,$255,$256,$257,$258,$259), (null,null,$260,$261,$262,null)`; + const expected = { + timestamp: '2024-09-24 07:25:08.658 UTC', + logLevel: 'STATEMENT', + message: + 'INSERT INTO "public"."languages" ("compile_cmd","source_file","id","name","is_archived","run_cmd") VALUES (null,$1,$2,$3,$4,$5), (null,$6,$7,$8,$9,$10), ($11,$12,$13,$14,$15,$16), (null,$17,$18,$19,$20,$21), ($22,$23,$24,$25,$26,$27), ($28,$29,$30,$31,$32,$33), ($34,$35,$36,$37,$38,$39), ($40,$41,$42,$43,$44,$45), ($46,$47,$48,$49,$50,$51), ($52,$53,$54,$55,$56,$57), ($58,$59,$60,$61,$62,$63), ($64,$65,$66,$67,$68,$69), (null,$70,$71,$72,$73,$74), ($75,$76,$77,$78,$79,$80), (null,$81,$82,$83,$84,$85), (null,$86,$87,$88,$89,$90), ($91,$92,$93,$94,$95,$96), ($97,$98,$99,$100,$101,$102), ($103,$104,$105,$106,$107,$108), ($109,$110,$111,$112,$113,$114), (null,$115,$116,$117,$118,$119), ($120,$121,$122,$123,$124,$125), ($126,$127,$128,$129,$130,$131), (null,$132,$133,$134,$135,$136), ($137,$138,$139,$140,$141,$142), (null,$143,$144,$145,$146,$147), ($148,$149,$150,$151,$152,$153), (null,$154,$155,$156,$157,$158), (null,$159,$160,$161,$162,$163), (null,$164,$165,$166,$167,$168), ($169,$170,$171,$172,$173,$174), ($175,$176,$177,$178,$179,$180), ($181,$182,$183,$184,$185,$186), ($187,$188,$189,$190,$191,$192), ($193,$194,$195,$196,$197,$198), ($199,$200,$201,$202,$203,$204), ($205,$206,$207,$208,$209,$210), (null,$211,$212,$213,$214,$215), ($216,$217,$218,$219,$220,$221), (null,$222,$223,$224,$225,$226), ($227,$228,$229,$230,$231,$232), ($233,$234,$235,$236,$237,$238), (null,$239,$240,$241,$242,$243), (null,$244,$245,$246,$247,$248), (null,$249,$250,$251,$252,$253), ($254,$255,$256,$257,$258,$259), (null,null,$260,$261,$262,null)' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('POSTGRES : Should parse postgres with different log lines', () => { + const rawLog = `2021-12-03 07:14:48.594 UTC [4913] DETAIL: parameters: $1 = '1234', $2 = 'njchar + hello', $3 = 'aaaa'`; + + const expected = { + timestamp: '2021-12-03 07:14:48.594 UTC', + logLevel: 'DETAIL', + message: String.raw`parameters: $1 = '1234', $2 = 'njchar + hello', $3 = 'aaaa'` + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('REDIS : Should parse redis log line', () => { + const rawLog = `51:C 08 Oct 2024 04:21:40.147 # Redis version=7.0.15, bits=64, commit=00000000, modified=0, pid=51, just started`; + const expected = { + role: '51:C', + timestamp: '08 Oct 2024 04:21:40.147', + logLevel: 'WARNING', + message: + 'Redis version=7.0.15, bits=64, commit=00000000, modified=0, pid=51, just started' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('REDIS : Should parse redis log line with different log level : # -> WARNING', () => { + const rawLog = `51:M 08 Oct 2024 04:21:40.149 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.`; + const expected = { + role: '51:M', + timestamp: '08 Oct 2024 04:21:40.149', + logLevel: 'WARNING', + message: + "WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect." + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('REDIS : Should parse redis log line with different log level : * -> NOTICE', () => { + const rawLog = `51:M 08 Oct 2024 04:21:40.150 * Ready to accept connections`; + const expected = { + role: '51:M', + timestamp: '08 Oct 2024 04:21:40.150', + logLevel: 'NOTICE', + message: 'Ready to accept connections' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('REDIS : Should parse redis log line with different log level : * -> NOTICE', () => { + const rawLog = `51:M 08 Oct 2024 04:21:40.150 * Ready to accept connections`; + const expected = { + role: '51:M', + timestamp: '08 Oct 2024 04:21:40.150', + logLevel: 'NOTICE', + message: 'Ready to accept connections' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); + + it('REDIS : Should parse redis log line with different log level : - -> INFO', () => { + const rawLog = `53:C 08 Oct 2024 05:05:20.600 - INFO Connected to the Redis server at 127.0.0.1:6379`; + const expected = { + role: '53:C', + timestamp: '08 Oct 2024 05:05:20.600', + logLevel: 'INFO', + message: 'INFO Connected to the Redis server at 127.0.0.1:6379' + }; + expect(parseLogLine(rawLog)).toEqual(expected); + }); +}); From 0dc350eae02ab36ce1dbe3e150284fb67de44c69 Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Thu, 10 Oct 2024 17:42:52 +0530 Subject: [PATCH 07/14] lint fix --- web-server/src/utils/logFormatter.ts | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts index 5da0db55..c3e6ca27 100644 --- a/web-server/src/utils/logFormatter.ts +++ b/web-server/src/utils/logFormatter.ts @@ -6,11 +6,16 @@ interface ParsedLog { ip?: string; } -const generalLogRegex =/^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; -const httpLogRegex =/^(\S+) (\S+) (\S+) \[([^\]]+)\] "([^"]*)" (\d+) (\d+) "([^"]*)" "([^"]*)"$/; -const redisLogRegex = /^(?\d+:[XCMS]) (?\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2}\.\d{3}) (?[\.\-\#\*]) (?.*)$/; -const postgresLogRegex =/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \w+) \[(\d+)\] (\w+):(.+)(?:\n(?:\s+.*)?)*$/m; -const postgresMultiLineLogRegex = /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[\d+\] (?[A-Z]+):\s*(?(.|\n)+?)$/; +const generalLogRegex = + /^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; +const httpLogRegex = + /^(\S+) (\S+) (\S+) \[([^\]]+)\] "([^"]*)" (\d+) (\d+) "([^"]*)" "([^"]*)"$/; +const redisLogRegex = + /^(?\d+:[XCMS]) (?\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2}\.\d{3}) (?[\.\-\#\*]) (?.*)$/; +const postgresLogRegex = + /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \w+) \[(\d+)\] (\w+):(.+)(?:\n(?:\s+.*)?)*$/m; +const postgresMultiLineLogRegex = + /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[\d+\] (?[A-Z]+):\s*(?(.|\n)+?)$/; const dataSyncLogRegex = /\[(\w+)\]\s(.+?)\sfor\s(\w+)\s(.+)/; const validLogLevels = new Set([ 'DEBUG', @@ -26,7 +31,6 @@ const validLogLevels = new Set([ ]); export const parseLogLine = (rawLogLine: string): ParsedLog | null => { - const generalLogMatch = rawLogLine.match(generalLogRegex); if (generalLogMatch) { const [, timestamp, , logLevel, message] = generalLogMatch; @@ -39,7 +43,8 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { const httpLogMatch = rawLogLine.match(httpLogRegex); if (httpLogMatch) { - const [, ip, , , timestamp, request, status, bytes, referer, userAgent] = httpLogMatch; + const [, ip, , , timestamp, request, status, bytes, referer, userAgent] = + httpLogMatch; const [method, path] = request.split(' '); return { timestamp, @@ -85,8 +90,8 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { return { timestamp: timestamp, logLevel: validLogLevels.has(normalizedLogLevel) - ? normalizedLogLevel - : 'INFO', + ? normalizedLogLevel + : 'INFO', message: message.trim() }; } @@ -105,7 +110,6 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { }; } - const dataSyncLogMatch = rawLogLine.match(dataSyncLogRegex); if (dataSyncLogMatch) { const [, logLevel, action, service, message] = dataSyncLogMatch; @@ -116,6 +120,5 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { }; } - return null; }; From 5eddb700e5dd061f7e62873571c2fe1401a03d9f Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Thu, 10 Oct 2024 21:26:16 +0530 Subject: [PATCH 08/14] Add Error Fallback for log formatter --- web-server/src/content/Service/SystemLogs.tsx | 65 ++++++----- .../Service/SystemLogsErrorFallback.tsx | 105 ++++++++++++++++++ 2 files changed, 142 insertions(+), 28 deletions(-) create mode 100644 web-server/src/content/Service/SystemLogsErrorFallback.tsx diff --git a/web-server/src/content/Service/SystemLogs.tsx b/web-server/src/content/Service/SystemLogs.tsx index 695e0005..994cd541 100644 --- a/web-server/src/content/Service/SystemLogs.tsx +++ b/web-server/src/content/Service/SystemLogs.tsx @@ -1,5 +1,5 @@ import { CircularProgress, useTheme } from '@mui/material'; -import { useEffect, useRef, useMemo, useCallback } from 'react'; +import { useEffect, useRef, useMemo, useCallback, useState } from 'react'; import { FlexBox } from '@/components/FlexBox'; import { Line } from '@/components/Text'; @@ -7,9 +7,12 @@ import { ServiceNames } from '@/constants/service'; import { useSelector } from '@/store'; import { parseLogLine } from '@/utils/logFormatter'; +import { SystemLogsErrorFallback } from './SystemLogsErrorFallback'; + export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { const services = useSelector((state) => state.service.services); const loading = useSelector((state) => state.service.loading); + const [error, setError] = useState(null); const theme = useTheme(); const logs = useMemo(() => { return services[serviceName].logs || []; @@ -56,11 +59,24 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { Loading... - ) : ( - services && + ) : !error && services ? ( logs.map((log, index) => { - const parsedLog = parseLogLine(log); - if (!parsedLog) { + try { + const parsedLog = parseLogLine(log); + + if (!parsedLog) { + return ( + + {log} + + ); + } + const { timestamp, ip, logLevel, message } = parsedLog; return ( { fontSize={'14px'} fontFamily={'monospace'} > - {log} + + {timestamp} + {' '} + {ip && ( + + {ip}{' '} + + )} + + [{logLevel}] + {' '} + {message} ); + } catch (error: any) { + setError(error); } - const { timestamp, ip, logLevel, message } = parsedLog; - return ( - - - {timestamp} - {' '} - {ip && ( - - {ip}{' '} - - )} - - [{logLevel}] - {' '} - {message} - - ); }) + ) : ( + )} diff --git a/web-server/src/content/Service/SystemLogsErrorFallback.tsx b/web-server/src/content/Service/SystemLogsErrorFallback.tsx new file mode 100644 index 00000000..8648d0c5 --- /dev/null +++ b/web-server/src/content/Service/SystemLogsErrorFallback.tsx @@ -0,0 +1,105 @@ +import { CopyAll } from '@mui/icons-material'; +import { Button, Typography } from '@mui/material'; +import CircularProgress from '@mui/material/CircularProgress'; +import { useSnackbar } from 'notistack'; +import { useEffect, useRef, useMemo } from 'react'; +import CopyToClipboard from 'react-copy-to-clipboard'; + +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { track } from '@/constants/events'; +import { ServiceNames } from '@/constants/service'; +import { useSelector } from '@/store'; + +export const SystemLogsErrorFallback = ({ + error, + serviceName +}: { + error: Error; + serviceName: ServiceNames; +}) => { + const services = useSelector((state) => state.service.services); + const loading = useSelector((state) => state.service.loading); + const logs = useMemo(() => { + return services[serviceName].logs || []; + }, [serviceName, services]); + + const containerRef = useRef(null); + const { enqueueSnackbar } = useSnackbar(); + const errorBody = useMemo( + () => ({ + message: error?.message?.replace('\\n', '\n'), + stack: error?.stack?.replace('\\n', '\n') + }), + [error?.message, error?.stack] + ); + + useEffect(() => { + if (containerRef.current) { + containerRef.current.scrollIntoView({ behavior: 'smooth' }); + } + }, [logs]); + + useEffect(() => { + track('ERR_FALLBACK_SHOWN', { err: errorBody }); + console.error(error); + }, [errorBody, error]); + + return ( + + {loading ? ( + + + Loading... + + ) : ( + services && + logs.map((log, index) => ( + + {log} + + )) + )} + + + + Something went wrong displaying the logs. + + An error occurred while processing the logs. Please report this issue. + + { + enqueueSnackbar(`Error logs copied to clipboard`, { + variant: 'success' + }); + }} + > + + + + + + + ); +}; From c4684673aabb77dcd1e83f32b1e3bba116782dde Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Sat, 12 Oct 2024 02:05:05 +0530 Subject: [PATCH 09/14] improvements in error handling in systemlogs.tsx --- web-server/src/constants/log-formatter.ts | 31 ++++ web-server/src/content/Service/SystemLogs.tsx | 157 +++++++++++++----- .../Service/SystemLogsErrorFallback.tsx | 105 ------------ web-server/src/utils/logFormatter.ts | 41 ++--- 4 files changed, 161 insertions(+), 173 deletions(-) create mode 100644 web-server/src/constants/log-formatter.ts delete mode 100644 web-server/src/content/Service/SystemLogsErrorFallback.tsx diff --git a/web-server/src/constants/log-formatter.ts b/web-server/src/constants/log-formatter.ts new file mode 100644 index 00000000..885343bb --- /dev/null +++ b/web-server/src/constants/log-formatter.ts @@ -0,0 +1,31 @@ +export interface ParsedLog { + timestamp: string; + logLevel: string; + message: string; + role?: string; + ip?: string; +} + +export const generalLogRegex = + /^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; +export const httpLogRegex = + /^(\S+) (\S+) (\S+) \[([^\]]+)\] "([^"]*)" (\d+) (\d+) "([^"]*)" "([^"]*)"$/; +export const redisLogRegex = + /^(?\d+:[XCMS]) (?\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2}\.\d{3}) (?[\.\-\#\*]) (?.*)$/; +export const postgresLogRegex = + /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \w+) \[(\d+)\] (\w+):(.+)(?:\n(?:\s+.*)?)*$/m; +export const postgresMultiLineLogRegex = + /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[\d+\] (?[A-Z]+):\s*(?(.|\n)+?)$/; +export const dataSyncLogRegex = /\[(\w+)\]\s(.+?)\sfor\s(\w+)\s(.+)/; +export const validLogLevels = new Set([ + 'DEBUG', + 'INFO', + 'NOTICE', + 'WARNING', + 'ERROR', + 'LOG', + 'FATAL', + 'PANIC', + 'STATEMENT', + 'DETAIL' +]); diff --git a/web-server/src/content/Service/SystemLogs.tsx b/web-server/src/content/Service/SystemLogs.tsx index 994cd541..1ccd209f 100644 --- a/web-server/src/content/Service/SystemLogs.tsx +++ b/web-server/src/content/Service/SystemLogs.tsx @@ -1,14 +1,17 @@ -import { CircularProgress, useTheme } from '@mui/material'; -import { useEffect, useRef, useMemo, useCallback, useState } from 'react'; +import { CopyAll } from '@mui/icons-material'; +import { Button, CircularProgress, Typography, useTheme } from '@mui/material'; +import { useSnackbar } from 'notistack'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import CopyToClipboard from 'react-copy-to-clipboard'; import { FlexBox } from '@/components/FlexBox'; import { Line } from '@/components/Text'; +import { track } from '@/constants/events'; +import { ParsedLog } from '@/constants/log-formatter'; import { ServiceNames } from '@/constants/service'; import { useSelector } from '@/store'; import { parseLogLine } from '@/utils/logFormatter'; -import { SystemLogsErrorFallback } from './SystemLogsErrorFallback'; - export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { const services = useSelector((state) => state.service.services); const loading = useSelector((state) => state.service.loading); @@ -19,6 +22,14 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { }, [serviceName, services]); const containerRef = useRef(null); + const { enqueueSnackbar } = useSnackbar(); + const errorBody = useMemo( + () => ({ + message: error?.message?.replace('\\n', '\n'), + stack: error?.stack?.replace('\\n', '\n') + }), + [error?.message, error?.stack] + ); const getLevelColor = useCallback( (level: string) => { @@ -52,6 +63,100 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { } }, [logs]); + useEffect(() => { + if (error) { + track('ERR_FALLBACK_SHOWN', { err: errorBody }); + console.error(error); + } + }, [errorBody, error]); + + const SystemLogErrorMessage = useCallback( + ({ errorBody }: { errorBody: any }) => { + return ( + + + + Something went wrong displaying the logs. + + An error occurred while processing the logs. Please report this + issue. + + { + enqueueSnackbar(`Error logs copied to clipboard`, { + variant: 'success' + }); + }} + > + + + + + ); + }, + [enqueueSnackbar] + ); + + const PlainLog = useCallback( + ({ log, index }: { log: string; index: number }) => { + return ( + + {log} + + ); + }, + [] + ); + + const FormattedLog = useCallback( + ({ log, index }: { log: ParsedLog; index: number }) => { + const { timestamp, ip, logLevel, message } = log; + return ( + + + {timestamp} + {' '} + {ip && ( + + {ip}{' '} + + )} + + [{logLevel}] + {' '} + {message} + + ); + }, + [getLevelColor] + ); + return ( {loading ? ( @@ -65,45 +170,23 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { const parsedLog = parseLogLine(log); if (!parsedLog) { - return ( - - {log} - - ); + return ; } - const { timestamp, ip, logLevel, message } = parsedLog; - return ( - - - {timestamp} - {' '} - {ip && ( - - {ip}{' '} - - )} - - [{logLevel}] - {' '} - {message} - - ); + return ; } catch (error: any) { setError(error); + return null; } }) ) : ( - + services && ( + <> + {logs.map((log, index) => ( + + ))} + + + ) )} diff --git a/web-server/src/content/Service/SystemLogsErrorFallback.tsx b/web-server/src/content/Service/SystemLogsErrorFallback.tsx deleted file mode 100644 index 8648d0c5..00000000 --- a/web-server/src/content/Service/SystemLogsErrorFallback.tsx +++ /dev/null @@ -1,105 +0,0 @@ -import { CopyAll } from '@mui/icons-material'; -import { Button, Typography } from '@mui/material'; -import CircularProgress from '@mui/material/CircularProgress'; -import { useSnackbar } from 'notistack'; -import { useEffect, useRef, useMemo } from 'react'; -import CopyToClipboard from 'react-copy-to-clipboard'; - -import { FlexBox } from '@/components/FlexBox'; -import { Line } from '@/components/Text'; -import { track } from '@/constants/events'; -import { ServiceNames } from '@/constants/service'; -import { useSelector } from '@/store'; - -export const SystemLogsErrorFallback = ({ - error, - serviceName -}: { - error: Error; - serviceName: ServiceNames; -}) => { - const services = useSelector((state) => state.service.services); - const loading = useSelector((state) => state.service.loading); - const logs = useMemo(() => { - return services[serviceName].logs || []; - }, [serviceName, services]); - - const containerRef = useRef(null); - const { enqueueSnackbar } = useSnackbar(); - const errorBody = useMemo( - () => ({ - message: error?.message?.replace('\\n', '\n'), - stack: error?.stack?.replace('\\n', '\n') - }), - [error?.message, error?.stack] - ); - - useEffect(() => { - if (containerRef.current) { - containerRef.current.scrollIntoView({ behavior: 'smooth' }); - } - }, [logs]); - - useEffect(() => { - track('ERR_FALLBACK_SHOWN', { err: errorBody }); - console.error(error); - }, [errorBody, error]); - - return ( - - {loading ? ( - - - Loading... - - ) : ( - services && - logs.map((log, index) => ( - - {log} - - )) - )} - - - - Something went wrong displaying the logs. - - An error occurred while processing the logs. Please report this issue. - - { - enqueueSnackbar(`Error logs copied to clipboard`, { - variant: 'success' - }); - }} - > - - - - - - - ); -}; diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts index c3e6ca27..078768c7 100644 --- a/web-server/src/utils/logFormatter.ts +++ b/web-server/src/utils/logFormatter.ts @@ -1,34 +1,13 @@ -interface ParsedLog { - timestamp: string; - logLevel: string; - message: string; - role?: string; - ip?: string; -} - -const generalLogRegex = - /^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; -const httpLogRegex = - /^(\S+) (\S+) (\S+) \[([^\]]+)\] "([^"]*)" (\d+) (\d+) "([^"]*)" "([^"]*)"$/; -const redisLogRegex = - /^(?\d+:[XCMS]) (?\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2}\.\d{3}) (?[\.\-\#\*]) (?.*)$/; -const postgresLogRegex = - /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \w+) \[(\d+)\] (\w+):(.+)(?:\n(?:\s+.*)?)*$/m; -const postgresMultiLineLogRegex = - /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[\d+\] (?[A-Z]+):\s*(?(.|\n)+?)$/; -const dataSyncLogRegex = /\[(\w+)\]\s(.+?)\sfor\s(\w+)\s(.+)/; -const validLogLevels = new Set([ - 'DEBUG', - 'INFO', - 'NOTICE', - 'WARNING', - 'ERROR', - 'LOG', - 'FATAL', - 'PANIC', - 'STATEMENT', - 'DETAIL' -]); +import { + ParsedLog, + generalLogRegex, + httpLogRegex, + redisLogRegex, + postgresLogRegex, + postgresMultiLineLogRegex, + dataSyncLogRegex, + validLogLevels +} from '../constants/log-formatter'; export const parseLogLine = (rawLogLine: string): ParsedLog | null => { const generalLogMatch = rawLogLine.match(generalLogRegex); From 26dabfa2634ce1a2896b30a921ba3e7259800c23 Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Tue, 15 Oct 2024 00:57:22 +0530 Subject: [PATCH 10/14] Implementation of ErrorBoundary --- .../Service/SystemLog/FormattedLog.tsx | 63 ++++++ .../components/Service/SystemLog/PlainLog.tsx | 14 ++ .../SystemLog/SystemLogErrorMessage.tsx | 47 +++++ .../SystemLog/SystemLogsErrorFllback.tsx | 60 ++++++ web-server/src/content/Service/SystemLogs.tsx | 197 +++--------------- web-server/src/hooks/useSystemLogs.tsx | 22 ++ 6 files changed, 233 insertions(+), 170 deletions(-) create mode 100644 web-server/src/components/Service/SystemLog/FormattedLog.tsx create mode 100644 web-server/src/components/Service/SystemLog/PlainLog.tsx create mode 100644 web-server/src/components/Service/SystemLog/SystemLogErrorMessage.tsx create mode 100644 web-server/src/components/Service/SystemLog/SystemLogsErrorFllback.tsx create mode 100644 web-server/src/hooks/useSystemLogs.tsx diff --git a/web-server/src/components/Service/SystemLog/FormattedLog.tsx b/web-server/src/components/Service/SystemLog/FormattedLog.tsx new file mode 100644 index 00000000..d85ec920 --- /dev/null +++ b/web-server/src/components/Service/SystemLog/FormattedLog.tsx @@ -0,0 +1,63 @@ +import { useTheme } from '@mui/material'; +import { useCallback } from 'react'; + +import { Line } from '@/components/Text'; +import { ParsedLog } from '@/constants/log-formatter'; + +export const FormattedLog = ({ + log, + index +}: { + log: ParsedLog; + index: number; +}) => { + const theme = useTheme(); + const getLevelColor = useCallback( + (level: string) => { + const colors: { [key: string]: string } = { + INFO: theme.colors.success.main, + MAIN_INFO: theme.colors.success.main, + CHILD_INFO: theme.colors.success.main, + SENTINEL_INFO: theme.colors.success.main, + WARN: theme.colors.warning.main, + WARNING: theme.colors.warning.main, + NOTICE: theme.colors.warning.dark, + ERROR: theme.colors.error.main, + FATAL: theme.colors.error.main, + PANIC: theme.colors.error.main, + DEBUG: theme.colors.info.light, + MAIN_SYSTEM: theme.colors.primary.main, + CHILD_SYSTEM: theme.colors.primary.main, + SENTINEL_SYSTEM: theme.colors.primary.main, + LOG: theme.colors.info.main, + CRITICAL: theme.colors.error.main + }; + + return colors[level.toUpperCase()] || theme.colors.info.main; + }, + [theme] + ); + + const { timestamp, ip, logLevel, message } = log; + return ( + + + {timestamp} + {' '} + {ip && ( + + {ip}{' '} + + )} + + [{logLevel}] + {' '} + {message} + + ); +}; diff --git a/web-server/src/components/Service/SystemLog/PlainLog.tsx b/web-server/src/components/Service/SystemLog/PlainLog.tsx new file mode 100644 index 00000000..ac96d36c --- /dev/null +++ b/web-server/src/components/Service/SystemLog/PlainLog.tsx @@ -0,0 +1,14 @@ +import { Line } from '@/components/Text'; + +export const PlainLog = ({ log, index }: { log: string; index: number }) => { + return ( + + {log} + + ); +}; diff --git a/web-server/src/components/Service/SystemLog/SystemLogErrorMessage.tsx b/web-server/src/components/Service/SystemLog/SystemLogErrorMessage.tsx new file mode 100644 index 00000000..b009d240 --- /dev/null +++ b/web-server/src/components/Service/SystemLog/SystemLogErrorMessage.tsx @@ -0,0 +1,47 @@ +import { CopyAll } from '@mui/icons-material'; +import { Button, Typography } from '@mui/material'; +import { useSnackbar } from 'notistack'; +import CopyToClipboard from 'react-copy-to-clipboard'; + +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; + +export const SystemLogErrorMessage = ({ errorBody }: { errorBody: any }) => { + const { enqueueSnackbar } = useSnackbar(); + return ( + + + + Something went wrong displaying the logs. + + An error occurred while processing the logs. Please report this issue. + + { + enqueueSnackbar(`Error logs copied to clipboard`, { + variant: 'success' + }); + }} + > + + + + + ); +}; diff --git a/web-server/src/components/Service/SystemLog/SystemLogsErrorFllback.tsx b/web-server/src/components/Service/SystemLog/SystemLogsErrorFllback.tsx new file mode 100644 index 00000000..823ac72c --- /dev/null +++ b/web-server/src/components/Service/SystemLog/SystemLogsErrorFllback.tsx @@ -0,0 +1,60 @@ +import { CircularProgress } from '@mui/material'; +import { useEffect, useMemo, useRef } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { PlainLog } from '@/components/Service/SystemLog/PlainLog'; +import { SystemLogErrorMessage } from '@/components/Service/SystemLog/SystemLogErrorMessage'; +import { Line } from '@/components/Text'; +import { track } from '@/constants/events'; +import { ServiceNames } from '@/constants/service'; +import { useSystemLogs } from '@/hooks/useSystemLogs'; + +export const SystemLogsErrorFallback = ({ + error, + serviceName +}: { + error: Error; + serviceName: ServiceNames; +}) => { + const { services, loading, logs } = useSystemLogs({ serviceName }); + + const containerRef = useRef(null); + const errorBody = useMemo( + () => ({ + message: error?.message?.replace('\\n', '\n') || '', + stack: error?.stack?.replace('\\n', '\n') || '' + }), + [error] + ); + + useEffect(() => { + if (error) { + track('ERR_FALLBACK_SHOWN', { err: errorBody }); + console.error(error); + } + }, [errorBody, error]); + + useEffect(() => { + if (containerRef.current) { + containerRef.current.scrollIntoView({ behavior: 'smooth' }); + } + }, [logs]); + + return ( + + {loading ? ( + + + Loading... + + ) : ( + services && + logs.map((log, index) => { + return ; + }) + )} + + + + ); +}; diff --git a/web-server/src/content/Service/SystemLogs.tsx b/web-server/src/content/Service/SystemLogs.tsx index 1ccd209f..e125f199 100644 --- a/web-server/src/content/Service/SystemLogs.tsx +++ b/web-server/src/content/Service/SystemLogs.tsx @@ -1,61 +1,20 @@ -import { CopyAll } from '@mui/icons-material'; -import { Button, CircularProgress, Typography, useTheme } from '@mui/material'; -import { useSnackbar } from 'notistack'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import CopyToClipboard from 'react-copy-to-clipboard'; +import { CircularProgress } from '@mui/material'; +import { useEffect, useRef } from 'react'; +import { ErrorBoundary } from 'react-error-boundary'; import { FlexBox } from '@/components/FlexBox'; +import { FormattedLog } from '@/components/Service/SystemLog/FormattedLog'; +import { PlainLog } from '@/components/Service/SystemLog/PlainLog'; +import { SystemLogsErrorFallback } from '@/components/Service/SystemLog/SystemLogsErrorFllback'; import { Line } from '@/components/Text'; -import { track } from '@/constants/events'; -import { ParsedLog } from '@/constants/log-formatter'; import { ServiceNames } from '@/constants/service'; -import { useSelector } from '@/store'; +import { useSystemLogs } from '@/hooks/useSystemLogs'; import { parseLogLine } from '@/utils/logFormatter'; export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { - const services = useSelector((state) => state.service.services); - const loading = useSelector((state) => state.service.loading); - const [error, setError] = useState(null); - const theme = useTheme(); - const logs = useMemo(() => { - return services[serviceName].logs || []; - }, [serviceName, services]); + const { services, loading, logs } = useSystemLogs({ serviceName }); const containerRef = useRef(null); - const { enqueueSnackbar } = useSnackbar(); - const errorBody = useMemo( - () => ({ - message: error?.message?.replace('\\n', '\n'), - stack: error?.stack?.replace('\\n', '\n') - }), - [error?.message, error?.stack] - ); - - const getLevelColor = useCallback( - (level: string) => { - const colors: { [key: string]: string } = { - INFO: theme.colors.success.main, - MAIN_INFO: theme.colors.success.main, - CHILD_INFO: theme.colors.success.main, - SENTINEL_INFO: theme.colors.success.main, - WARN: theme.colors.warning.main, - WARNING: theme.colors.warning.main, - NOTICE: theme.colors.warning.dark, - ERROR: theme.colors.error.main, - FATAL: theme.colors.error.main, - PANIC: theme.colors.error.main, - DEBUG: theme.colors.info.light, - MAIN_SYSTEM: theme.colors.primary.main, - CHILD_SYSTEM: theme.colors.primary.main, - SENTINEL_SYSTEM: theme.colors.primary.main, - LOG: theme.colors.info.main, - CRITICAL: theme.colors.error.main - }; - - return colors[level.toUpperCase()] || theme.colors.info.main; - }, - [theme] - ); useEffect(() => { if (containerRef.current) { @@ -63,132 +22,30 @@ export const SystemLogs = ({ serviceName }: { serviceName: ServiceNames }) => { } }, [logs]); - useEffect(() => { - if (error) { - track('ERR_FALLBACK_SHOWN', { err: errorBody }); - console.error(error); - } - }, [errorBody, error]); - - const SystemLogErrorMessage = useCallback( - ({ errorBody }: { errorBody: any }) => { - return ( - - - - Something went wrong displaying the logs. - - An error occurred while processing the logs. Please report this - issue. - - { - enqueueSnackbar(`Error logs copied to clipboard`, { - variant: 'success' - }); - }} - > - - - - - ); - }, - [enqueueSnackbar] - ); - - const PlainLog = useCallback( - ({ log, index }: { log: string; index: number }) => { - return ( - - {log} - - ); - }, - [] - ); - - const FormattedLog = useCallback( - ({ log, index }: { log: ParsedLog; index: number }) => { - const { timestamp, ip, logLevel, message } = log; - return ( - - - {timestamp} - {' '} - {ip && ( - - {ip}{' '} - - )} - - [{logLevel}] - {' '} - {message} - - ); - }, - [getLevelColor] - ); - return ( - - {loading ? ( - - - Loading... - - ) : !error && services ? ( - logs.map((log, index) => { - try { + ( + + )} + > + + {loading ? ( + + + Loading... + + ) : ( + services && + logs.map((log, index) => { const parsedLog = parseLogLine(log); - if (!parsedLog) { return ; } return ; - } catch (error: any) { - setError(error); - return null; - } - }) - ) : ( - services && ( - <> - {logs.map((log, index) => ( - - ))} - - - ) - )} - - + }) + )} + + + ); }; diff --git a/web-server/src/hooks/useSystemLogs.tsx b/web-server/src/hooks/useSystemLogs.tsx new file mode 100644 index 00000000..b795e97d --- /dev/null +++ b/web-server/src/hooks/useSystemLogs.tsx @@ -0,0 +1,22 @@ +import { useMemo } from 'react'; + +import { ServiceNames } from '@/constants/service'; +import { useSelector } from '@/store'; +export const useSystemLogs = ({ + serviceName +}: { + serviceName: ServiceNames; +}) => { + const services = useSelector((state) => state.service.services); + const loading = useSelector((state) => state.service.loading); + + const logs = useMemo(() => { + return services[serviceName]?.logs || []; + }, [serviceName, services]); + + return { + services, + loading, + logs + }; +}; From c2c65ecd386059be3d41fb05ad5e6d4dbc20f49e Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Wed, 16 Oct 2024 01:53:21 +0530 Subject: [PATCH 11/14] refactore : Postgres log parsing logic and enum declaration. --- web-server/src/constants/log-formatter.ts | 14 ----- web-server/src/types/resources.ts | 13 ++++ .../src/utils/__tests__/logFormatter.test.ts | 44 ++++++------- web-server/src/utils/logFormatter.ts | 62 ++++++++----------- 4 files changed, 63 insertions(+), 70 deletions(-) diff --git a/web-server/src/constants/log-formatter.ts b/web-server/src/constants/log-formatter.ts index 885343bb..6039a371 100644 --- a/web-server/src/constants/log-formatter.ts +++ b/web-server/src/constants/log-formatter.ts @@ -13,19 +13,5 @@ export const httpLogRegex = export const redisLogRegex = /^(?\d+:[XCMS]) (?\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2}\.\d{3}) (?[\.\-\#\*]) (?.*)$/; export const postgresLogRegex = - /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \w+) \[(\d+)\] (\w+):(.+)(?:\n(?:\s+.*)?)*$/m; -export const postgresMultiLineLogRegex = /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} UTC) \[\d+\] (?[A-Z]+):\s*(?(.|\n)+?)$/; export const dataSyncLogRegex = /\[(\w+)\]\s(.+?)\sfor\s(\w+)\s(.+)/; -export const validLogLevels = new Set([ - 'DEBUG', - 'INFO', - 'NOTICE', - 'WARNING', - 'ERROR', - 'LOG', - 'FATAL', - 'PANIC', - 'STATEMENT', - 'DETAIL' -]); diff --git a/web-server/src/types/resources.ts b/web-server/src/types/resources.ts index 2d740451..56c90512 100644 --- a/web-server/src/types/resources.ts +++ b/web-server/src/types/resources.ts @@ -1039,3 +1039,16 @@ export type DB_OrgRepo = { deployment_type: 'PR_MERGE' | 'WORKFLOW'; repo_workflows: RepoWorkflow[]; }; + +export enum LogLevel { + 'DEBUG' = 'DEBUG', + 'INFO' = 'INFO', + 'NOTICE' = 'NOTICE', + 'WARNING' = 'WARNING', + 'ERROR' = 'ERROR', + 'LOG' = 'LOG', + 'FATAL' = 'FATAL', + 'PANIC' = 'PANIC', + 'STATEMENT' = 'STATEMENT', + 'DETAIL' = 'DETAIL' +} diff --git a/web-server/src/utils/__tests__/logFormatter.test.ts b/web-server/src/utils/__tests__/logFormatter.test.ts index 4b1975f9..84aeab1b 100644 --- a/web-server/src/utils/__tests__/logFormatter.test.ts +++ b/web-server/src/utils/__tests__/logFormatter.test.ts @@ -1,3 +1,5 @@ +import { LogLevel } from '@/types/resources'; + import { parseLogLine } from '../logFormatter'; describe('Should correctly parse logs', () => { @@ -6,7 +8,7 @@ describe('Should correctly parse logs', () => { '[2024-10-08 04:21:42 +0000] [164] [INFO] Booting worker with pid: 164'; const expected = { timestamp: '2024-10-08 04:21:42 +0000', - logLevel: 'INFO', + logLevel: LogLevel.INFO, message: 'Booting worker with pid: 164' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -17,7 +19,7 @@ describe('Should correctly parse logs', () => { '[2024-10-08 04:21:42 +0000] [164] [ERROR] Booting worker with pid: 164'; const expected = { timestamp: '2024-10-08 04:21:42 +0000', - logLevel: 'ERROR', + logLevel: LogLevel.ERROR, message: 'Booting worker with pid: 164' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -28,7 +30,7 @@ describe('Should correctly parse logs', () => { '127.0.0.1 - - [08/Oct/2024:04:25:26 +0000] "GET / HTTP/1.1" 200 26 "-" "axios/0.26.1"'; const expected = { timestamp: '08/Oct/2024:04:25:26 +0000', - logLevel: 'INFO', + logLevel: LogLevel.INFO, message: 'GET / 200 26 "-" "axios/0.26.1"', ip: '127.0.0.1' }; @@ -40,7 +42,7 @@ describe('Should correctly parse logs', () => { '[INFO] Data sync for sync_org_incidents completed successfully'; const expected = { timestamp: '', - logLevel: 'INFO', + logLevel: LogLevel.INFO, message: 'Data sync for sync_org_incidents completed successfully' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -51,7 +53,7 @@ describe('Should correctly parse logs', () => { '127.0.0.1 - - [08/Oct/2024:04:25:16 +0000] "GET / HTTP/1.1" 200 26 "-" "axios/0.26.1"'; const expected = { ip: '127.0.0.1', - logLevel: 'INFO', + logLevel: LogLevel.INFO, message: 'GET / 200 26 "-" "axios/0.26.1"', timestamp: '08/Oct/2024:04:25:16 +0000' }; @@ -63,7 +65,7 @@ describe('Should correctly parse logs', () => { '2024-10-08 04:21:40.288 UTC [49] LOG: starting PostgreSQL 15.8 (Debian 15.8-0+deb12u1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit'; const expected = { timestamp: '2024-10-08 04:21:40.288 UTC', - logLevel: 'LOG', + logLevel: LogLevel.LOG, message: 'starting PostgreSQL 15.8 (Debian 15.8-0+deb12u1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit' }; @@ -75,7 +77,7 @@ describe('Should correctly parse logs', () => { const expected = { role: '51:M', timestamp: '08 Oct 2024 04:21:40.149', - logLevel: 'WARNING', + logLevel: LogLevel.WARNING, message: 'Server initialized' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -85,7 +87,7 @@ describe('Should correctly parse logs', () => { const rawLog = `2024-09-27 12:14:45.280 UTC [56] LOG: checkpoint starting: time`; const expected = { timestamp: '2024-09-27 12:14:45.280 UTC', - logLevel: 'LOG', + logLevel: LogLevel.LOG, message: 'checkpoint starting: time' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -95,7 +97,7 @@ describe('Should correctly parse logs', () => { const rawLog = `2024-09-24 07:25:08.658 UTC [160] ERROR: duplicate key value violates unique constraint "languages_pkey"`; const expected = { timestamp: '2024-09-24 07:25:08.658 UTC', - logLevel: 'ERROR', + logLevel: LogLevel.ERROR, message: 'duplicate key value violates unique constraint "languages_pkey"' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -105,7 +107,7 @@ describe('Should correctly parse logs', () => { const rawLog = `2024-09-24 07:25:07.487 UTC [159] STATEMENT: INSERT INTO "public"."Language" ("createdAt","id","judge0Id","updatedAt","name") VALUES ($1,$2,$3,$4,$5), ($6,$7,$8,$9,$10)`; const expected = { timestamp: '2024-09-24 07:25:07.487 UTC', - logLevel: 'STATEMENT', + logLevel: LogLevel.STATEMENT, message: 'INSERT INTO "public"."Language" ("createdAt","id","judge0Id","updatedAt","name") VALUES ($1,$2,$3,$4,$5), ($6,$7,$8,$9,$10)' }; @@ -116,7 +118,7 @@ describe('Should correctly parse logs', () => { const rawLog = `2024-09-24 07:25:08.658 UTC [160] ERROR: duplicate key value violates unique constraint "languages_pkey"`; const expected = { timestamp: '2024-09-24 07:25:08.658 UTC', - logLevel: 'ERROR', + logLevel: LogLevel.ERROR, message: 'duplicate key value violates unique constraint "languages_pkey"' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -126,7 +128,7 @@ describe('Should correctly parse logs', () => { const rawLog = `2024-10-05 12:00:43.377 UTC [48661] FATAL: unsupported frontend protocol 65363.19778: server supports 3.0 to 3.0`; const expected = { timestamp: '2024-10-05 12:00:43.377 UTC', - logLevel: 'FATAL', + logLevel: LogLevel.FATAL, message: 'unsupported frontend protocol 65363.19778: server supports 3.0 to 3.0' }; @@ -137,7 +139,7 @@ describe('Should correctly parse logs', () => { const rawLog = `2024-09-24 07:25:08.658 UTC [160] DETAIL: Key (id)=(43) already exists.`; const expected = { timestamp: '2024-09-24 07:25:08.658 UTC', - logLevel: 'DETAIL', + logLevel: LogLevel.DETAIL, message: 'Key (id)=(43) already exists.' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -147,7 +149,7 @@ describe('Should correctly parse logs', () => { const rawLog = `2024-09-24 07:25:08.658 UTC [160] DETAIL: Key (id)=(43) already exists.`; const expected = { timestamp: '2024-09-24 07:25:08.658 UTC', - logLevel: 'DETAIL', + logLevel: LogLevel.DETAIL, message: 'Key (id)=(43) already exists.' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -157,7 +159,7 @@ describe('Should correctly parse logs', () => { const rawLog = `2024-09-24 07:25:08.658 UTC [160] STATEMENT: INSERT INTO "public"."languages" ("compile_cmd","source_file","id","name","is_archived","run_cmd") VALUES (null,$1,$2,$3,$4,$5), (null,$6,$7,$8,$9,$10), ($11,$12,$13,$14,$15,$16), (null,$17,$18,$19,$20,$21), ($22,$23,$24,$25,$26,$27), ($28,$29,$30,$31,$32,$33), ($34,$35,$36,$37,$38,$39), ($40,$41,$42,$43,$44,$45), ($46,$47,$48,$49,$50,$51), ($52,$53,$54,$55,$56,$57), ($58,$59,$60,$61,$62,$63), ($64,$65,$66,$67,$68,$69), (null,$70,$71,$72,$73,$74), ($75,$76,$77,$78,$79,$80), (null,$81,$82,$83,$84,$85), (null,$86,$87,$88,$89,$90), ($91,$92,$93,$94,$95,$96), ($97,$98,$99,$100,$101,$102), ($103,$104,$105,$106,$107,$108), ($109,$110,$111,$112,$113,$114), (null,$115,$116,$117,$118,$119), ($120,$121,$122,$123,$124,$125), ($126,$127,$128,$129,$130,$131), (null,$132,$133,$134,$135,$136), ($137,$138,$139,$140,$141,$142), (null,$143,$144,$145,$146,$147), ($148,$149,$150,$151,$152,$153), (null,$154,$155,$156,$157,$158), (null,$159,$160,$161,$162,$163), (null,$164,$165,$166,$167,$168), ($169,$170,$171,$172,$173,$174), ($175,$176,$177,$178,$179,$180), ($181,$182,$183,$184,$185,$186), ($187,$188,$189,$190,$191,$192), ($193,$194,$195,$196,$197,$198), ($199,$200,$201,$202,$203,$204), ($205,$206,$207,$208,$209,$210), (null,$211,$212,$213,$214,$215), ($216,$217,$218,$219,$220,$221), (null,$222,$223,$224,$225,$226), ($227,$228,$229,$230,$231,$232), ($233,$234,$235,$236,$237,$238), (null,$239,$240,$241,$242,$243), (null,$244,$245,$246,$247,$248), (null,$249,$250,$251,$252,$253), ($254,$255,$256,$257,$258,$259), (null,null,$260,$261,$262,null)`; const expected = { timestamp: '2024-09-24 07:25:08.658 UTC', - logLevel: 'STATEMENT', + logLevel: LogLevel.STATEMENT, message: 'INSERT INTO "public"."languages" ("compile_cmd","source_file","id","name","is_archived","run_cmd") VALUES (null,$1,$2,$3,$4,$5), (null,$6,$7,$8,$9,$10), ($11,$12,$13,$14,$15,$16), (null,$17,$18,$19,$20,$21), ($22,$23,$24,$25,$26,$27), ($28,$29,$30,$31,$32,$33), ($34,$35,$36,$37,$38,$39), ($40,$41,$42,$43,$44,$45), ($46,$47,$48,$49,$50,$51), ($52,$53,$54,$55,$56,$57), ($58,$59,$60,$61,$62,$63), ($64,$65,$66,$67,$68,$69), (null,$70,$71,$72,$73,$74), ($75,$76,$77,$78,$79,$80), (null,$81,$82,$83,$84,$85), (null,$86,$87,$88,$89,$90), ($91,$92,$93,$94,$95,$96), ($97,$98,$99,$100,$101,$102), ($103,$104,$105,$106,$107,$108), ($109,$110,$111,$112,$113,$114), (null,$115,$116,$117,$118,$119), ($120,$121,$122,$123,$124,$125), ($126,$127,$128,$129,$130,$131), (null,$132,$133,$134,$135,$136), ($137,$138,$139,$140,$141,$142), (null,$143,$144,$145,$146,$147), ($148,$149,$150,$151,$152,$153), (null,$154,$155,$156,$157,$158), (null,$159,$160,$161,$162,$163), (null,$164,$165,$166,$167,$168), ($169,$170,$171,$172,$173,$174), ($175,$176,$177,$178,$179,$180), ($181,$182,$183,$184,$185,$186), ($187,$188,$189,$190,$191,$192), ($193,$194,$195,$196,$197,$198), ($199,$200,$201,$202,$203,$204), ($205,$206,$207,$208,$209,$210), (null,$211,$212,$213,$214,$215), ($216,$217,$218,$219,$220,$221), (null,$222,$223,$224,$225,$226), ($227,$228,$229,$230,$231,$232), ($233,$234,$235,$236,$237,$238), (null,$239,$240,$241,$242,$243), (null,$244,$245,$246,$247,$248), (null,$249,$250,$251,$252,$253), ($254,$255,$256,$257,$258,$259), (null,null,$260,$261,$262,null)' }; @@ -170,7 +172,7 @@ describe('Should correctly parse logs', () => { const expected = { timestamp: '2021-12-03 07:14:48.594 UTC', - logLevel: 'DETAIL', + logLevel: LogLevel.DETAIL, message: String.raw`parameters: $1 = '1234', $2 = 'njchar hello', $3 = 'aaaa'` }; @@ -182,7 +184,7 @@ describe('Should correctly parse logs', () => { const expected = { role: '51:C', timestamp: '08 Oct 2024 04:21:40.147', - logLevel: 'WARNING', + logLevel: LogLevel.WARNING, message: 'Redis version=7.0.15, bits=64, commit=00000000, modified=0, pid=51, just started' }; @@ -194,7 +196,7 @@ describe('Should correctly parse logs', () => { const expected = { role: '51:M', timestamp: '08 Oct 2024 04:21:40.149', - logLevel: 'WARNING', + logLevel: LogLevel.WARNING, message: "WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect." }; @@ -206,7 +208,7 @@ describe('Should correctly parse logs', () => { const expected = { role: '51:M', timestamp: '08 Oct 2024 04:21:40.150', - logLevel: 'NOTICE', + logLevel: LogLevel.NOTICE, message: 'Ready to accept connections' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -217,7 +219,7 @@ describe('Should correctly parse logs', () => { const expected = { role: '51:M', timestamp: '08 Oct 2024 04:21:40.150', - logLevel: 'NOTICE', + logLevel: LogLevel.NOTICE, message: 'Ready to accept connections' }; expect(parseLogLine(rawLog)).toEqual(expected); @@ -228,7 +230,7 @@ describe('Should correctly parse logs', () => { const expected = { role: '53:C', timestamp: '08 Oct 2024 05:05:20.600', - logLevel: 'INFO', + logLevel: LogLevel.INFO, message: 'INFO Connected to the Redis server at 127.0.0.1:6379' }; expect(parseLogLine(rawLog)).toEqual(expected); diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts index 078768c7..db4d2dfe 100644 --- a/web-server/src/utils/logFormatter.ts +++ b/web-server/src/utils/logFormatter.ts @@ -1,18 +1,18 @@ +import { LogLevel } from '@/types/resources'; + import { ParsedLog, generalLogRegex, httpLogRegex, redisLogRegex, postgresLogRegex, - postgresMultiLineLogRegex, - dataSyncLogRegex, - validLogLevels + dataSyncLogRegex } from '../constants/log-formatter'; export const parseLogLine = (rawLogLine: string): ParsedLog | null => { const generalLogMatch = rawLogLine.match(generalLogRegex); if (generalLogMatch) { - const [, timestamp, , logLevel, message] = generalLogMatch; + const [_fullLog, timestamp, _unused, logLevel, message] = generalLogMatch; return { timestamp, logLevel, @@ -22,12 +22,22 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { const httpLogMatch = rawLogLine.match(httpLogRegex); if (httpLogMatch) { - const [, ip, , , timestamp, request, status, bytes, referer, userAgent] = - httpLogMatch; + const [ + _fullLog, + ip, + _unused, + _unused2, + timestamp, + request, + status, + bytes, + referer, + userAgent + ] = httpLogMatch; const [method, path] = request.split(' '); return { timestamp, - logLevel: 'INFO', // Assuming all HTTP logs are INFO level + logLevel: LogLevel.INFO, // Assuming all HTTP logs are INFO level message: `${method} ${path} ${status} ${bytes} "${referer}" "${userAgent}"`, ip }; @@ -39,19 +49,19 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { let logLevel: string; switch (loglevel) { case '.': - logLevel = 'DEBUG'; + logLevel = LogLevel.DEBUG; break; case '-': - logLevel = 'INFO'; + logLevel = LogLevel.INFO; break; case '*': - logLevel = 'NOTICE'; + logLevel = LogLevel.NOTICE; break; case '#': - logLevel = 'WARNING'; + logLevel = LogLevel.WARNING; break; default: - logLevel = 'INFO'; + logLevel = LogLevel.INFO; } return { role, @@ -61,37 +71,19 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { }; } - const postgresMultiLineLogMatch = rawLogLine.match(postgresMultiLineLogRegex); - if (postgresMultiLineLogMatch) { - const { timestamp, loglevel, message } = postgresMultiLineLogMatch.groups; - const normalizedLogLevel = loglevel.toUpperCase(); - - return { - timestamp: timestamp, - logLevel: validLogLevels.has(normalizedLogLevel) - ? normalizedLogLevel - : 'INFO', - message: message.trim() - }; - } - const postgresLogMatch = rawLogLine.match(postgresLogRegex); if (postgresLogMatch) { - const [, timestamp, , logLevel, message] = postgresLogMatch; - - const normalizedLogLevel = logLevel.toUpperCase(); + const { timestamp, loglevel, message } = postgresLogMatch.groups; return { - timestamp, - logLevel: validLogLevels.has(normalizedLogLevel) - ? normalizedLogLevel - : 'INFO', - message + timestamp: timestamp, + logLevel: loglevel, + message: message.trim() }; } const dataSyncLogMatch = rawLogLine.match(dataSyncLogRegex); if (dataSyncLogMatch) { - const [, logLevel, action, service, message] = dataSyncLogMatch; + const [_fullLog, logLevel, action, service, message] = dataSyncLogMatch; return { timestamp: '', logLevel: logLevel.toUpperCase(), From 55f404ebac0afffb2a88828609b2d8908fbf3755 Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Wed, 16 Oct 2024 15:23:26 +0530 Subject: [PATCH 12/14] fix variables names in Parser --- web-server/src/utils/logFormatter.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts index db4d2dfe..06aa553f 100644 --- a/web-server/src/utils/logFormatter.ts +++ b/web-server/src/utils/logFormatter.ts @@ -12,7 +12,7 @@ import { export const parseLogLine = (rawLogLine: string): ParsedLog | null => { const generalLogMatch = rawLogLine.match(generalLogRegex); if (generalLogMatch) { - const [_fullLog, timestamp, _unused, logLevel, message] = generalLogMatch; + const [_fullLog, timestamp, _pid, logLevel, message] = generalLogMatch; return { timestamp, logLevel, @@ -25,8 +25,8 @@ export const parseLogLine = (rawLogLine: string): ParsedLog | null => { const [ _fullLog, ip, - _unused, - _unused2, + _, + _username, timestamp, request, status, From b26aa4efa7a682880c18bff33ca9495809d73c6a Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Fri, 18 Oct 2024 12:10:41 +0530 Subject: [PATCH 13/14] refactore codes --- .../Service/SystemLog/FormattedLog.tsx | 21 +++++-------------- .../components/Service/SystemLog/PlainLog.tsx | 9 ++------ web-server/src/constants/log-formatter.ts | 8 ------- web-server/src/hooks/useSystemLogs.tsx | 8 +++---- web-server/src/types/resources.ts | 8 +++++++ web-server/src/utils/logFormatter.ts | 2 +- 6 files changed, 20 insertions(+), 36 deletions(-) diff --git a/web-server/src/components/Service/SystemLog/FormattedLog.tsx b/web-server/src/components/Service/SystemLog/FormattedLog.tsx index d85ec920..fe1c970a 100644 --- a/web-server/src/components/Service/SystemLog/FormattedLog.tsx +++ b/web-server/src/components/Service/SystemLog/FormattedLog.tsx @@ -2,15 +2,9 @@ import { useTheme } from '@mui/material'; import { useCallback } from 'react'; import { Line } from '@/components/Text'; -import { ParsedLog } from '@/constants/log-formatter'; +import { ParsedLog } from '@/types/resources'; -export const FormattedLog = ({ - log, - index -}: { - log: ParsedLog; - index: number; -}) => { +export const FormattedLog = ({ log }: { log: ParsedLog; index: number }) => { const theme = useTheme(); const getLevelColor = useCallback( (level: string) => { @@ -40,17 +34,12 @@ export const FormattedLog = ({ const { timestamp, ip, logLevel, message } = log; return ( - - + + {timestamp} {' '} {ip && ( - + {ip}{' '} )} diff --git a/web-server/src/components/Service/SystemLog/PlainLog.tsx b/web-server/src/components/Service/SystemLog/PlainLog.tsx index ac96d36c..484cb6c7 100644 --- a/web-server/src/components/Service/SystemLog/PlainLog.tsx +++ b/web-server/src/components/Service/SystemLog/PlainLog.tsx @@ -1,13 +1,8 @@ import { Line } from '@/components/Text'; -export const PlainLog = ({ log, index }: { log: string; index: number }) => { +export const PlainLog = ({ log }: { log: string; index: number }) => { return ( - + {log} ); diff --git a/web-server/src/constants/log-formatter.ts b/web-server/src/constants/log-formatter.ts index 6039a371..352ce5a2 100644 --- a/web-server/src/constants/log-formatter.ts +++ b/web-server/src/constants/log-formatter.ts @@ -1,11 +1,3 @@ -export interface ParsedLog { - timestamp: string; - logLevel: string; - message: string; - role?: string; - ip?: string; -} - export const generalLogRegex = /^\[(.*?)\] \[(\d+)\] \[(INFO|ERROR|WARN|DEBUG|WARNING|CRITICAL)\] (.+)$/; export const httpLogRegex = diff --git a/web-server/src/hooks/useSystemLogs.tsx b/web-server/src/hooks/useSystemLogs.tsx index b795e97d..5996fa18 100644 --- a/web-server/src/hooks/useSystemLogs.tsx +++ b/web-server/src/hooks/useSystemLogs.tsx @@ -9,10 +9,10 @@ export const useSystemLogs = ({ }) => { const services = useSelector((state) => state.service.services); const loading = useSelector((state) => state.service.loading); - - const logs = useMemo(() => { - return services[serviceName]?.logs || []; - }, [serviceName, services]); + const logs = useMemo( + () => services[serviceName]?.logs || [], + [serviceName, services] + ); return { services, diff --git a/web-server/src/types/resources.ts b/web-server/src/types/resources.ts index 56c90512..6d56d2d0 100644 --- a/web-server/src/types/resources.ts +++ b/web-server/src/types/resources.ts @@ -1052,3 +1052,11 @@ export enum LogLevel { 'STATEMENT' = 'STATEMENT', 'DETAIL' = 'DETAIL' } + +export interface ParsedLog { + timestamp: string; + logLevel: string; + message: string; + role?: string; + ip?: string; +} diff --git a/web-server/src/utils/logFormatter.ts b/web-server/src/utils/logFormatter.ts index 06aa553f..40774b8e 100644 --- a/web-server/src/utils/logFormatter.ts +++ b/web-server/src/utils/logFormatter.ts @@ -1,7 +1,7 @@ import { LogLevel } from '@/types/resources'; +import { ParsedLog } from '@/types/resources'; import { - ParsedLog, generalLogRegex, httpLogRegex, redisLogRegex, From 9ec9da3d5bf6ab84eb7771a593e8c84bd550abc5 Mon Sep 17 00:00:00 2001 From: vishalmishraa Date: Fri, 18 Oct 2024 13:14:19 +0530 Subject: [PATCH 14/14] Updated margin and font styles for formatted and plain logs --- web-server/src/components/Service/SystemLog/FormattedLog.tsx | 2 +- web-server/src/components/Service/SystemLog/PlainLog.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web-server/src/components/Service/SystemLog/FormattedLog.tsx b/web-server/src/components/Service/SystemLog/FormattedLog.tsx index fe1c970a..d327e969 100644 --- a/web-server/src/components/Service/SystemLog/FormattedLog.tsx +++ b/web-server/src/components/Service/SystemLog/FormattedLog.tsx @@ -34,7 +34,7 @@ export const FormattedLog = ({ log }: { log: ParsedLog; index: number }) => { const { timestamp, ip, logLevel, message } = log; return ( - + {timestamp} {' '} diff --git a/web-server/src/components/Service/SystemLog/PlainLog.tsx b/web-server/src/components/Service/SystemLog/PlainLog.tsx index 484cb6c7..c777f62a 100644 --- a/web-server/src/components/Service/SystemLog/PlainLog.tsx +++ b/web-server/src/components/Service/SystemLog/PlainLog.tsx @@ -2,7 +2,7 @@ import { Line } from '@/components/Text'; export const PlainLog = ({ log }: { log: string; index: number }) => { return ( - + {log} );