Improve logging (#4921)

* improve logging

* Update logger.ts

---------

Co-authored-by: Henry Heng <henryheng@flowiseai.com>

Author: matekungl-byborg

packages/server/src/utils/logger.ts

@@ -20,6 +20,8 @@ let gcsServerStream: any
 let gcsErrorStream: any
 let gcsServerReqStream: any
 
+let requestLogger: any
+
 if (process.env.STORAGE_TYPE === 's3') {
     const accessKeyId = process.env.S3_STORAGE_ACCESS_KEY_ID
     const secretAccessKey = process.env.S3_STORAGE_SECRET_ACCESS_KEY
@@ -165,6 +167,32 @@ const logger = createLogger({
     ]
 })
 
+requestLogger = createLogger({
+    format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json(), errors({ stack: true })),
+    defaultMeta: {
+        package: 'server'
+    },
+    transports: [
+        ...(process.env.DEBUG && process.env.DEBUG === 'true' ? [new transports.Console()] : []),
+        ...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
+            ? [
+                  new transports.File({
+                      filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
+                      level: config.logging.express.level ?? 'debug'
+                  })
+              ]
+            : []),
+        ...(process.env.STORAGE_TYPE === 's3'
+            ? [
+                  new transports.Stream({
+                      stream: s3ServerReqStream
+                  })
+              ]
+            : []),
+        ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsServerReqStream] : [])
+    ]
+})
+
 export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void {
     const unwantedLogURLs = ['/api/v1/node-icon/', '/api/v1/components-credentials-icon/', '/api/v1/ping']
 
@@ -175,39 +203,17 @@ export function expressRequestLogger(req: Request, res: Response, next: NextFunc
             sanitizedBody.password = '********'
         }
 
-        const fileLogger = createLogger({
-            format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json(), errors({ stack: true })),
-            defaultMeta: {
-                package: 'server',
-                request: {
-                    method: req.method,
-                    url: req.url,
-                    body: sanitizedBody, // Use sanitized body instead of raw body
-                    query: req.query,
-                    params: req.params,
-                    headers: req.headers
-                }
-            },
-            transports: [
-                ...(process.env.DEBUG && process.env.DEBUG === 'true' ? [new transports.Console()] : []),
-                ...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
-                    ? [
-                          new transports.File({
-                              filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
-                              level: config.logging.express.level ?? 'debug'
-                          })
-                      ]
-                    : []),
-                ...(process.env.STORAGE_TYPE === 's3'
-                    ? [
-                          new transports.Stream({
-                              stream: s3ServerReqStream
-                          })
-                      ]
-                    : []),
-                ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsServerReqStream] : [])
-            ]
-        })
+        // Use the shared requestLogger with request-specific metadata
+        const requestMetadata = {
+            request: {
+                method: req.method,
+                url: req.url,
+                body: sanitizedBody, // Use sanitized body instead of raw body
+                query: req.query,
+                params: req.params,
+                headers: req.headers
+            }
+        }
 
         const getRequestEmoji = (method: string) => {
             const requetsEmojis: Record<string, string> = {
@@ -222,10 +228,10 @@ export function expressRequestLogger(req: Request, res: Response, next: NextFunc
         }
 
         if (req.method !== 'GET') {
-            fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
+            requestLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`, requestMetadata)
             logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
         } else {
-            fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
+            requestLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`, requestMetadata)
         }
     }