fix: overall perf improvements

* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert

packages/logger/index.ts

@@ -6,9 +6,12 @@ export { winston };
 export type ILogger = winston.Logger;
 
 const logLevel = process.env.LOG_LEVEL ?? 'info';
+const silent = process.env.LOG_SILENT === 'true';
 
 export function createLogger({ name }: { name: string }): ILogger {
-  const service = `${name}-${process.env.NODE_ENV ?? 'dev'}`;
+  const service = [process.env.LOG_PREFIX, name, process.env.NODE_ENV ?? 'dev']
+    .filter(Boolean)
+    .join('-');
 
   const prettyError = (error: Error) => ({
     ...error,
@@ -64,13 +67,9 @@ export function createLogger({ name }: { name: string }): ILogger {
     return Object.assign({}, info, redactObject(info));
   });
 
-  const format = winston.format.combine(
-    errorFormatter(),
-    redactSensitiveInfo(),
-    winston.format.json(),
-  );
+  const transports: winston.transport[] = [];
+  let format: winston.Logform.Format;
 
-  const transports: winston.transport[] = [new winston.transports.Console()];
   if (process.env.HYPERDX_API_KEY) {
     transports.push(
       HyperDX.getWinstonTransport(logLevel, {
@@ -78,6 +77,24 @@ export function createLogger({ name }: { name: string }): ILogger {
         service,
       }),
     );
+    format = winston.format.combine(
+      errorFormatter(),
+      redactSensitiveInfo(),
+      winston.format.json(),
+    );
+  } else {
+    transports.push(new winston.transports.Console());
+    format = winston.format.combine(
+      errorFormatter(),
+      redactSensitiveInfo(),
+      winston.format.colorize(),
+      winston.format.printf((info) => {
+        const { level, message, service, ...meta } = info;
+        const metaStr =
+          Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : '';
+        return `${level} ${message}${metaStr}`;
+      }),
+    );
   }
 
   const logger = winston.createLogger({
@@ -85,7 +102,7 @@ export function createLogger({ name }: { name: string }): ILogger {
     level: logLevel,
     format,
     transports,
-    silent: process.env.NODE_ENV === 'test',
+    silent,
     // Add ISO levels of logging from PINO
     levels: Object.assign(
       { fatal: 0, warn: 4, trace: 7 },