fix: overall perf improvements

* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert

apps/worker/src/boot-cron.ts

@@ -44,43 +44,30 @@ export async function bootCron() {
     });
   }
 
-  // Add repeatable jobs
-  for (const job of jobs) {
-    await cronQueue.add(
-      job.name,
-      {
-        type: job.type,
-        payload: undefined,
-      },
-      {
-        jobId: job.type,
-        repeat:
-          typeof job.pattern === 'number'
-            ? {
-                every: job.pattern,
-              }
-            : {
-                pattern: job.pattern,
-              },
-      },
-    );
+  logger.info('Updating cron jobs');
+
+  const jobSchedulers = await cronQueue.getJobSchedulers();
+  for (const jobScheduler of jobSchedulers) {
+    await cronQueue.removeJobScheduler(jobScheduler.key);
   }
 
-  // Remove outdated repeatable jobs
-  const repeatableJobs = await cronQueue.getRepeatableJobs();
-  for (const repeatableJob of repeatableJobs) {
-    const match = jobs.find(
-      (job) => `${job.name}:${job.type}:::${job.pattern}` === repeatableJob.key,
+  // Add repeatable jobs
+  for (const job of jobs) {
+    await cronQueue.upsertJobScheduler(
+      job.type,
+      typeof job.pattern === 'number'
+        ? {
+            every: job.pattern,
+          }
+        : {
+            pattern: job.pattern,
+          },
+      {
+        data: {
+          type: job.type,
+          payload: undefined,
+        },
+      },
     );
-    if (match) {
-      logger.info('Repeatable job exists', {
-        key: repeatableJob.key,
-      });
-    } else {
-      logger.info('Removing repeatable job', {
-        key: repeatableJob.key,
-      });
-      cronQueue.removeRepeatableByKey(repeatableJob.key);
-    }
   }
 }