fix: salt issues

apps/worker/src/boot-cron.ts

@@ -3,6 +3,30 @@ import { cronQueue } from '@openpanel/queue';
 
 import { logger } from './utils/logger';
 
+async function removeConflictingJobs(schedulerKey: string) {
+  // Remove any existing jobs that might conflict with the scheduler
+  // BullMQ scheduler jobs have IDs like "repeat:<key>:<timestamp>"
+  const jobStates = ['delayed', 'waiting', 'completed', 'failed'] as const;
+
+  for (const state of jobStates) {
+    try {
+      const jobs = await cronQueue.getJobs([state]);
+      for (const job of jobs) {
+        // Check if this job was created by the scheduler we're about to upsert
+        if (job.id?.startsWith(`repeat:${schedulerKey}:`)) {
+          await job.remove();
+          logger.info('Removed conflicting scheduler job', {
+            jobId: job.id,
+            schedulerKey,
+          });
+        }
+      }
+    } catch (error) {
+      // Ignore errors during cleanup
+    }
+  }
+}
+
 export async function bootCron() {
   const jobs: {
     name: string;
@@ -56,28 +80,98 @@ export async function bootCron() {
 
   logger.info('Updating cron jobs');
 
-  const jobSchedulers = await cronQueue.getJobSchedulers();
-  for (const jobScheduler of jobSchedulers) {
-    await cronQueue.removeJobScheduler(jobScheduler.key);
+  const jobsToKeep = new Set(jobs.map((job) => job.type));
+
+  const currentJobSchedulers = await cronQueue
+    .getJobSchedulers()
+    .catch((error) => {
+      logger.error('Error getting job schedulers', {
+        error,
+      });
+      return [];
+    });
+  for (const jobScheduler of currentJobSchedulers) {
+    if (!jobsToKeep.has(jobScheduler.key as CronQueueType)) {
+      await cronQueue.removeJobScheduler(jobScheduler.key).catch((error) => {
+        logger.error('Error removing job scheduler', {
+          error,
+          jobScheduler: jobScheduler.key,
+        });
+      });
+    }
   }
 
-  // Add repeatable jobs
   for (const job of jobs) {
-    await cronQueue.upsertJobScheduler(
-      job.type,
-      typeof job.pattern === 'number'
-        ? {
-            every: job.pattern,
-          }
-        : {
-            pattern: job.pattern,
+    try {
+      await cronQueue.upsertJobScheduler(
+        job.type,
+        typeof job.pattern === 'number'
+          ? {
+              every: job.pattern,
+            }
+          : {
+              pattern: job.pattern,
+            },
+        {
+          data: {
+            type: job.type,
+            payload: undefined,
           },
-      {
-        data: {
-          type: job.type,
-          payload: undefined,
         },
-      },
-    );
+      );
+    } catch (error) {
+      // If upsert fails due to conflicting job, try to clean up and retry
+      const isConflictError =
+        error instanceof Error &&
+        error.message.includes('job ID already exists');
+
+      if (isConflictError) {
+        logger.warn('Job scheduler conflict detected, attempting cleanup', {
+          job: job.type,
+        });
+
+        await removeConflictingJobs(job.type);
+
+        // Also try removing the scheduler itself to start fresh
+        try {
+          await cronQueue.removeJobScheduler(job.type);
+        } catch {
+          // Ignore - scheduler might not exist
+        }
+
+        // Retry the upsert
+        try {
+          await cronQueue.upsertJobScheduler(
+            job.type,
+            typeof job.pattern === 'number'
+              ? {
+                  every: job.pattern,
+                }
+              : {
+                  pattern: job.pattern,
+                },
+            {
+              data: {
+                type: job.type,
+                payload: undefined,
+              },
+            },
+          );
+          logger.info('Job scheduler created after cleanup', {
+            job: job.type,
+          });
+        } catch (retryError) {
+          logger.error('Error upserting job scheduler after cleanup', {
+            error: retryError,
+            job: job.type,
+          });
+        }
+      } else {
+        logger.error('Error upserting job scheduler', {
+          error,
+          job: job.type,
+        });
+      }
+    }
   }
 }