wip

packages/db/src/clickhouse/migration.ts

@@ -247,32 +247,114 @@ export function moveDataBetweenTables({
   let currentDate = endDate;
   const interval = batch.interval || 'day';
 
-  while (currentDate > startDate) {
+  // Helper function to get the start of the week (Monday) for a given date
+  const getWeekStart = (date: Date): Date => {
+    const d = new Date(date);
+    const day = d.getDay();
+    const diff = d.getDate() - day + (day === 0 ? -6 : 1); // Adjust to Monday
+    d.setDate(diff);
+    d.setHours(0, 0, 0, 0); // Normalize to start of day
+    return d;
+  };
+
+  // Helper function to compare dates based on interval
+  const shouldContinue = (
+    current: Date,
+    start: Date,
+    intervalType: string,
+  ): boolean => {
+    if (intervalType === 'month') {
+      // For months, compare by year and month
+      // Continue if current month is >= start month
+      const currentYear = current.getFullYear();
+      const currentMonth = current.getMonth();
+      const startYear = start.getFullYear();
+      const startMonth = start.getMonth();
+      return (
+        currentYear > startYear ||
+        (currentYear === startYear && currentMonth >= startMonth)
+      );
+    }
+    if (intervalType === 'week') {
+      // For weeks, compare by week start dates
+      const currentWeekStart = getWeekStart(current);
+      const startWeekStart = getWeekStart(start);
+      return currentWeekStart >= startWeekStart;
+    }
+    return current > start;
+  };
+
+  while (shouldContinue(currentDate, startDate, interval)) {
     const previousDate = new Date(currentDate);
 
     switch (interval) {
       case 'month':
         previousDate.setMonth(previousDate.getMonth() - 1);
-        break;
-      case 'week':
-        previousDate.setDate(previousDate.getDate() - 7);
-        // Ensure we don't go below startDate
-        if (previousDate < startDate) {
-          previousDate.setTime(startDate.getTime());
+        // If we've gone below startDate's month, adjust to start of startDate's month
+        // This ensures we generate SQL for the month containing startDate
+        if (
+          previousDate.getFullYear() < startDate.getFullYear() ||
+          (previousDate.getFullYear() === startDate.getFullYear() &&
+            previousDate.getMonth() < startDate.getMonth())
+        ) {
+          previousDate.setFullYear(startDate.getFullYear());
+          previousDate.setMonth(startDate.getMonth());
+          previousDate.setDate(1);
         }
         break;
+      case 'week': {
+        previousDate.setDate(previousDate.getDate() - 7);
+        // If we've gone below startDate's week, adjust to start of startDate's week
+        const startWeekStart = getWeekStart(startDate);
+        const prevWeekStart = getWeekStart(previousDate);
+        if (prevWeekStart < startWeekStart) {
+          previousDate.setTime(startWeekStart.getTime());
+        }
+        break;
+      }
       // day
       default:
         previousDate.setDate(previousDate.getDate() - 1);
         break;
     }
 
+    // For monthly/weekly intervals with transform, we need to use the start of the next period for the upper bound
+    let upperBoundDate = currentDate;
+    if (interval === 'month' && batch.transform) {
+      const nextMonth = new Date(currentDate);
+      nextMonth.setMonth(nextMonth.getMonth() + 1);
+      nextMonth.setDate(1);
+      upperBoundDate = nextMonth;
+    } else if (interval === 'week' && batch.transform) {
+      const nextWeek = new Date(currentDate);
+      nextWeek.setDate(nextWeek.getDate() + 7);
+      const nextWeekStart = getWeekStart(nextWeek);
+      upperBoundDate = nextWeekStart;
+    }
+
     const sql = `INSERT INTO ${to} 
       SELECT * FROM ${from} 
       WHERE ${batch.column} > '${batch.transform ? batch.transform(previousDate) : formatClickhouseDate(previousDate, true)}' 
-      AND ${batch.column} <= '${batch.transform ? batch.transform(currentDate) : formatClickhouseDate(currentDate, true)}'`;
+      AND ${batch.column} <= '${batch.transform ? batch.transform(upperBoundDate) : formatClickhouseDate(upperBoundDate, true)}'`;
     sqls.push(sql);
 
+    // For monthly/weekly intervals, stop if we've reached the start period
+    if (interval === 'month') {
+      const prevYear = previousDate.getFullYear();
+      const prevMonth = previousDate.getMonth();
+      const startYear = startDate.getFullYear();
+      const startMonth = startDate.getMonth();
+      if (prevYear === startYear && prevMonth === startMonth) {
+        break;
+      }
+    } else if (interval === 'week') {
+      const prevWeekStart = getWeekStart(previousDate);
+      const startWeekStart = getWeekStart(startDate);
+      if (prevWeekStart.getTime() === startWeekStart.getTime()) {
+        break;
+      }
+    }
+
     currentDate = previousDate;
   }