formulas

packages/trpc/src/routers/chart.helpers.ts

@@ -29,6 +29,8 @@ import {
 import type {
   FinalChart,
   IChartEvent,
+  IChartEventItem,
+  IChartFormula,
   IChartInput,
   IChartInputWithDates,
   IGetChartDataInput,
@@ -59,10 +61,14 @@ export function withFormula(
   const hasBreakdowns = series.some(
     (serie) =>
       serie.name.length > 0 &&
-      !events.some(
-        (event) =>
-          serie.name[0] === event.name || serie.name[0] === event.displayName,
-      ),
+      !events.some((event) => {
+        if (event.type === 'event') {
+          return (
+            serie.name[0] === event.name || serie.name[0] === event.displayName
+          );
+        }
+        return false;
+      }),
   );
 
   const seriesByBreakdown = new Map<string, typeof series>();
@@ -128,6 +134,11 @@ export function withFormula(
         }
 
         // Find the series for this event in this breakdown group
+        // Only events (not formulas) are used in the old formula system
+        if (event.type !== 'event') {
+          scope[readableId] = 0;
+          return;
+        }
         const eventId = event.id ?? event.name;
         const matchingSerie = seriesByEvent.get(eventId);
 
@@ -212,17 +223,27 @@ export async function getFunnelData({
     };
   }
 
-  const funnels = payload.events.map((event) => {
-    const { sb, getWhere } = createSqlBuilder();
-    sb.where = getEventFiltersWhereClause(event.filters);
-    sb.where.name = `name = ${sqlstring.escape(event.name)}`;
-    return getWhere().replace('WHERE ', '');
-  });
+  const funnels = payload.events
+    .filter(
+      (event): event is IChartEventItem & { type: 'event' } =>
+        event.type === 'event',
+    )
+    .map((event) => {
+      const { sb, getWhere } = createSqlBuilder();
+      sb.where = getEventFiltersWhereClause(event.filters);
+      sb.where.name = `name = ${sqlstring.escape(event.name)}`;
+      return getWhere().replace('WHERE ', '');
+    });
 
   const commonWhere = `project_id = ${sqlstring.escape(projectId)} AND 
     created_at >= '${formatClickhouseDate(startDate)}' AND 
     created_at <= '${formatClickhouseDate(endDate)}'`;
 
+  // Filter to only events (funnels don't support formulas)
+  const eventNames = payload.events
+    .filter((e): e is IChartEventItem & { type: 'event' } => e.type === 'event')
+    .map((event) => sqlstring.escape(event.name));
+
   const innerSql = `SELECT
     ${funnelGroup[0]} AS ${funnelGroup[1]},
     windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
@@ -230,7 +251,7 @@ export async function getFunnelData({
   ${funnelGroup[0] === 'session_id' ? '' : `LEFT JOIN (SELECT profile_id, id FROM sessions WHERE ${commonWhere}) AS s ON s.id = e.session_id`}
   WHERE 
     ${commonWhere} AND
-    name IN (${payload.events.map((event) => sqlstring.escape(event.name)).join(', ')})
+    name IN (${eventNames.join(', ')})
   GROUP BY ${funnelGroup[0]}`;
 
   const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`;
@@ -243,7 +264,12 @@ export async function getFunnelData({
   const steps = reverse(filledFunnelRes).reduce(
     (acc, item, index, list) => {
       const prev = list[index - 1] ?? { count: totalSessions };
-      const event = payload.events[item.level - 1]!;
+      const eventItem = payload.events[item.level - 1]!;
+      // Funnels only work with events, not formulas
+      if (eventItem.type !== 'event') {
+        return acc;
+      }
+      const event = eventItem;
       return [
         ...acc,
         {
@@ -307,30 +333,249 @@ export async function getChartSerie(
   });
 }
 
+// Normalize events to ensure they have a type field
+function normalizeEventItem(
+  item: IChartEventItem | IChartEvent,
+): IChartEventItem {
+  if ('type' in item) {
+    return item;
+  }
+  // Old format without type field - assume it's an event
+  return { ...item, type: 'event' as const };
+}
+
+// Calculate formula result from previous series
+function calculateFormulaSeries(
+  formula: IChartFormula,
+  previousSeries: Awaited<ReturnType<typeof getChartSerie>>,
+  normalizedEvents: IChartEventItem[],
+  formulaIndex: number,
+): Awaited<ReturnType<typeof getChartSerie>> {
+  if (!previousSeries || previousSeries.length === 0) {
+    return [];
+  }
+
+  if (!previousSeries[0]?.data) {
+    return [];
+  }
+
+  // Detect if we have breakdowns by checking if series names contain breakdown values
+  // (not event/formula names)
+  const hasBreakdowns = previousSeries.some(
+    (serie) =>
+      serie.name.length > 1 || // Multiple name parts = breakdowns
+      (serie.name.length === 1 &&
+        !normalizedEvents
+          .slice(0, formulaIndex)
+          .some(
+            (event) =>
+              event.type === 'event' &&
+              (serie.name[0] === event.name ||
+                serie.name[0] === event.displayName),
+          ) &&
+        !normalizedEvents
+          .slice(0, formulaIndex)
+          .some(
+            (event) =>
+              event.type === 'formula' &&
+              (serie.name[0] === event.displayName ||
+                serie.name[0] === event.formula),
+          )),
+  );
+
+  const seriesByBreakdown = new Map<
+    string,
+    Awaited<ReturnType<typeof getChartSerie>>
+  >();
+
+  previousSeries.forEach((serie) => {
+    let breakdownKey: string;
+
+    if (hasBreakdowns) {
+      // With breakdowns: use the entire name array as the breakdown key
+      // Skip the first element (event/formula name) and use breakdown values
+      breakdownKey = serie.name.slice(1).join(':::');
+    } else {
+      // Without breakdowns: group all series together
+      // This allows formulas to combine multiple events/formulas
+      breakdownKey = '';
+    }
+
+    if (!seriesByBreakdown.has(breakdownKey)) {
+      seriesByBreakdown.set(breakdownKey, []);
+    }
+    seriesByBreakdown.get(breakdownKey)!.push(serie);
+  });
+
+  const result: Awaited<ReturnType<typeof getChartSerie>> = [];
+
+  for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) {
+    // Group series by event index to ensure we have one series per event
+    const seriesByEventIndex = new Map<
+      number,
+      (typeof previousSeries)[number]
+    >();
+
+    breakdownSeries.forEach((serie) => {
+      // Find which event index this series belongs to
+      const eventIndex = normalizedEvents
+        .slice(0, formulaIndex)
+        .findIndex((event) => {
+          if (event.type === 'event') {
+            const eventId = event.id ?? event.name;
+            return (
+              serie.event.id === eventId || serie.event.name === event.name
+            );
+          }
+          return false;
+        });
+
+      if (eventIndex >= 0 && !seriesByEventIndex.has(eventIndex)) {
+        seriesByEventIndex.set(eventIndex, serie);
+      }
+    });
+
+    // Get all unique dates across all series in this breakdown group
+    const allDates = new Set<string>();
+    breakdownSeries.forEach((serie) => {
+      serie.data.forEach((item) => {
+        allDates.add(item.date);
+      });
+    });
+
+    // Sort dates chronologically
+    const sortedDates = Array.from(allDates).sort(
+      (a, b) => new Date(a).getTime() - new Date(b).getTime(),
+    );
+
+    // Apply formula for each date
+    const formulaData = sortedDates.map((date) => {
+      const scope: Record<string, number> = {};
+
+      // Build scope using alphabet IDs (A, B, C, etc.) for each event before this formula
+      normalizedEvents.slice(0, formulaIndex).forEach((event, eventIndex) => {
+        const readableId = alphabetIds[eventIndex];
+        if (!readableId) {
+          return;
+        }
+
+        if (event.type === 'event') {
+          const matchingSerie = seriesByEventIndex.get(eventIndex);
+          const dataPoint = matchingSerie?.data.find((d) => d.date === date);
+          scope[readableId] = dataPoint?.count ?? 0;
+        } else {
+          // If it's a formula, we need to get its calculated value
+          // This handles nested formulas
+          const formulaSerie = breakdownSeries.find(
+            (s) => s.event.id === event.id,
+          );
+          const dataPoint = formulaSerie?.data.find((d) => d.date === date);
+          scope[readableId] = dataPoint?.count ?? 0;
+        }
+      });
+
+      // Evaluate the formula with the scope
+      let count: number;
+      try {
+        count = mathjs
+          .parse(formula.formula)
+          .compile()
+          .evaluate(scope) as number;
+      } catch (error) {
+        count = 0;
+      }
+
+      return {
+        date,
+        count:
+          Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2),
+        total_count: breakdownSeries[0]?.data.find((d) => d.date === date)
+          ?.total_count,
+      };
+    });
+
+    // Use the first series as a template
+    const templateSerie = breakdownSeries[0]!;
+
+    // For formulas, construct the name array:
+    // - Without breakdowns: use formula displayName/formula
+    // - With breakdowns: use formula displayName/formula as first element, then breakdown values
+    let formulaName: string[];
+    if (hasBreakdowns) {
+      // With breakdowns: formula name + breakdown values (skip first element which is event/formula name)
+      const formulaDisplayName = formula.displayName || formula.formula;
+      formulaName = [formulaDisplayName, ...templateSerie.name.slice(1)];
+    } else {
+      // Without breakdowns: just formula name
+      formulaName = [formula.displayName || formula.formula];
+    }
+
+    result.push({
+      ...templateSerie,
+      name: formulaName,
+      // For formulas, create a simplified event object
+      // We use 'as' because formulas don't have segment/filters, but the event
+      // object is only used for id/name lookups later, so this is safe
+      event: {
+        id: formula.id,
+        name: formula.displayName || formula.formula,
+        displayName: formula.displayName,
+        segment: 'event' as const,
+        filters: [],
+      } as IChartEvent,
+      data: formulaData,
+    });
+  }
+
+  return result;
+}
+
 export type IGetChartSerie = Awaited<ReturnType<typeof getChartSeries>>[number];
 export async function getChartSeries(
   input: IChartInputWithDates,
   timezone: string,
 ) {
-  const series = (
-    await Promise.all(
-      input.events.map(async (event) =>
-        getChartSerie(
-          {
-            ...input,
-            event,
-          },
-          timezone,
-        ),
-      ),
-    )
-  ).flat();
+  // Normalize all events to have type field
+  const normalizedEvents = input.events.map(normalizeEventItem);
 
-  try {
-    return withFormula(input, series);
-  } catch (e) {
-    return series;
+  // Process events sequentially - events fetch data, formulas calculate from previous series
+  const allSeries: Awaited<ReturnType<typeof getChartSerie>> = [];
+
+  for (let i = 0; i < normalizedEvents.length; i++) {
+    const item = normalizedEvents[i]!;
+
+    if (item.type === 'event') {
+      // Fetch data for event
+      const eventSeries = await getChartSerie(
+        {
+          ...input,
+          event: item,
+        },
+        timezone,
+      );
+      allSeries.push(...eventSeries);
+    } else if (item.type === 'formula') {
+      // Calculate formula from previous series
+      const formulaSeries = calculateFormulaSeries(
+        item,
+        allSeries,
+        normalizedEvents,
+        i,
+      );
+      allSeries.push(...formulaSeries);
+    }
   }
+
+  // Apply top-level formula if present (for backward compatibility)
+  try {
+    if (input.formula) {
+      return withFormula(input, allSeries);
+    }
+  } catch (e) {
+    // If formula evaluation fails, return series as-is
+  }
+
+  return allSeries;
 }
 
 export async function getChart(input: IChartInput) {
@@ -361,6 +606,9 @@ export async function getChart(input: IChartInput) {
     );
   }
 
+  // Normalize events for consistent handling
+  const normalizedEvents = input.events.map(normalizeEventItem);
+
   const getSerieId = (serie: IGetChartSerie) =>
     [slug(serie.name.join('-')), serie.event.id].filter(Boolean).join('-');
   const result = await Promise.all(promises);
@@ -368,36 +616,163 @@ export async function getChart(input: IChartInput) {
   const previousSeries = result[1];
   const limit = input.limit || 300;
   const offset = input.offset || 0;
-  const includeEventAlphaId = input.events.length > 1;
+  const includeEventAlphaId = normalizedEvents.length > 1;
+
+  // Calculate metrics cache for formulas
+  // Map<eventIndex, Map<breakdownSignature, metrics>>
+  const metricsCache = new Map<
+    number,
+    Map<
+      string,
+      {
+        sum: number;
+        average: number;
+        min: number;
+        max: number;
+        count: number;
+      }
+    >
+  >();
+
+  // Initialize cache
+  for (let i = 0; i < normalizedEvents.length; i++) {
+    metricsCache.set(i, new Map());
+  }
+
+  // First pass: calculate standard metrics for all series and populate cache
+  // We iterate through series in order, but since series array is flattened, we need to be careful.
+  // Fortunately, events are processed sequentially, so dependencies usually appear before formulas.
+  // However, to be safe, we'll compute metrics for all series first.
+
+  const seriesWithMetrics = series.map((serie) => {
+    // Find the index of the event/formula that produced this series
+    const eventIndex = normalizedEvents.findIndex((event) => {
+      if (event.type === 'event') {
+        return event.id === serie.event.id || event.name === serie.event.name;
+      }
+      return event.id === serie.event.id;
+    });
+
+    const standardMetrics = {
+      sum: sum(serie.data.map((item) => item.count)),
+      average: round(average(serie.data.map((item) => item.count)), 2),
+      min: min(serie.data.map((item) => item.count)),
+      max: max(serie.data.map((item) => item.count)),
+      count: serie.data.find((item) => !!item.total_count)?.total_count || 0,
+    };
+
+    // Store in cache
+    if (eventIndex >= 0) {
+      const breakdownSignature = serie.name.slice(1).join(':::');
+      metricsCache.get(eventIndex)?.set(breakdownSignature, standardMetrics);
+    }
+
+    return {
+      serie,
+      eventIndex,
+      metrics: standardMetrics,
+    };
+  });
+
+  // Second pass: Re-calculate metrics for formulas using dependency metrics
+  // We iterate through normalizedEvents to process in dependency order
+  normalizedEvents.forEach((event, eventIndex) => {
+    if (event.type !== 'formula') return;
+
+    // We dont have count on formulas so use sum instead
+    const property = 'count';
+    // Iterate through all series corresponding to this formula
+    seriesWithMetrics.forEach((item) => {
+      if (item.eventIndex !== eventIndex) return;
+
+      const breakdownSignature = item.serie.name.slice(1).join(':::');
+      const scope: Record<string, number> = {};
+
+      // Build scope from dependency metrics
+      normalizedEvents.slice(0, eventIndex).forEach((depEvent, depIndex) => {
+        const readableId = alphabetIds[depIndex];
+        if (!readableId) return;
+
+        // Get metric from cache for the dependency with the same breakdown signature
+        const depMetrics = metricsCache.get(depIndex)?.get(breakdownSignature);
+        // Use sum as the default metric for formula calculation on totals
+        scope[readableId] = depMetrics?.[property] ?? 0;
+      });
+
+      // Evaluate formula
+      let calculatedSum: number;
+      try {
+        calculatedSum = mathjs
+          .parse(event.formula)
+          .compile()
+          .evaluate(scope) as number;
+      } catch (error) {
+        calculatedSum = 0;
+      }
+
+      // Update metrics with calculated sum
+      // For formulas, the "sum" metric (Total) should be the result of the formula applied to the totals
+      // The "average" metric usually remains average of data points, or calculatedSum / intervals
+      item.metrics = {
+        ...item.metrics,
+        [property]:
+          Number.isNaN(calculatedSum) || !Number.isFinite(calculatedSum)
+            ? 0
+            : round(calculatedSum, 2),
+      };
+
+      // Update cache with new metrics so dependent formulas can use it
+      metricsCache.get(eventIndex)?.set(breakdownSignature, item.metrics);
+    });
+  });
+
   const final: FinalChart = {
-    series: series.map((serie, index) => {
-      const eventIndex = input.events.findIndex(
-        (event) => event.id === serie.event.id,
-      );
+    series: seriesWithMetrics.map(({ serie, eventIndex, metrics }) => {
       const alphaId = alphabetIds[eventIndex];
       const previousSerie = previousSeries?.find(
         (prevSerie) => getSerieId(prevSerie) === getSerieId(serie),
       );
-      const metrics = {
-        sum: sum(serie.data.map((item) => item.count)),
-        average: round(average(serie.data.map((item) => item.count)), 2),
-        min: min(serie.data.map((item) => item.count)),
-        max: max(serie.data.map((item) => item.count)),
-        count: serie.data[0]?.total_count, // We can grab any since all are the same
-      };
+
+      // Determine if this is a formula series
+      const isFormula = normalizedEvents[eventIndex]?.type === 'formula';
+      const eventItem = normalizedEvents[eventIndex];
+
       const event = {
         id: serie.event.id,
         name: serie.event.displayName || serie.event.name,
       };
 
-      return {
-        id: getSerieId(serie),
-        names:
+      // Construct names array based on whether it's a formula or event
+      let names: string[];
+      if (isFormula && eventItem?.type === 'formula') {
+        // For formulas:
+        // - Without breakdowns: use displayName/formula (with optional alpha ID)
+        // - With breakdowns: use displayName/formula + breakdown values (with optional alpha ID)
+        const formulaDisplayName = eventItem.displayName || eventItem.formula;
+        if (input.breakdowns.length === 0) {
+          // No breakdowns: just formula name
+          names = includeEventAlphaId
+            ? [`(${alphaId}) ${formulaDisplayName}`]
+            : [formulaDisplayName];
+        } else {
+          // With breakdowns: formula name + breakdown values
+          names = includeEventAlphaId
+            ? [`(${alphaId}) ${formulaDisplayName}`, ...serie.name.slice(1)]
+            : [formulaDisplayName, ...serie.name.slice(1)];
+        }
+      } else {
+        // For events: use existing logic
+        names =
           input.breakdowns.length === 0 && serie.event.displayName
             ? [serie.event.displayName]
             : includeEventAlphaId
               ? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)]
-              : serie.name,
+              : serie.name;
+      }
+
+      return {
+        id: getSerieId(serie),
+        names,
         event,
         metrics: {
           ...metrics,

packages/trpc/src/routers/chart.ts

@@ -10,15 +10,20 @@ import {
   chQuery,
   clix,
   conversionService,
+  createSqlBuilder,
   db,
+  formatClickhouseDate,
   funnelService,
   getChartPrevStartEndDate,
   getChartStartEndDate,
+  getEventFiltersWhereClause,
   getEventMetasCached,
+  getProfilesCached,
   getSelectPropertyKey,
   getSettingsForProject,
 } from '@openpanel/db';
 import {
+  zChartEvent,
   zChartInput,
   zCriteria,
   zRange,
@@ -532,6 +537,166 @@ export const chartRouter = createTRPCRouter({
 
       return processCohortData(cohortData, diffInterval);
     }),
+
+  getProfiles: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        event: zChartEvent,
+        date: z.string().describe('The date for the data point (ISO string)'),
+        breakdowns: z
+          .array(
+            z.object({
+              id: z.string().optional(),
+              name: z.string(),
+            }),
+          )
+          .default([]),
+        interval: zTimeInterval.default('day'),
+        startDate: z.string(),
+        endDate: z.string(),
+        filters: z
+          .array(
+            z.object({
+              id: z.string().optional(),
+              name: z.string(),
+              operator: z.string(),
+              value: z.array(
+                z.union([z.string(), z.number(), z.boolean(), z.null()]),
+              ),
+            }),
+          )
+          .default([]),
+        limit: z.number().default(100),
+      }),
+    )
+    .query(async ({ input }) => {
+      const { timezone } = await getSettingsForProject(input.projectId);
+      const {
+        projectId,
+        event,
+        date,
+        breakdowns,
+        interval,
+        startDate,
+        endDate,
+        filters,
+        limit,
+      } = input;
+
+      // Build the date range for the specific interval bucket
+      const dateObj = new Date(date);
+      let bucketStart: Date;
+      let bucketEnd: Date;
+
+      switch (interval) {
+        case 'minute':
+          bucketStart = new Date(
+            dateObj.getFullYear(),
+            dateObj.getMonth(),
+            dateObj.getDate(),
+            dateObj.getHours(),
+            dateObj.getMinutes(),
+          );
+          bucketEnd = new Date(bucketStart.getTime() + 60 * 1000);
+          break;
+        case 'hour':
+          bucketStart = new Date(
+            dateObj.getFullYear(),
+            dateObj.getMonth(),
+            dateObj.getDate(),
+            dateObj.getHours(),
+          );
+          bucketEnd = new Date(bucketStart.getTime() + 60 * 60 * 1000);
+          break;
+        case 'day':
+          bucketStart = new Date(
+            dateObj.getFullYear(),
+            dateObj.getMonth(),
+            dateObj.getDate(),
+          );
+          bucketEnd = new Date(bucketStart.getTime() + 24 * 60 * 60 * 1000);
+          break;
+        case 'week':
+          bucketStart = new Date(dateObj);
+          bucketStart.setDate(dateObj.getDate() - dateObj.getDay());
+          bucketStart.setHours(0, 0, 0, 0);
+          bucketEnd = new Date(bucketStart.getTime() + 7 * 24 * 60 * 60 * 1000);
+          break;
+        case 'month':
+          bucketStart = new Date(dateObj.getFullYear(), dateObj.getMonth(), 1);
+          bucketEnd = new Date(
+            dateObj.getFullYear(),
+            dateObj.getMonth() + 1,
+            1,
+          );
+          break;
+        default:
+          bucketStart = new Date(
+            dateObj.getFullYear(),
+            dateObj.getMonth(),
+            dateObj.getDate(),
+          );
+          bucketEnd = new Date(bucketStart.getTime() + 24 * 60 * 60 * 1000);
+      }
+
+      // Build query to get unique profile_ids for this time bucket
+      const { sb, join, getWhere, getFrom, getJoins } = createSqlBuilder();
+
+      sb.where = getEventFiltersWhereClause([...event.filters, ...filters]);
+      sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
+      sb.where.dateRange = `created_at >= '${formatClickhouseDate(bucketStart.toISOString())}' AND created_at < '${formatClickhouseDate(bucketEnd.toISOString())}'`;
+
+      if (event.name !== '*') {
+        sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
+      }
+
+      // Handle breakdowns if provided
+      const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
+        breakdown.name.startsWith('profile.'),
+      );
+      const anyFilterOnProfile = [...event.filters, ...filters].some((filter) =>
+        filter.name.startsWith('profile.'),
+      );
+
+      if (anyFilterOnProfile || anyBreakdownOnProfile) {
+        sb.joins.profiles = `LEFT ANY JOIN (SELECT 
+          id as "profile.id",
+          email as "profile.email",
+          first_name as "profile.first_name",
+          last_name as "profile.last_name",
+          properties as "profile.properties"
+        FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
+      }
+
+      // Apply breakdown filters if provided
+      breakdowns.forEach((breakdown) => {
+        // This is simplified - in reality we'd need to match the breakdown value
+        // For now, we'll just get all profiles for the time bucket
+      });
+
+      // Get unique profile IDs
+      const profileIdsQuery = `
+        SELECT DISTINCT profile_id
+        FROM ${TABLE_NAMES.events}
+        ${getJoins()}
+        WHERE ${join(sb.where, ' AND ')}
+          AND profile_id != ''
+        LIMIT ${limit}
+      `;
+
+      const profileIds = await chQuery<{ profile_id: string }>(profileIdsQuery);
+
+      if (profileIds.length === 0) {
+        return [];
+      }
+
+      // Fetch profile details
+      const ids = profileIds.map((p) => p.profile_id).filter(Boolean);
+      const profiles = await getProfilesCached(ids, projectId);
+
+      return profiles;
+    }),
 });
 
 function processCohortData(