Revert "improve(funnel): make sure group by profile id works as you would think"

This reverts commit 56edb91dd0.

packages/trpc/src/routers/chart.helpers.ts

@@ -13,9 +13,9 @@ import {
   subYears,
 } from 'date-fns';
 import * as mathjs from 'mathjs';
-import { pluck, uniq } from 'ramda';
+import { last, pluck, repeat, reverse, uniq } from 'ramda';
+import { escape } from 'sqlstring';
 
-import type { ISerieDataItem } from '@openpanel/common';
 import {
   average,
   completeSerie,
@@ -26,8 +26,17 @@ import {
   slug,
   sum,
 } from '@openpanel/common';
+import type { ISerieDataItem } from '@openpanel/common';
 import { alphabetIds } from '@openpanel/constants';
-import { chQuery, getChartSql } from '@openpanel/db';
+import {
+  TABLE_NAMES,
+  chQuery,
+  createSqlBuilder,
+  formatClickhouseDate,
+  getChartSql,
+  getEventFiltersWhereClause,
+  getProfiles,
+} from '@openpanel/db';
 import type {
   FinalChart,
   IChartEvent,
@@ -232,6 +241,28 @@ export function getDatesFromRange(range: IChartRange) {
   };
 }
 
+function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
+  const filled = Array.from({ length: steps }, (_, index) => {
+    const level = index + 1;
+    const matchingResult = funnel.find((res) => res.level === level);
+    return {
+      level,
+      count: matchingResult ? matchingResult.count : 0,
+    };
+  });
+
+  // Accumulate counts from top to bottom of the funnel
+  for (let i = filled.length - 1; i >= 0; i--) {
+    const step = filled[i];
+    const prevStep = filled[i + 1];
+    // If there's a previous step, add the count to the current step
+    if (step && prevStep) {
+      step.count += prevStep.count;
+    }
+  }
+  return filled.reverse();
+}
+
 export function getChartStartEndDate({
   startDate,
   endDate,
@@ -257,6 +288,147 @@ export function getChartPrevStartEndDate({
   };
 }
 
+export async function getFunnelData({
+  projectId,
+  startDate,
+  endDate,
+  ...payload
+}: IChartInput) {
+  const funnelWindow = (payload.funnelWindow || 24) * 3600;
+  const funnelGroup = payload.funnelGroup || 'session_id';
+
+  if (!startDate || !endDate) {
+    throw new Error('startDate and endDate are required');
+  }
+
+  if (payload.events.length === 0) {
+    return {
+      totalSessions: 0,
+      steps: [],
+    };
+  }
+
+  const funnels = payload.events.map((event) => {
+    const { sb, getWhere } = createSqlBuilder();
+    sb.where = getEventFiltersWhereClause(event.filters);
+    sb.where.name = `name = ${escape(event.name)}`;
+    return getWhere().replace('WHERE ', '');
+  });
+
+  const innerSql = `SELECT
+    ${funnelGroup},
+    windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
+  FROM ${TABLE_NAMES.events}
+  WHERE 
+    project_id = ${escape(projectId)} AND 
+    created_at >= '${formatClickhouseDate(startDate)}' AND 
+    created_at <= '${formatClickhouseDate(endDate)}' AND
+    name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
+  GROUP BY ${funnelGroup}`;
+
+  const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`;
+
+  const funnel = await chQuery<{ level: number; count: number }>(sql);
+  const maxLevel = payload.events.length;
+  const filledFunnelRes = fillFunnel(funnel, maxLevel);
+
+  const totalSessions = last(filledFunnelRes)?.count ?? 0;
+  const steps = reverse(filledFunnelRes).reduce(
+    (acc, item, index, list) => {
+      const prev = list[index - 1] ?? { count: totalSessions };
+      const event = payload.events[item.level - 1]!;
+      return [
+        ...acc,
+        {
+          event: {
+            ...event,
+            displayName: event.displayName ?? event.name,
+          },
+          count: item.count,
+          percent: (item.count / totalSessions) * 100,
+          dropoffCount: prev.count - item.count,
+          dropoffPercent: 100 - (item.count / prev.count) * 100,
+          previousCount: prev.count,
+        },
+      ];
+    },
+    [] as {
+      event: IChartEvent & { displayName: string };
+      count: number;
+      percent: number;
+      dropoffCount: number;
+      dropoffPercent: number;
+      previousCount: number;
+    }[],
+  );
+
+  return {
+    totalSessions,
+    steps,
+  };
+}
+
+export async function getFunnelStep({
+  projectId,
+  startDate,
+  endDate,
+  step,
+  ...payload
+}: IChartInput & {
+  step: number;
+}) {
+  throw new Error('not implemented');
+  // if (!startDate || !endDate) {
+  //   throw new Error('startDate and endDate are required');
+  // }
+
+  // if (payload.events.length === 0) {
+  //   throw new Error('no events selected');
+  // }
+
+  // const funnels = payload.events.map((event) => {
+  //   const { sb, getWhere } = createSqlBuilder();
+  //   sb.where = getEventFiltersWhereClause(event.filters);
+  //   sb.where.name = `name = ${escape(event.name)}`;
+  //   return getWhere().replace('WHERE ', '');
+  // });
+
+  // const innerSql = `SELECT
+  //   session_id,
+  //   windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
+  // FROM ${TABLE_NAMES.events}
+  // WHERE
+  //   project_id = ${escape(projectId)} AND
+  //   created_at >= '${formatClickhouseDate(startDate)}' AND
+  //   created_at <= '${formatClickhouseDate(endDate)}' AND
+  //   name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
+  // GROUP BY session_id`;
+
+  // const profileIdsQuery = `WITH sessions AS (${innerSql})
+  //   SELECT
+  //     DISTINCT e.profile_id as id
+  //   FROM sessions s
+  //   JOIN ${TABLE_NAMES.events} e ON s.session_id = e.session_id
+  //   WHERE
+  //     s.level = ${step} AND
+  //     e.project_id = ${escape(projectId)} AND
+  //     e.created_at >= '${formatClickhouseDate(startDate)}' AND
+  //     e.created_at <= '${formatClickhouseDate(endDate)}' AND
+  //     name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
+  //   ORDER BY e.created_at DESC
+  //   LIMIT 500
+  //   `;
+
+  // const res = await chQuery<{
+  //   id: string;
+  // }>(profileIdsQuery);
+
+  // return getProfiles(
+  //   res.map((r) => r.id),
+  //   projectId,
+  // );
+}
+
 export async function getChartSerie(payload: IGetChartDataInput) {
   async function getSeries() {
     const result = await chQuery<ISerieDataItem>(getChartSql(payload));

packages/trpc/src/routers/chart.ts

@@ -7,7 +7,6 @@ import {
   chQuery,
   createSqlBuilder,
   db,
-  getFunnelData,
   getSelectPropertyKey,
   toDate,
 } from '@openpanel/db';
@@ -32,6 +31,8 @@ import {
   getChart,
   getChartPrevStartEndDate,
   getChartStartEndDate,
+  getFunnelData,
+  getFunnelStep,
 } from './chart.helpers';
 
 function utc(date: string | Date) {
@@ -86,12 +87,9 @@ export const chartRouter = createTRPCRouter({
         .map((item) => item.replace(/\.([0-9]+)/g, '[*]'))
         .map((item) => `properties.${item}`);
 
-      if (event === '*') {
-        properties.push('name');
-      }
-
       properties.push(
         'has_profile',
+        'name',
         'path',
         'origin',
         'referrer',
@@ -186,9 +184,7 @@ export const chartRouter = createTRPCRouter({
 
     const [current, previous] = await Promise.all([
       getFunnelData({ ...input, ...currentPeriod }),
-      input.previous
-        ? getFunnelData({ ...input, ...previousPeriod })
-        : Promise.resolve(null),
+      getFunnelData({ ...input, ...previousPeriod }),
     ]);
 
     return {
@@ -197,6 +193,17 @@ export const chartRouter = createTRPCRouter({
     };
   }),
 
+  funnelStep: protectedProcedure
+    .input(
+      zChartInput.extend({
+        step: z.number(),
+      }),
+    )
+    .query(async ({ input }) => {
+      const currentPeriod = getChartStartEndDate(input);
+      return getFunnelStep({ ...input, ...currentPeriod });
+    }),
+
   chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => {
     if (ctx.session.userId) {
       const access = await getProjectAccessCached({