feat: group analytics

* wip * wip * wip * wip * wip * add buffer * wip * wip * fixes * fix * wip * group validation * fix group issues * docs: add groups
2026-03-20 10:46:09 +01:00
parent 88a2d876ce
commit 11e9ecac1a
99 changed files with 5944 additions and 1432 deletions
--- a/packages/trpc/src/root.ts
+++ b/packages/trpc/src/root.ts
@@ -1,11 +1,12 @@
 import { authRouter } from './routers/auth';
-import { gscRouter } from './routers/gsc';
 import { chartRouter } from './routers/chart';
 import { chatRouter } from './routers/chat';
 import { clientRouter } from './routers/client';
 import { dashboardRouter } from './routers/dashboard';
 import { emailRouter } from './routers/email';
 import { eventRouter } from './routers/event';
+import { groupRouter } from './routers/group';
+import { gscRouter } from './routers/gsc';
 import { importRouter } from './routers/import';
 import { insightRouter } from './routers/insight';
 import { integrationRouter } from './routers/integration';
@@ -55,6 +56,7 @@ export const appRouter = createTRPCRouter({
  widget: widgetRouter,
  email: emailRouter,
  gsc: gscRouter,
+  group: groupRouter,
 });

 // export type definition of API
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -13,11 +13,12 @@ import {
  getChartStartEndDate,
  getEventFiltersWhereClause,
  getEventMetasCached,
+  getGroupPropertySelect,
+  getProfilePropertySelect,
  getProfilesCached,
  getReportById,
  getSelectPropertyKey,
  getSettingsForProject,
-  type IClickhouseProfile,
  type IServiceProfile,
  onlyReportEvents,
  sankeyService,
@@ -354,6 +355,33 @@ export const chartRouter = createTRPCRouter({
        const res = await query.execute();

        values.push(...res.map((e) => e.property_value));
+      } else if (property.startsWith('profile.')) {
+        const selectExpr = getProfilePropertySelect(property);
+        const query = clix(ch)
+          .select<{ values: string }>([`distinct ${selectExpr} as values`])
+          .from(TABLE_NAMES.profiles, true)
+          .where('project_id', '=', projectId)
+          .where(selectExpr, '!=', '')
+          .where(selectExpr, 'IS NOT NULL', null)
+          .orderBy('created_at', 'DESC')
+          .limit(100_000);
+
+        const res = await query.execute();
+        values.push(...res.map((r) => String(r.values)).filter(Boolean));
+      } else if (property.startsWith('group.')) {
+        const selectExpr = getGroupPropertySelect(property);
+        const query = clix(ch)
+          .select<{ values: string }>([`distinct ${selectExpr} as values`])
+          .from(TABLE_NAMES.groups, true)
+          .where('project_id', '=', projectId)
+          .where('deleted', '=', 0)
+          .where(selectExpr, '!=', '')
+          .where(selectExpr, 'IS NOT NULL', null)
+          .orderBy('created_at', 'DESC')
+          .limit(100_000);
+
+        const res = await query.execute();
+        values.push(...res.map((r) => String(r.values)).filter(Boolean));
      } else {
        const query = clix(ch)
          .select<{ values: string[] }>([
@@ -369,17 +397,6 @@ export const chartRouter = createTRPCRouter({
          query.where('name', '=', event);
        }

-        if (property.startsWith('profile.')) {
-          query.leftAnyJoin(
-            clix(ch)
-              .select<IClickhouseProfile>([])
-              .from(TABLE_NAMES.profiles)
-              .where('project_id', '=', projectId),
-            'profile.id = profile_id',
-            'profile'
-          );
-        }
-
        const events = await query.execute();

        values.push(
@@ -785,7 +802,7 @@ export const chartRouter = createTRPCRouter({
      const { sb, getSql } = createSqlBuilder();

      sb.select.profile_id = 'DISTINCT profile_id';
-      sb.where = getEventFiltersWhereClause(serie.filters);
+      sb.where = getEventFiltersWhereClause(serie.filters, projectId);
      sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
      sb.where.dateRange = `${clix.toStartOf('created_at', input.interval)} = ${clix.toDate(sqlstring.escape(formatClickhouseDate(dateObj)), input.interval)}`;
      if (serie.name !== '*') {
@@ -812,10 +829,22 @@ export const chartRouter = createTRPCRouter({
        sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
      }

+      // Check for group filters/breakdowns and add ARRAY JOIN if needed
+      const anyFilterOnGroup = serie.filters.some((f) =>
+        f.name.startsWith('group.')
+      );
+      const anyBreakdownOnGroup = input.breakdowns
+        ? Object.keys(input.breakdowns).some((key) => key.startsWith('group.'))
+        : false;
+      if (anyFilterOnGroup || anyBreakdownOnGroup) {
+        sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
+        sb.joins.groups_cte = `LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`;
+      }
+
      if (input.breakdowns) {
        Object.entries(input.breakdowns).forEach(([key, value]) => {
          // Transform property keys (e.g., properties.method -> properties['method'])
-          const propertyKey = getSelectPropertyKey(key);
+          const propertyKey = getSelectPropertyKey(key, projectId);
          sb.where[`breakdown_${key}`] =
            `${propertyKey} = ${sqlstring.escape(value)}`;
        });
@@ -858,6 +887,7 @@ export const chartRouter = createTRPCRouter({
        funnelWindow: z.number().optional(),
        funnelGroup: z.string().optional(),
        breakdowns: z.array(z.object({ name: z.string() })).optional(),
+        breakdownValues: z.array(z.string()).optional(),
        range: zRange,
      })
    )
@@ -870,6 +900,8 @@ export const chartRouter = createTRPCRouter({
        showDropoffs = false,
        funnelWindow,
        funnelGroup,
+        breakdowns = [],
+        breakdownValues = [],
      } = input;

      const { startDate, endDate } = getChartStartEndDate(input, timezone);
@@ -889,9 +921,21 @@ export const chartRouter = createTRPCRouter({
      // Get the grouping strategy (profile_id or session_id)
      const group = funnelService.getFunnelGroup(funnelGroup);

+      const anyFilterOnGroup = (eventSeries as IChartEvent[]).some((e) =>
+        e.filters?.some((f) => f.name.startsWith('group.'))
+      );
+      const anyBreakdownOnGroup = breakdowns.some((b) =>
+        b.name.startsWith('group.')
+      );
+      const needsGroupArrayJoin = anyFilterOnGroup || anyBreakdownOnGroup;
+
+      // Breakdown selects/groupBy so we can filter by specific breakdown values
+      const breakdownSelects = breakdowns.map(
+        (b, index) => `${getSelectPropertyKey(b.name, projectId)} as b_${index}`
+      );
+      const breakdownGroupBy = breakdowns.map((_, index) => `b_${index}`);
+
      // Create funnel CTE using funnel service
-      // Note: buildFunnelCte always computes windowFunnel per session_id and extracts
-      // profile_id via argMax to handle identity changes mid-session correctly.
      const funnelCte = funnelService.buildFunnelCte({
        projectId,
        startDate,
@@ -899,8 +943,8 @@ export const chartRouter = createTRPCRouter({
        eventSeries: eventSeries as IChartEvent[],
        funnelWindowMilliseconds,
        timezone,
-        // No need to add profile_id to additionalSelects/additionalGroupBy
-        // since buildFunnelCte already extracts it via argMax(profile_id, created_at)
+        additionalSelects: breakdownSelects,
+        additionalGroupBy: breakdownGroupBy,
      });

      // Check for profile filters and add profile join if needed
@@ -917,36 +961,50 @@ export const chartRouter = createTRPCRouter({
        );
      }

+      if (needsGroupArrayJoin) {
+        funnelCte.rawJoin('ARRAY JOIN groups AS _group_id');
+        funnelCte.rawJoin('LEFT ANY JOIN _g ON _g.id = _group_id');
+      }
+
      // Build main query
      const query = clix(ch, timezone);
+      if (needsGroupArrayJoin) {
+        query.with(
+          '_g',
+          `SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`
+        );
+      }
      query.with('session_funnel', funnelCte);

      if (group === 'profile_id') {
-        // For profile grouping: re-aggregate by profile_id, taking MAX level per profile.
-        // This ensures a user who completed the funnel with identity change is counted correctly.
-        // NOTE: Wrap in subquery to avoid ClickHouse resolving `level` in WHERE to the
-        // `max(level) AS level` alias (ILLEGAL_AGGREGATION error).
+        const breakdownAggregates =
+          breakdowns.length > 0
+            ? `, ${breakdowns.map((_, index) => `any(b_${index}) AS b_${index}`).join(', ')}`
+            : '';
        query.with(
          'funnel',
-          'SELECT profile_id, max(level) AS level FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id'
+          `SELECT profile_id, max(level) AS level${breakdownAggregates} FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id`
        );
      } else {
-        // For session grouping: filter out level = 0 inside the CTE
        query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');
      }

-      // Get distinct profile IDs
-      // NOTE: level != 0 is already filtered inside the funnel CTE above
      query.select(['DISTINCT profile_id']).from('funnel');

      if (showDropoffs) {
-        // Show users who dropped off at this step (completed this step but not the next)
        query.where('level', '=', targetLevel);
      } else {
-        // Show users who completed at least this step
        query.where('level', '>=', targetLevel);
      }

+      // Filter by specific breakdown values when a breakdown row was clicked
+      breakdowns.forEach((_, index) => {
+        const value = breakdownValues[index];
+        if (value !== undefined) {
+          query.where(`b_${index}`, '=', value);
+        }
+      });
+
      // Cap the number of profiles to avoid exceeding ClickHouse max_query_size
      // when passing IDs to the next query
      query.limit(1000);
--- a/packages/trpc/src/routers/event.ts
+++ b/packages/trpc/src/routers/event.ts
@@ -122,6 +122,7 @@ export const eventRouter = createTRPCRouter({
        projectId: z.string(),
        profileId: z.string().optional(),
        sessionId: z.string().optional(),
+        groupId: z.string().optional(),
        cursor: z.string().optional(),
        filters: z.array(zChartEventFilter).default([]),
        startDate: z.date().optional(),
--- a/packages/trpc/src/routers/group.ts
+++ b/packages/trpc/src/routers/group.ts
@@ -0,0 +1,233 @@
+import {
+  chQuery,
+  createGroup,
+  deleteGroup,
+  getGroupById,
+  getGroupList,
+  getGroupListCount,
+  getGroupMemberProfiles,
+  getGroupPropertyKeys,
+  getGroupStats,
+  getGroupsByIds,
+  getGroupTypes,
+  TABLE_NAMES,
+  toNullIfDefaultMinDate,
+  updateGroup,
+} from '@openpanel/db';
+import { zCreateGroup, zUpdateGroup } from '@openpanel/validation';
+import sqlstring from 'sqlstring';
+import { z } from 'zod';
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export const groupRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        cursor: z.number().optional(),
+        take: z.number().default(50),
+        search: z.string().optional(),
+        type: z.string().optional(),
+      })
+    )
+    .query(async ({ input }) => {
+      const [data, count] = await Promise.all([
+        getGroupList(input),
+        getGroupListCount(input),
+      ]);
+      const stats = await getGroupStats(
+        input.projectId,
+        data.map((g) => g.id)
+      );
+      return {
+        data: data.map((g) => ({
+          ...g,
+          memberCount: stats.get(g.id)?.memberCount ?? 0,
+          lastActiveAt: stats.get(g.id)?.lastActiveAt ?? null,
+        })),
+        meta: { count, take: input.take },
+      };
+    }),
+
+  byId: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .query(({ input: { id, projectId } }) => {
+      return getGroupById(id, projectId);
+    }),
+
+  create: protectedProcedure
+    .input(zCreateGroup)
+    .mutation(({ input }) => {
+      return createGroup(input);
+    }),
+
+  update: protectedProcedure
+    .input(zUpdateGroup)
+    .mutation(({ input: { id, projectId, ...data } }) => {
+      return updateGroup(id, projectId, data);
+    }),
+
+  delete: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .mutation(({ input: { id, projectId } }) => {
+      return deleteGroup(id, projectId);
+    }),
+
+  types: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(({ input: { projectId } }) => {
+      return getGroupTypes(projectId);
+    }),
+
+  metrics: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .query(async ({ input: { id, projectId } }) => {
+      const data = await chQuery<{
+        totalEvents: number;
+        uniqueProfiles: number;
+        firstSeen: string;
+        lastSeen: string;
+      }>(`
+        SELECT
+          count() AS totalEvents,
+          uniqExact(profile_id) AS uniqueProfiles,
+          min(created_at) AS firstSeen,
+          max(created_at) AS lastSeen
+        FROM ${TABLE_NAMES.events}
+        WHERE project_id = ${sqlstring.escape(projectId)}
+          AND has(groups, ${sqlstring.escape(id)})
+      `);
+
+      return {
+        totalEvents: data[0]?.totalEvents ?? 0,
+        uniqueProfiles: data[0]?.uniqueProfiles ?? 0,
+        firstSeen: toNullIfDefaultMinDate(data[0]?.firstSeen),
+        lastSeen: toNullIfDefaultMinDate(data[0]?.lastSeen),
+      };
+    }),
+
+  activity: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .query(({ input: { id, projectId } }) => {
+      return chQuery<{ count: number; date: string }>(`
+        SELECT count() AS count, toStartOfDay(created_at) AS date
+        FROM ${TABLE_NAMES.events}
+        WHERE project_id = ${sqlstring.escape(projectId)}
+          AND has(groups, ${sqlstring.escape(id)})
+        GROUP BY date
+        ORDER BY date DESC
+      `);
+    }),
+
+  members: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .query(({ input: { id, projectId } }) => {
+      return chQuery<{
+        profileId: string;
+        lastSeen: string;
+        eventCount: number;
+      }>(`
+        SELECT
+          profile_id AS profileId,
+          max(created_at) AS lastSeen,
+          count() AS eventCount
+        FROM ${TABLE_NAMES.events}
+        WHERE project_id = ${sqlstring.escape(projectId)}
+          AND has(groups, ${sqlstring.escape(id)})
+          AND profile_id != device_id
+        GROUP BY profile_id
+        ORDER BY lastSeen DESC, eventCount DESC
+        LIMIT 50
+      `);
+    }),
+
+  listProfiles: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        groupId: z.string(),
+        cursor: z.number().optional(),
+        take: z.number().default(50),
+        search: z.string().optional(),
+      })
+    )
+    .query(async ({ input }) => {
+      const { data, count } = await getGroupMemberProfiles({
+        projectId: input.projectId,
+        groupId: input.groupId,
+        cursor: input.cursor,
+        take: input.take,
+        search: input.search,
+      });
+      return {
+        data,
+        meta: { count, pageCount: input.take },
+      };
+    }),
+
+  mostEvents: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .query(({ input: { id, projectId } }) => {
+      return chQuery<{ count: number; name: string }>(`
+        SELECT count() as count, name
+        FROM ${TABLE_NAMES.events}
+        WHERE project_id = ${sqlstring.escape(projectId)}
+          AND has(groups, ${sqlstring.escape(id)})
+          AND name NOT IN ('screen_view', 'session_start', 'session_end')
+        GROUP BY name
+        ORDER BY count DESC
+        LIMIT 10
+      `);
+    }),
+
+  popularRoutes: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .query(({ input: { id, projectId } }) => {
+      return chQuery<{ count: number; path: string }>(`
+        SELECT count() as count, path
+        FROM ${TABLE_NAMES.events}
+        WHERE project_id = ${sqlstring.escape(projectId)}
+          AND has(groups, ${sqlstring.escape(id)})
+          AND name = 'screen_view'
+        GROUP BY path
+        ORDER BY count DESC
+        LIMIT 10
+      `);
+    }),
+
+  memberGrowth: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .query(({ input: { id, projectId } }) => {
+      return chQuery<{ date: string; count: number }>(`
+        SELECT
+          toDate(toStartOfDay(min_date)) AS date,
+          count() AS count
+        FROM (
+          SELECT profile_id, min(created_at) AS min_date
+          FROM ${TABLE_NAMES.events}
+          WHERE project_id = ${sqlstring.escape(projectId)}
+            AND has(groups, ${sqlstring.escape(id)})
+            AND profile_id != device_id
+            AND created_at >= now() - INTERVAL 30 DAY
+          GROUP BY profile_id
+        )
+        GROUP BY date
+        ORDER BY date ASC WITH FILL
+          FROM toDate(now() - INTERVAL 29 DAY)
+          TO toDate(now() + INTERVAL 1 DAY)
+          STEP 1
+      `);
+    }),
+
+  properties: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(({ input: { projectId } }) => {
+      return getGroupPropertyKeys(projectId);
+    }),
+
+  listByIds: protectedProcedure
+    .input(z.object({ projectId: z.string(), ids: z.array(z.string()) }))
+    .query(({ input: { projectId, ids } }) => {
+      return getGroupsByIds(projectId, ids);
+    }),
+});