diff --git a/apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx b/apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx index 9bda730c..e33bb3f9 100644 --- a/apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx +++ b/apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx @@ -29,7 +29,7 @@ interface PreviousDiffIndicatorProps { children?: React.ReactNode; inverted?: boolean; className?: string; - size?: 'sm' | 'lg' | 'md'; + size?: 'sm' | 'lg'; } export function PreviousDiffIndicator({ @@ -80,7 +80,6 @@ export function PreviousDiffIndicator({ 'flex size-4 items-center justify-center rounded-full', variant, size === 'lg' && 'size-8', - size === 'md' && 'size-6', )} > {renderIcon()} diff --git a/apps/dashboard/src/components/report-chart/funnel/chart.tsx b/apps/dashboard/src/components/report-chart/funnel/chart.tsx index e09215bd..1502a36a 100644 --- a/apps/dashboard/src/components/report-chart/funnel/chart.tsx +++ b/apps/dashboard/src/components/report-chart/funnel/chart.tsx @@ -39,10 +39,15 @@ export function Chart({ const { isEditMode } = useReportChartContext(); const mostDropoffs = findMostDropoffs(steps); const lastStep = last(steps)!; - const prevLastStep = previous?.steps ? last(previous.steps) : null; + const prevLastStep = last(previous.steps); return ( -
+
-
+
} /> } /> } />
+
+ {steps.map((step) => { + return ( +
+
+
+ ); + })} +
-
+
{steps.map((step, index) => { const percent = (step.count / totalSessions) * 100; const isMostDropoffs = mostDropoffs.event.id === step.event.id; return (
@@ -101,27 +123,27 @@ export function Chart({
{step.event.displayName}
-
+
Last period:{' '} - {previous?.steps?.[index]?.previousCount} + {previous.steps[index]?.previousCount}
} > -
+
Total: @@ -133,26 +155,26 @@ export function Chart({
Last period:{' '} - {previous?.steps?.[index]?.dropoffCount} + {previous.steps[index]?.dropoffCount}
} > -
+
Dropoff: @@ -170,25 +192,25 @@ export function Chart({
Last period:{' '} - {previous?.steps?.[index]?.count} + {previous.steps[index]?.count}
} > -
+
Current: @@ -197,42 +219,12 @@ export function Chart({
- - - Last period:{' '} - - {previous?.steps?.[index]?.count} - - - -
- } - > -
- - Percent: - -
- - {Number.isNaN(percent) ? 0 : round(percent, 2)}% - -
-
-
+
{label}
-
+
{value}
{enhancer}
diff --git a/apps/dashboard/src/components/report/ReportChartType.tsx b/apps/dashboard/src/components/report/ReportChartType.tsx index 8e7a5fb9..ad7e48cc 100644 --- a/apps/dashboard/src/components/report/ReportChartType.tsx +++ b/apps/dashboard/src/components/report/ReportChartType.tsx @@ -1,32 +1,10 @@ import { useDispatch, useSelector } from '@/redux'; -import { - AreaChartIcon, - ChartBarIcon, - ChartColumnIncreasingIcon, - ConeIcon, - GaugeIcon, - Globe2Icon, - LineChartIcon, - type LucideIcon, - PieChartIcon, - UsersIcon, -} from 'lucide-react'; +import { LineChartIcon } from 'lucide-react'; import { chartTypes } from '@openpanel/constants'; import { objectToZodEnums } from '@openpanel/validation'; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuGroup, - DropdownMenuItem, - DropdownMenuLabel, - DropdownMenuSeparator, - DropdownMenuShortcut, - DropdownMenuTrigger, -} from '@/components/ui/dropdown-menu'; -import { cn } from '@/utils/cn'; -import { Button } from '../ui/button'; +import { Combobox } from '../ui/combobox'; import { changeChartType } from './reportSlice'; interface ReportChartTypeProps { @@ -35,57 +13,20 @@ interface ReportChartTypeProps { export function ReportChartType({ className }: ReportChartTypeProps) { const dispatch = useDispatch(); const type = useSelector((state) => state.report.chartType); - const items = objectToZodEnums(chartTypes).map((key) => ({ - label: chartTypes[key], - value: key, - })); - - const Icons: Record = { - area: AreaChartIcon, - bar: ChartBarIcon, - pie: PieChartIcon, - funnel: ((props) => ( - - )) as LucideIcon, - histogram: ChartColumnIncreasingIcon, - linear: LineChartIcon, - metric: GaugeIcon, - retention: UsersIcon, - map: Globe2Icon, - }; return ( - - - - - - Available charts - - - - {items.map((item) => { - const Icon = Icons[item.value]; - return ( - dispatch(changeChartType(item.value))} - > - {item.label} - - - - - ); - })} - - - + { + dispatch(changeChartType(value)); + }} + value={type} + items={objectToZodEnums(chartTypes).map((key) => ({ + label: chartTypes[key], + value: key, + }))} + /> ); } diff --git a/apps/dashboard/src/components/tooltip-complete.tsx b/apps/dashboard/src/components/tooltip-complete.tsx index f17d2397..20022a8c 100644 --- a/apps/dashboard/src/components/tooltip-complete.tsx +++ b/apps/dashboard/src/components/tooltip-complete.tsx @@ -1,4 +1,3 @@ -import { TooltipPortal } from '@radix-ui/react-tooltip'; import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip'; interface TooltipCompleteProps { @@ -16,17 +15,12 @@ export function TooltipComplete({ }: TooltipCompleteProps) { return ( - + {children} - - - {content} - - + + {content} + ); } diff --git a/apps/dashboard/src/components/ui/progress.tsx b/apps/dashboard/src/components/ui/progress.tsx index 34bfc144..7aa20a5e 100644 --- a/apps/dashboard/src/components/ui/progress.tsx +++ b/apps/dashboard/src/components/ui/progress.tsx @@ -30,7 +30,11 @@ const Progress = React.forwardRef< }} /> {value && size !== 'sm' && ( -
+
{round(value, 2)}%
)} diff --git a/packages/db/index.ts b/packages/db/index.ts index 75c4af23..05990e53 100644 --- a/packages/db/index.ts +++ b/packages/db/index.ts @@ -16,6 +16,5 @@ export * from './src/services/reference.service'; export * from './src/services/id.service'; export * from './src/services/retention.service'; export * from './src/services/notification.service'; -export * from './src/services/funnel.service'; export * from './src/buffers'; export * from './src/types'; diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts index e397a7cd..62e0e53b 100644 --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -245,30 +245,30 @@ export async function getEvents( ): Promise { const events = await chQuery(sql); const projectId = events[0]?.project_id; - const [meta, profiles] = await Promise.all([ - options.meta && projectId - ? db.eventMeta.findMany({ - where: { - name: { - in: uniq(events.map((e) => e.name)), - }, - }, - }) - : null, - options.profile && projectId - ? getProfiles(uniq(events.map((e) => e.profile_id)), projectId) - : null, - ]); + if (options.profile && projectId) { + const ids = events.map((e) => e.profile_id); + const profiles = await getProfiles(ids, projectId); - for (const event of events) { - if (profiles) { + for (const event of events) { event.profile = profiles.find((p) => p.id === event.profile_id); } - if (meta) { - event.meta = meta.find((m) => m.name === event.name); - } } + if (options.meta && projectId) { + const names = uniq(events.map((e) => e.name)); + const metas = await db.eventMeta.findMany({ + where: { + name: { + in: names, + }, + projectId, + }, + select: options.meta === true ? undefined : options.meta, + }); + for (const event of events) { + event.meta = metas.find((m) => m.name === event.name); + } + } return events.map(transformEvent); } @@ -477,7 +477,7 @@ export async function getEventList({ } if (profileId) { - sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND device_id != '' AND profile_id = ${escape(profileId)} group by did) OR profile_id = ${escape(profileId)})`; + sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE device_id != '' AND profile_id = ${escape(profileId)} group by did) OR profile_id = ${escape(profileId)})`; } if (startDate && endDate) { diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.ts deleted file mode 100644 index 044f6f0c..00000000 --- a/packages/db/src/services/funnel.service.ts +++ /dev/null @@ -1,200 +0,0 @@ -import type { IChartEvent, IChartInput } from '@openpanel/validation'; -import { escape } from 'sqlstring'; -import { - TABLE_NAMES, - chQuery, - formatClickhouseDate, -} from '../clickhouse-client'; -import { createSqlBuilder } from '../sql-builder'; -import { getEventFiltersWhereClause } from './chart.service'; - -interface FunnelStep { - event: IChartEvent & { displayName: string }; - count: number; - percent: number; - dropoffCount: number; - dropoffPercent: number; - previousCount: number; -} - -interface FunnelResult { - totalSessions: number; - steps: FunnelStep[]; -} - -interface RawFunnelData { - level: number; - count: number; -} - -interface StepMetrics { - currentStep: number; - currentCount: number; - previousCount: number; - totalUsers: number; -} - -// Main function -export async function getFunnelData({ - projectId, - startDate, - endDate, - ...payload -}: IChartInput): Promise { - if (!startDate || !endDate) { - throw new Error('startDate and endDate are required'); - } - - if (payload.events.length === 0) { - return { totalSessions: 0, steps: [] }; - } - - const funnelWindow = (payload.funnelWindow || 24) * 3600; - const funnelGroup = payload.funnelGroup || 'session_id'; - - const sql = buildFunnelQuery( - payload.events, - projectId, - startDate, - endDate, - funnelWindow, - funnelGroup, - ); - - return await chQuery(sql) - .then((funnel) => fillFunnel(funnel, payload.events.length)) - .then((funnel) => ({ - totalSessions: funnel[0]?.count ?? 0, - steps: calculateStepMetrics( - funnel, - payload.events, - funnel[0]?.count ?? 0, - ), - })); -} - -// Helper functions -function buildFunnelQuery( - events: IChartEvent[], - projectId: string, - startDate: string, - endDate: string, - funnelWindow: number, - funnelGroup: string, -): string { - const funnelConditions = events.map((event) => { - const { sb, getWhere } = createSqlBuilder(); - sb.where = getEventFiltersWhereClause(event.filters); - sb.where.name = `name = ${escape(event.name)}`; - return getWhere().replace('WHERE ', ''); - }); - - const innerSql = ` - SELECT - sp.${funnelGroup}, - windowFunnel(${funnelWindow}, 'strict_increase')( - toUnixTimestamp(created_at), - ${funnelConditions.join(', ')} - ) AS level - FROM ${TABLE_NAMES.events} - LEFT JOIN ( - SELECT - session_id, - any(profile_id) AS profile_id - FROM ${TABLE_NAMES.events} - WHERE project_id = ${escape(projectId)} - AND created_at >= '${formatClickhouseDate(startDate)}' - AND created_at <= '${formatClickhouseDate(endDate)}' - GROUP BY session_id - HAVING profile_id IS NOT NULL - ) AS sp ON session_id = sp.session_id - WHERE - project_id = ${escape(projectId)} AND - created_at >= '${formatClickhouseDate(startDate)}' AND - created_at <= '${formatClickhouseDate(endDate)}' AND - name IN (${events.map((event) => escape(event.name)).join(', ')}) - GROUP BY sp.${funnelGroup} - `; - - const sql = ` - SELECT - level, - count() AS count - FROM (${innerSql}) - WHERE level != 0 - GROUP BY level - ORDER BY level DESC`; - - return sql; -} - -function calculateStepMetrics( - funnelData: RawFunnelData[], - events: IChartEvent[], - totalSessions: number, -): FunnelStep[] { - return funnelData - .sort((a, b) => a.level - b.level) // Ensure steps are in order - .map((data, index, array): FunnelStep => { - const metrics: StepMetrics = { - currentStep: data.level, - currentCount: data.count, - previousCount: index === 0 ? totalSessions : array[index - 1]!.count, - totalUsers: totalSessions, - }; - - const event = events[data.level - 1]!; - - return { - event: { - ...event, - displayName: event.displayName ?? event.name, - }, - count: metrics.currentCount, - percent: calculatePercent(metrics.currentCount, metrics.totalUsers), - dropoffCount: calculateDropoff(metrics), - dropoffPercent: calculateDropoffPercent(metrics), - previousCount: metrics.previousCount, - }; - }); -} - -function calculatePercent(count: number, total: number): number { - return (count / total) * 100; -} - -function calculateDropoff({ - currentCount, - previousCount, -}: StepMetrics): number { - return previousCount - currentCount; -} - -function calculateDropoffPercent({ - currentCount, - previousCount, -}: StepMetrics): number { - return 100 - (currentCount / previousCount) * 100; -} - -function fillFunnel(funnel: RawFunnelData[], steps: number): RawFunnelData[] { - const filled = Array.from({ length: steps }, (_, index) => { - const level = index + 1; - const matchingResult = funnel.find((res) => res.level === level); - return { - level, - count: matchingResult ? matchingResult.count : 0, - }; - }); - - // Accumulate counts from top to bottom of the funnel - for (let i = filled.length - 1; i >= 0; i--) { - const step = filled[i]!; - const prevStep = filled[i + 1]; - if (prevStep) { - step.count += prevStep.count; - } - } - - return filled; -} diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts index 003d228b..5d608a18 100644 --- a/packages/trpc/src/routers/chart.helpers.ts +++ b/packages/trpc/src/routers/chart.helpers.ts @@ -13,9 +13,9 @@ import { subYears, } from 'date-fns'; import * as mathjs from 'mathjs'; -import { pluck, uniq } from 'ramda'; +import { last, pluck, repeat, reverse, uniq } from 'ramda'; +import { escape } from 'sqlstring'; -import type { ISerieDataItem } from '@openpanel/common'; import { average, completeSerie, @@ -26,8 +26,17 @@ import { slug, sum, } from '@openpanel/common'; +import type { ISerieDataItem } from '@openpanel/common'; import { alphabetIds } from '@openpanel/constants'; -import { chQuery, getChartSql } from '@openpanel/db'; +import { + TABLE_NAMES, + chQuery, + createSqlBuilder, + formatClickhouseDate, + getChartSql, + getEventFiltersWhereClause, + getProfiles, +} from '@openpanel/db'; import type { FinalChart, IChartEvent, @@ -232,6 +241,28 @@ export function getDatesFromRange(range: IChartRange) { }; } +function fillFunnel(funnel: { level: number; count: number }[], steps: number) { + const filled = Array.from({ length: steps }, (_, index) => { + const level = index + 1; + const matchingResult = funnel.find((res) => res.level === level); + return { + level, + count: matchingResult ? matchingResult.count : 0, + }; + }); + + // Accumulate counts from top to bottom of the funnel + for (let i = filled.length - 1; i >= 0; i--) { + const step = filled[i]; + const prevStep = filled[i + 1]; + // If there's a previous step, add the count to the current step + if (step && prevStep) { + step.count += prevStep.count; + } + } + return filled.reverse(); +} + export function getChartStartEndDate({ startDate, endDate, @@ -257,6 +288,147 @@ export function getChartPrevStartEndDate({ }; } +export async function getFunnelData({ + projectId, + startDate, + endDate, + ...payload +}: IChartInput) { + const funnelWindow = (payload.funnelWindow || 24) * 3600; + const funnelGroup = payload.funnelGroup || 'session_id'; + + if (!startDate || !endDate) { + throw new Error('startDate and endDate are required'); + } + + if (payload.events.length === 0) { + return { + totalSessions: 0, + steps: [], + }; + } + + const funnels = payload.events.map((event) => { + const { sb, getWhere } = createSqlBuilder(); + sb.where = getEventFiltersWhereClause(event.filters); + sb.where.name = `name = ${escape(event.name)}`; + return getWhere().replace('WHERE ', ''); + }); + + const innerSql = `SELECT + ${funnelGroup}, + windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level + FROM ${TABLE_NAMES.events} + WHERE + project_id = ${escape(projectId)} AND + created_at >= '${formatClickhouseDate(startDate)}' AND + created_at <= '${formatClickhouseDate(endDate)}' AND + name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) + GROUP BY ${funnelGroup}`; + + const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`; + + const funnel = await chQuery<{ level: number; count: number }>(sql); + const maxLevel = payload.events.length; + const filledFunnelRes = fillFunnel(funnel, maxLevel); + + const totalSessions = last(filledFunnelRes)?.count ?? 0; + const steps = reverse(filledFunnelRes).reduce( + (acc, item, index, list) => { + const prev = list[index - 1] ?? { count: totalSessions }; + const event = payload.events[item.level - 1]!; + return [ + ...acc, + { + event: { + ...event, + displayName: event.displayName ?? event.name, + }, + count: item.count, + percent: (item.count / totalSessions) * 100, + dropoffCount: prev.count - item.count, + dropoffPercent: 100 - (item.count / prev.count) * 100, + previousCount: prev.count, + }, + ]; + }, + [] as { + event: IChartEvent & { displayName: string }; + count: number; + percent: number; + dropoffCount: number; + dropoffPercent: number; + previousCount: number; + }[], + ); + + return { + totalSessions, + steps, + }; +} + +export async function getFunnelStep({ + projectId, + startDate, + endDate, + step, + ...payload +}: IChartInput & { + step: number; +}) { + throw new Error('not implemented'); + // if (!startDate || !endDate) { + // throw new Error('startDate and endDate are required'); + // } + + // if (payload.events.length === 0) { + // throw new Error('no events selected'); + // } + + // const funnels = payload.events.map((event) => { + // const { sb, getWhere } = createSqlBuilder(); + // sb.where = getEventFiltersWhereClause(event.filters); + // sb.where.name = `name = ${escape(event.name)}`; + // return getWhere().replace('WHERE ', ''); + // }); + + // const innerSql = `SELECT + // session_id, + // windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level + // FROM ${TABLE_NAMES.events} + // WHERE + // project_id = ${escape(projectId)} AND + // created_at >= '${formatClickhouseDate(startDate)}' AND + // created_at <= '${formatClickhouseDate(endDate)}' AND + // name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) + // GROUP BY session_id`; + + // const profileIdsQuery = `WITH sessions AS (${innerSql}) + // SELECT + // DISTINCT e.profile_id as id + // FROM sessions s + // JOIN ${TABLE_NAMES.events} e ON s.session_id = e.session_id + // WHERE + // s.level = ${step} AND + // e.project_id = ${escape(projectId)} AND + // e.created_at >= '${formatClickhouseDate(startDate)}' AND + // e.created_at <= '${formatClickhouseDate(endDate)}' AND + // name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) + // ORDER BY e.created_at DESC + // LIMIT 500 + // `; + + // const res = await chQuery<{ + // id: string; + // }>(profileIdsQuery); + + // return getProfiles( + // res.map((r) => r.id), + // projectId, + // ); +} + export async function getChartSerie(payload: IGetChartDataInput) { async function getSeries() { const result = await chQuery(getChartSql(payload)); diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index 9ed18b79..80227a4d 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -7,7 +7,6 @@ import { chQuery, createSqlBuilder, db, - getFunnelData, getSelectPropertyKey, toDate, } from '@openpanel/db'; @@ -32,6 +31,8 @@ import { getChart, getChartPrevStartEndDate, getChartStartEndDate, + getFunnelData, + getFunnelStep, } from './chart.helpers'; function utc(date: string | Date) { @@ -86,12 +87,9 @@ export const chartRouter = createTRPCRouter({ .map((item) => item.replace(/\.([0-9]+)/g, '[*]')) .map((item) => `properties.${item}`); - if (event === '*') { - properties.push('name'); - } - properties.push( 'has_profile', + 'name', 'path', 'origin', 'referrer', @@ -186,9 +184,7 @@ export const chartRouter = createTRPCRouter({ const [current, previous] = await Promise.all([ getFunnelData({ ...input, ...currentPeriod }), - input.previous - ? getFunnelData({ ...input, ...previousPeriod }) - : Promise.resolve(null), + getFunnelData({ ...input, ...previousPeriod }), ]); return { @@ -197,6 +193,17 @@ export const chartRouter = createTRPCRouter({ }; }), + funnelStep: protectedProcedure + .input( + zChartInput.extend({ + step: z.number(), + }), + ) + .query(async ({ input }) => { + const currentPeriod = getChartStartEndDate(input); + return getFunnelStep({ ...input, ...currentPeriod }); + }), + chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => { if (ctx.session.userId) { const access = await getProjectAccessCached({