diff --git a/apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx b/apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx index e33bb3f9..9bda730c 100644 --- a/apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx +++ b/apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx @@ -29,7 +29,7 @@ interface PreviousDiffIndicatorProps { children?: React.ReactNode; inverted?: boolean; className?: string; - size?: 'sm' | 'lg'; + size?: 'sm' | 'lg' | 'md'; } export function PreviousDiffIndicator({ @@ -80,6 +80,7 @@ export function PreviousDiffIndicator({ 'flex size-4 items-center justify-center rounded-full', variant, size === 'lg' && 'size-8', + size === 'md' && 'size-6', )} > {renderIcon()} diff --git a/apps/dashboard/src/components/report-chart/funnel/chart.tsx b/apps/dashboard/src/components/report-chart/funnel/chart.tsx index 1502a36a..e09215bd 100644 --- a/apps/dashboard/src/components/report-chart/funnel/chart.tsx +++ b/apps/dashboard/src/components/report-chart/funnel/chart.tsx @@ -39,15 +39,10 @@ export function Chart({ const { isEditMode } = useReportChartContext(); const mostDropoffs = findMostDropoffs(steps); const lastStep = last(steps)!; - const prevLastStep = last(previous.steps); + const prevLastStep = previous?.steps ? last(previous.steps) : null; return ( -
+
-
+
} /> } /> } />
-
- {steps.map((step) => { - return ( -
-
-
- ); - })} -
-
+
{steps.map((step, index) => { const percent = (step.count / totalSessions) * 100; const isMostDropoffs = mostDropoffs.event.id === step.event.id; return (
@@ -123,27 +101,27 @@ export function Chart({
{step.event.displayName}
-
+
Last period:{' '} - {previous.steps[index]?.previousCount} + {previous?.steps?.[index]?.previousCount}
} > -
+
Total: @@ -155,26 +133,26 @@ export function Chart({
Last period:{' '} - {previous.steps[index]?.dropoffCount} + {previous?.steps?.[index]?.dropoffCount}
} > -
+
Dropoff: @@ -192,25 +170,25 @@ export function Chart({
Last period:{' '} - {previous.steps[index]?.count} + {previous?.steps?.[index]?.count}
} > -
+
Current: @@ -219,12 +197,42 @@ export function Chart({
+ + + Last period:{' '} + + {previous?.steps?.[index]?.count} + + + +
+ } + > +
+ + Percent: + +
+ + {Number.isNaN(percent) ? 0 : round(percent, 2)}% + +
+
+
+
{label}
-
+
{value}
{enhancer}
diff --git a/apps/dashboard/src/components/report/ReportChartType.tsx b/apps/dashboard/src/components/report/ReportChartType.tsx index ad7e48cc..8e7a5fb9 100644 --- a/apps/dashboard/src/components/report/ReportChartType.tsx +++ b/apps/dashboard/src/components/report/ReportChartType.tsx @@ -1,10 +1,32 @@ import { useDispatch, useSelector } from '@/redux'; -import { LineChartIcon } from 'lucide-react'; +import { + AreaChartIcon, + ChartBarIcon, + ChartColumnIncreasingIcon, + ConeIcon, + GaugeIcon, + Globe2Icon, + LineChartIcon, + type LucideIcon, + PieChartIcon, + UsersIcon, +} from 'lucide-react'; import { chartTypes } from '@openpanel/constants'; import { objectToZodEnums } from '@openpanel/validation'; -import { Combobox } from '../ui/combobox'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuGroup, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuShortcut, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; +import { cn } from '@/utils/cn'; +import { Button } from '../ui/button'; import { changeChartType } from './reportSlice'; interface ReportChartTypeProps { @@ -13,20 +35,57 @@ interface ReportChartTypeProps { export function ReportChartType({ className }: ReportChartTypeProps) { const dispatch = useDispatch(); const type = useSelector((state) => state.report.chartType); + const items = objectToZodEnums(chartTypes).map((key) => ({ + label: chartTypes[key], + value: key, + })); + + const Icons: Record = { + area: AreaChartIcon, + bar: ChartBarIcon, + pie: PieChartIcon, + funnel: ((props) => ( + + )) as LucideIcon, + histogram: ChartColumnIncreasingIcon, + linear: LineChartIcon, + metric: GaugeIcon, + retention: UsersIcon, + map: Globe2Icon, + }; return ( - { - dispatch(changeChartType(value)); - }} - value={type} - items={objectToZodEnums(chartTypes).map((key) => ({ - label: chartTypes[key], - value: key, - }))} - /> + + + + + + Available charts + + + + {items.map((item) => { + const Icon = Icons[item.value]; + return ( + dispatch(changeChartType(item.value))} + > + {item.label} + + + + + ); + })} + + + ); } diff --git a/apps/dashboard/src/components/tooltip-complete.tsx b/apps/dashboard/src/components/tooltip-complete.tsx index 20022a8c..f17d2397 100644 --- a/apps/dashboard/src/components/tooltip-complete.tsx +++ b/apps/dashboard/src/components/tooltip-complete.tsx @@ -1,3 +1,4 @@ +import { TooltipPortal } from '@radix-ui/react-tooltip'; import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip'; interface TooltipCompleteProps { @@ -15,12 +16,17 @@ export function TooltipComplete({ }: TooltipCompleteProps) { return ( - + {children} - - {content} - + + + {content} + + ); } diff --git a/apps/dashboard/src/components/ui/progress.tsx b/apps/dashboard/src/components/ui/progress.tsx index 7aa20a5e..34bfc144 100644 --- a/apps/dashboard/src/components/ui/progress.tsx +++ b/apps/dashboard/src/components/ui/progress.tsx @@ -30,11 +30,7 @@ const Progress = React.forwardRef< }} /> {value && size !== 'sm' && ( -
+
{round(value, 2)}%
)} diff --git a/packages/db/index.ts b/packages/db/index.ts index 05990e53..75c4af23 100644 --- a/packages/db/index.ts +++ b/packages/db/index.ts @@ -16,5 +16,6 @@ export * from './src/services/reference.service'; export * from './src/services/id.service'; export * from './src/services/retention.service'; export * from './src/services/notification.service'; +export * from './src/services/funnel.service'; export * from './src/buffers'; export * from './src/types'; diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts index 62e0e53b..e397a7cd 100644 --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -245,30 +245,30 @@ export async function getEvents( ): Promise { const events = await chQuery(sql); const projectId = events[0]?.project_id; - if (options.profile && projectId) { - const ids = events.map((e) => e.profile_id); - const profiles = await getProfiles(ids, projectId); + const [meta, profiles] = await Promise.all([ + options.meta && projectId + ? db.eventMeta.findMany({ + where: { + name: { + in: uniq(events.map((e) => e.name)), + }, + }, + }) + : null, + options.profile && projectId + ? getProfiles(uniq(events.map((e) => e.profile_id)), projectId) + : null, + ]); - for (const event of events) { + for (const event of events) { + if (profiles) { event.profile = profiles.find((p) => p.id === event.profile_id); } - } - - if (options.meta && projectId) { - const names = uniq(events.map((e) => e.name)); - const metas = await db.eventMeta.findMany({ - where: { - name: { - in: names, - }, - projectId, - }, - select: options.meta === true ? undefined : options.meta, - }); - for (const event of events) { - event.meta = metas.find((m) => m.name === event.name); + if (meta) { + event.meta = meta.find((m) => m.name === event.name); } } + return events.map(transformEvent); } @@ -477,7 +477,7 @@ export async function getEventList({ } if (profileId) { - sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE device_id != '' AND profile_id = ${escape(profileId)} group by did) OR profile_id = ${escape(profileId)})`; + sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND device_id != '' AND profile_id = ${escape(profileId)} group by did) OR profile_id = ${escape(profileId)})`; } if (startDate && endDate) { diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.ts new file mode 100644 index 00000000..044f6f0c --- /dev/null +++ b/packages/db/src/services/funnel.service.ts @@ -0,0 +1,200 @@ +import type { IChartEvent, IChartInput } from '@openpanel/validation'; +import { escape } from 'sqlstring'; +import { + TABLE_NAMES, + chQuery, + formatClickhouseDate, +} from '../clickhouse-client'; +import { createSqlBuilder } from '../sql-builder'; +import { getEventFiltersWhereClause } from './chart.service'; + +interface FunnelStep { + event: IChartEvent & { displayName: string }; + count: number; + percent: number; + dropoffCount: number; + dropoffPercent: number; + previousCount: number; +} + +interface FunnelResult { + totalSessions: number; + steps: FunnelStep[]; +} + +interface RawFunnelData { + level: number; + count: number; +} + +interface StepMetrics { + currentStep: number; + currentCount: number; + previousCount: number; + totalUsers: number; +} + +// Main function +export async function getFunnelData({ + projectId, + startDate, + endDate, + ...payload +}: IChartInput): Promise { + if (!startDate || !endDate) { + throw new Error('startDate and endDate are required'); + } + + if (payload.events.length === 0) { + return { totalSessions: 0, steps: [] }; + } + + const funnelWindow = (payload.funnelWindow || 24) * 3600; + const funnelGroup = payload.funnelGroup || 'session_id'; + + const sql = buildFunnelQuery( + payload.events, + projectId, + startDate, + endDate, + funnelWindow, + funnelGroup, + ); + + return await chQuery(sql) + .then((funnel) => fillFunnel(funnel, payload.events.length)) + .then((funnel) => ({ + totalSessions: funnel[0]?.count ?? 0, + steps: calculateStepMetrics( + funnel, + payload.events, + funnel[0]?.count ?? 0, + ), + })); +} + +// Helper functions +function buildFunnelQuery( + events: IChartEvent[], + projectId: string, + startDate: string, + endDate: string, + funnelWindow: number, + funnelGroup: string, +): string { + const funnelConditions = events.map((event) => { + const { sb, getWhere } = createSqlBuilder(); + sb.where = getEventFiltersWhereClause(event.filters); + sb.where.name = `name = ${escape(event.name)}`; + return getWhere().replace('WHERE ', ''); + }); + + const innerSql = ` + SELECT + sp.${funnelGroup}, + windowFunnel(${funnelWindow}, 'strict_increase')( + toUnixTimestamp(created_at), + ${funnelConditions.join(', ')} + ) AS level + FROM ${TABLE_NAMES.events} + LEFT JOIN ( + SELECT + session_id, + any(profile_id) AS profile_id + FROM ${TABLE_NAMES.events} + WHERE project_id = ${escape(projectId)} + AND created_at >= '${formatClickhouseDate(startDate)}' + AND created_at <= '${formatClickhouseDate(endDate)}' + GROUP BY session_id + HAVING profile_id IS NOT NULL + ) AS sp ON session_id = sp.session_id + WHERE + project_id = ${escape(projectId)} AND + created_at >= '${formatClickhouseDate(startDate)}' AND + created_at <= '${formatClickhouseDate(endDate)}' AND + name IN (${events.map((event) => escape(event.name)).join(', ')}) + GROUP BY sp.${funnelGroup} + `; + + const sql = ` + SELECT + level, + count() AS count + FROM (${innerSql}) + WHERE level != 0 + GROUP BY level + ORDER BY level DESC`; + + return sql; +} + +function calculateStepMetrics( + funnelData: RawFunnelData[], + events: IChartEvent[], + totalSessions: number, +): FunnelStep[] { + return funnelData + .sort((a, b) => a.level - b.level) // Ensure steps are in order + .map((data, index, array): FunnelStep => { + const metrics: StepMetrics = { + currentStep: data.level, + currentCount: data.count, + previousCount: index === 0 ? totalSessions : array[index - 1]!.count, + totalUsers: totalSessions, + }; + + const event = events[data.level - 1]!; + + return { + event: { + ...event, + displayName: event.displayName ?? event.name, + }, + count: metrics.currentCount, + percent: calculatePercent(metrics.currentCount, metrics.totalUsers), + dropoffCount: calculateDropoff(metrics), + dropoffPercent: calculateDropoffPercent(metrics), + previousCount: metrics.previousCount, + }; + }); +} + +function calculatePercent(count: number, total: number): number { + return (count / total) * 100; +} + +function calculateDropoff({ + currentCount, + previousCount, +}: StepMetrics): number { + return previousCount - currentCount; +} + +function calculateDropoffPercent({ + currentCount, + previousCount, +}: StepMetrics): number { + return 100 - (currentCount / previousCount) * 100; +} + +function fillFunnel(funnel: RawFunnelData[], steps: number): RawFunnelData[] { + const filled = Array.from({ length: steps }, (_, index) => { + const level = index + 1; + const matchingResult = funnel.find((res) => res.level === level); + return { + level, + count: matchingResult ? matchingResult.count : 0, + }; + }); + + // Accumulate counts from top to bottom of the funnel + for (let i = filled.length - 1; i >= 0; i--) { + const step = filled[i]!; + const prevStep = filled[i + 1]; + if (prevStep) { + step.count += prevStep.count; + } + } + + return filled; +} diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts index 5d608a18..003d228b 100644 --- a/packages/trpc/src/routers/chart.helpers.ts +++ b/packages/trpc/src/routers/chart.helpers.ts @@ -13,9 +13,9 @@ import { subYears, } from 'date-fns'; import * as mathjs from 'mathjs'; -import { last, pluck, repeat, reverse, uniq } from 'ramda'; -import { escape } from 'sqlstring'; +import { pluck, uniq } from 'ramda'; +import type { ISerieDataItem } from '@openpanel/common'; import { average, completeSerie, @@ -26,17 +26,8 @@ import { slug, sum, } from '@openpanel/common'; -import type { ISerieDataItem } from '@openpanel/common'; import { alphabetIds } from '@openpanel/constants'; -import { - TABLE_NAMES, - chQuery, - createSqlBuilder, - formatClickhouseDate, - getChartSql, - getEventFiltersWhereClause, - getProfiles, -} from '@openpanel/db'; +import { chQuery, getChartSql } from '@openpanel/db'; import type { FinalChart, IChartEvent, @@ -241,28 +232,6 @@ export function getDatesFromRange(range: IChartRange) { }; } -function fillFunnel(funnel: { level: number; count: number }[], steps: number) { - const filled = Array.from({ length: steps }, (_, index) => { - const level = index + 1; - const matchingResult = funnel.find((res) => res.level === level); - return { - level, - count: matchingResult ? matchingResult.count : 0, - }; - }); - - // Accumulate counts from top to bottom of the funnel - for (let i = filled.length - 1; i >= 0; i--) { - const step = filled[i]; - const prevStep = filled[i + 1]; - // If there's a previous step, add the count to the current step - if (step && prevStep) { - step.count += prevStep.count; - } - } - return filled.reverse(); -} - export function getChartStartEndDate({ startDate, endDate, @@ -288,147 +257,6 @@ export function getChartPrevStartEndDate({ }; } -export async function getFunnelData({ - projectId, - startDate, - endDate, - ...payload -}: IChartInput) { - const funnelWindow = (payload.funnelWindow || 24) * 3600; - const funnelGroup = payload.funnelGroup || 'session_id'; - - if (!startDate || !endDate) { - throw new Error('startDate and endDate are required'); - } - - if (payload.events.length === 0) { - return { - totalSessions: 0, - steps: [], - }; - } - - const funnels = payload.events.map((event) => { - const { sb, getWhere } = createSqlBuilder(); - sb.where = getEventFiltersWhereClause(event.filters); - sb.where.name = `name = ${escape(event.name)}`; - return getWhere().replace('WHERE ', ''); - }); - - const innerSql = `SELECT - ${funnelGroup}, - windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level - FROM ${TABLE_NAMES.events} - WHERE - project_id = ${escape(projectId)} AND - created_at >= '${formatClickhouseDate(startDate)}' AND - created_at <= '${formatClickhouseDate(endDate)}' AND - name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) - GROUP BY ${funnelGroup}`; - - const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`; - - const funnel = await chQuery<{ level: number; count: number }>(sql); - const maxLevel = payload.events.length; - const filledFunnelRes = fillFunnel(funnel, maxLevel); - - const totalSessions = last(filledFunnelRes)?.count ?? 0; - const steps = reverse(filledFunnelRes).reduce( - (acc, item, index, list) => { - const prev = list[index - 1] ?? { count: totalSessions }; - const event = payload.events[item.level - 1]!; - return [ - ...acc, - { - event: { - ...event, - displayName: event.displayName ?? event.name, - }, - count: item.count, - percent: (item.count / totalSessions) * 100, - dropoffCount: prev.count - item.count, - dropoffPercent: 100 - (item.count / prev.count) * 100, - previousCount: prev.count, - }, - ]; - }, - [] as { - event: IChartEvent & { displayName: string }; - count: number; - percent: number; - dropoffCount: number; - dropoffPercent: number; - previousCount: number; - }[], - ); - - return { - totalSessions, - steps, - }; -} - -export async function getFunnelStep({ - projectId, - startDate, - endDate, - step, - ...payload -}: IChartInput & { - step: number; -}) { - throw new Error('not implemented'); - // if (!startDate || !endDate) { - // throw new Error('startDate and endDate are required'); - // } - - // if (payload.events.length === 0) { - // throw new Error('no events selected'); - // } - - // const funnels = payload.events.map((event) => { - // const { sb, getWhere } = createSqlBuilder(); - // sb.where = getEventFiltersWhereClause(event.filters); - // sb.where.name = `name = ${escape(event.name)}`; - // return getWhere().replace('WHERE ', ''); - // }); - - // const innerSql = `SELECT - // session_id, - // windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level - // FROM ${TABLE_NAMES.events} - // WHERE - // project_id = ${escape(projectId)} AND - // created_at >= '${formatClickhouseDate(startDate)}' AND - // created_at <= '${formatClickhouseDate(endDate)}' AND - // name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) - // GROUP BY session_id`; - - // const profileIdsQuery = `WITH sessions AS (${innerSql}) - // SELECT - // DISTINCT e.profile_id as id - // FROM sessions s - // JOIN ${TABLE_NAMES.events} e ON s.session_id = e.session_id - // WHERE - // s.level = ${step} AND - // e.project_id = ${escape(projectId)} AND - // e.created_at >= '${formatClickhouseDate(startDate)}' AND - // e.created_at <= '${formatClickhouseDate(endDate)}' AND - // name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) - // ORDER BY e.created_at DESC - // LIMIT 500 - // `; - - // const res = await chQuery<{ - // id: string; - // }>(profileIdsQuery); - - // return getProfiles( - // res.map((r) => r.id), - // projectId, - // ); -} - export async function getChartSerie(payload: IGetChartDataInput) { async function getSeries() { const result = await chQuery(getChartSql(payload)); diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index 80227a4d..9ed18b79 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -7,6 +7,7 @@ import { chQuery, createSqlBuilder, db, + getFunnelData, getSelectPropertyKey, toDate, } from '@openpanel/db'; @@ -31,8 +32,6 @@ import { getChart, getChartPrevStartEndDate, getChartStartEndDate, - getFunnelData, - getFunnelStep, } from './chart.helpers'; function utc(date: string | Date) { @@ -87,9 +86,12 @@ export const chartRouter = createTRPCRouter({ .map((item) => item.replace(/\.([0-9]+)/g, '[*]')) .map((item) => `properties.${item}`); + if (event === '*') { + properties.push('name'); + } + properties.push( 'has_profile', - 'name', 'path', 'origin', 'referrer', @@ -184,7 +186,9 @@ export const chartRouter = createTRPCRouter({ const [current, previous] = await Promise.all([ getFunnelData({ ...input, ...currentPeriod }), - getFunnelData({ ...input, ...previousPeriod }), + input.previous + ? getFunnelData({ ...input, ...previousPeriod }) + : Promise.resolve(null), ]); return { @@ -193,17 +197,6 @@ export const chartRouter = createTRPCRouter({ }; }), - funnelStep: protectedProcedure - .input( - zChartInput.extend({ - step: z.number(), - }), - ) - .query(async ({ input }) => { - const currentPeriod = getChartStartEndDate(input); - return getFunnelStep({ ...input, ...currentPeriod }); - }), - chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => { if (ctx.session.userId) { const access = await getProjectAccessCached({