zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Revert "improve(funnel): make sure group by profile id works as you would think"

Browse Source 
This reverts commit 56edb91dd0.
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-11-23 23:39:58 +01:00
parent 56edb91dd0
commit b0c8e25b0a
12 changed files with 285 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/db/index.ts
View File

@@ -16,6 +16,5 @@ export * from './src/services/reference.service';
export * from './src/services/id.service';
export * from './src/services/retention.service';
export * from './src/services/notification.service';
export * from './src/services/funnel.service';
export * from './src/buffers';
export * from './src/types';

40
packages/db/src/services/event.service.ts
View File

@@ -245,30 +245,30 @@ export async function getEvents(
): Promise<IServiceEvent[]> {
const events = await chQuery<IClickhouseEvent>(sql);
const projectId = events[0]?.project_id;
const [meta, profiles] = await Promise.all([
options.meta && projectId
? db.eventMeta.findMany({
where: {
name: {
in: uniq(events.map((e) => e.name)),
},
},
})
: null,
options.profile && projectId
? getProfiles(uniq(events.map((e) => e.profile_id)), projectId)
: null,
]);
if (options.profile && projectId) {
const ids = events.map((e) => e.profile_id);
const profiles = await getProfiles(ids, projectId);
for (const event of events) {
if (profiles) {
for (const event of events) {
event.profile = profiles.find((p) => p.id === event.profile_id);
}
if (meta) {
event.meta = meta.find((m) => m.name === event.name);
}
}
if (options.meta && projectId) {
const names = uniq(events.map((e) => e.name));
const metas = await db.eventMeta.findMany({
where: {
name: {
in: names,
},
projectId,
},
select: options.meta === true ? undefined : options.meta,
});
for (const event of events) {
event.meta = metas.find((m) => m.name === event.name);
}
}
return events.map(transformEvent);
}
@@ -477,7 +477,7 @@ export async function getEventList({
}
if (profileId) {
sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND device_id != '' AND profile_id = ${escape(profileId)} group by did) OR profile_id = ${escape(profileId)})`;
sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE device_id != '' AND profile_id = ${escape(profileId)} group by did) OR profile_id = ${escape(profileId)})`;
}
if (startDate && endDate) {

200
packages/db/src/services/funnel.service.ts
View File

@@ -1,200 +0,0 @@
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import { escape } from 'sqlstring';
import {
TABLE_NAMES,
chQuery,
formatClickhouseDate,
} from '../clickhouse-client';
import { createSqlBuilder } from '../sql-builder';
import { getEventFiltersWhereClause } from './chart.service';
interface FunnelStep {
event: IChartEvent & { displayName: string };
count: number;
percent: number;
dropoffCount: number;
dropoffPercent: number;
previousCount: number;
}
interface FunnelResult {
totalSessions: number;
steps: FunnelStep[];
}
interface RawFunnelData {
level: number;
count: number;
}
interface StepMetrics {
currentStep: number;
currentCount: number;
previousCount: number;
totalUsers: number;
}
// Main function
export async function getFunnelData({
projectId,
startDate,
endDate,
...payload
}: IChartInput): Promise<FunnelResult> {
if (!startDate || !endDate) {
throw new Error('startDate and endDate are required');
}
if (payload.events.length === 0) {
return { totalSessions: 0, steps: [] };
}
const funnelWindow = (payload.funnelWindow || 24) * 3600;
const funnelGroup = payload.funnelGroup || 'session_id';
const sql = buildFunnelQuery(
payload.events,
projectId,
startDate,
endDate,
funnelWindow,
funnelGroup,
);
return await chQuery<RawFunnelData>(sql)
.then((funnel) => fillFunnel(funnel, payload.events.length))
.then((funnel) => ({
totalSessions: funnel[0]?.count ?? 0,
steps: calculateStepMetrics(
funnel,
payload.events,
funnel[0]?.count ?? 0,
),
}));
}
// Helper functions
function buildFunnelQuery(
events: IChartEvent[],
projectId: string,
startDate: string,
endDate: string,
funnelWindow: number,
funnelGroup: string,
): string {
const funnelConditions = events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = ${escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
const innerSql = `
SELECT
sp.${funnelGroup},
windowFunnel(${funnelWindow}, 'strict_increase')(
toUnixTimestamp(created_at),
${funnelConditions.join(', ')}
) AS level
FROM ${TABLE_NAMES.events}
LEFT JOIN (
SELECT
session_id,
any(profile_id) AS profile_id
FROM ${TABLE_NAMES.events}
WHERE project_id = ${escape(projectId)}
AND created_at >= '${formatClickhouseDate(startDate)}'
AND created_at <= '${formatClickhouseDate(endDate)}'
GROUP BY session_id
HAVING profile_id IS NOT NULL
) AS sp ON session_id = sp.session_id
WHERE
project_id = ${escape(projectId)} AND
created_at >= '${formatClickhouseDate(startDate)}' AND
created_at <= '${formatClickhouseDate(endDate)}' AND
name IN (${events.map((event) => escape(event.name)).join(', ')})
GROUP BY sp.${funnelGroup}
`;
const sql = `
SELECT
level,
count() AS count
FROM (${innerSql})
WHERE level != 0
GROUP BY level
ORDER BY level DESC`;
return sql;
}
function calculateStepMetrics(
funnelData: RawFunnelData[],
events: IChartEvent[],
totalSessions: number,
): FunnelStep[] {
return funnelData
.sort((a, b) => a.level - b.level) // Ensure steps are in order
.map((data, index, array): FunnelStep => {
const metrics: StepMetrics = {
currentStep: data.level,
currentCount: data.count,
previousCount: index === 0 ? totalSessions : array[index - 1]!.count,
totalUsers: totalSessions,
};
const event = events[data.level - 1]!;
return {
event: {
...event,
displayName: event.displayName ?? event.name,
},
count: metrics.currentCount,
percent: calculatePercent(metrics.currentCount, metrics.totalUsers),
dropoffCount: calculateDropoff(metrics),
dropoffPercent: calculateDropoffPercent(metrics),
previousCount: metrics.previousCount,
};
});
}
function calculatePercent(count: number, total: number): number {
return (count / total) * 100;
}
function calculateDropoff({
currentCount,
previousCount,
}: StepMetrics): number {
return previousCount - currentCount;
}
function calculateDropoffPercent({
currentCount,
previousCount,
}: StepMetrics): number {
return 100 - (currentCount / previousCount) * 100;
}
function fillFunnel(funnel: RawFunnelData[], steps: number): RawFunnelData[] {
const filled = Array.from({ length: steps }, (_, index) => {
const level = index + 1;
const matchingResult = funnel.find((res) => res.level === level);
return {
level,
count: matchingResult ? matchingResult.count : 0,
};
});
// Accumulate counts from top to bottom of the funnel
for (let i = filled.length - 1; i >= 0; i--) {
const step = filled[i]!;
const prevStep = filled[i + 1];
if (prevStep) {
step.count += prevStep.count;
}
}
return filled;
}

178
packages/trpc/src/routers/chart.helpers.ts
View File

@@ -13,9 +13,9 @@ import {
subYears,
} from 'date-fns';
import * as mathjs from 'mathjs';
import { pluck, uniq } from 'ramda';
import { last, pluck, repeat, reverse, uniq } from 'ramda';
import { escape } from 'sqlstring';
import type { ISerieDataItem } from '@openpanel/common';
import {
average,
completeSerie,
@@ -26,8 +26,17 @@ import {
slug,
sum,
} from '@openpanel/common';
import type { ISerieDataItem } from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import { chQuery, getChartSql } from '@openpanel/db';
import {
TABLE_NAMES,
chQuery,
createSqlBuilder,
formatClickhouseDate,
getChartSql,
getEventFiltersWhereClause,
getProfiles,
} from '@openpanel/db';
import type {
FinalChart,
IChartEvent,
@@ -232,6 +241,28 @@ export function getDatesFromRange(range: IChartRange) {
};
}
function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
const filled = Array.from({ length: steps }, (_, index) => {
const level = index + 1;
const matchingResult = funnel.find((res) => res.level === level);
return {
level,
count: matchingResult ? matchingResult.count : 0,
};
});
// Accumulate counts from top to bottom of the funnel
for (let i = filled.length - 1; i >= 0; i--) {
const step = filled[i];
const prevStep = filled[i + 1];
// If there's a previous step, add the count to the current step
if (step && prevStep) {
step.count += prevStep.count;
}
}
return filled.reverse();
}
export function getChartStartEndDate({
startDate,
endDate,
@@ -257,6 +288,147 @@ export function getChartPrevStartEndDate({
};
}
export async function getFunnelData({
projectId,
startDate,
endDate,
...payload
}: IChartInput) {
const funnelWindow = (payload.funnelWindow || 24) * 3600;
const funnelGroup = payload.funnelGroup || 'session_id';
if (!startDate || !endDate) {
throw new Error('startDate and endDate are required');
}
if (payload.events.length === 0) {
return {
totalSessions: 0,
steps: [],
};
}
const funnels = payload.events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = ${escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
const innerSql = `SELECT
${funnelGroup},
windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
FROM ${TABLE_NAMES.events}
WHERE
project_id = ${escape(projectId)} AND
created_at >= '${formatClickhouseDate(startDate)}' AND
created_at <= '${formatClickhouseDate(endDate)}' AND
name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
GROUP BY ${funnelGroup}`;
const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`;
const funnel = await chQuery<{ level: number; count: number }>(sql);
const maxLevel = payload.events.length;
const filledFunnelRes = fillFunnel(funnel, maxLevel);
const totalSessions = last(filledFunnelRes)?.count ?? 0;
const steps = reverse(filledFunnelRes).reduce(
(acc, item, index, list) => {
const prev = list[index - 1] ?? { count: totalSessions };
const event = payload.events[item.level - 1]!;
return [
...acc,
{
event: {
...event,
displayName: event.displayName ?? event.name,
},
count: item.count,
percent: (item.count / totalSessions) * 100,
dropoffCount: prev.count - item.count,
dropoffPercent: 100 - (item.count / prev.count) * 100,
previousCount: prev.count,
},
];
},
[] as {
event: IChartEvent & { displayName: string };
count: number;
percent: number;
dropoffCount: number;
dropoffPercent: number;
previousCount: number;
}[],
);
return {
totalSessions,
steps,
};
}
export async function getFunnelStep({
projectId,
startDate,
endDate,
step,
...payload
}: IChartInput & {
step: number;
}) {
throw new Error('not implemented');
// if (!startDate || !endDate) {
// throw new Error('startDate and endDate are required');
// }
// if (payload.events.length === 0) {
// throw new Error('no events selected');
// }
// const funnels = payload.events.map((event) => {
// const { sb, getWhere } = createSqlBuilder();
// sb.where = getEventFiltersWhereClause(event.filters);
// sb.where.name = `name = ${escape(event.name)}`;
// return getWhere().replace('WHERE ', '');
// });
// const innerSql = `SELECT
// session_id,
// windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
// FROM ${TABLE_NAMES.events}
// WHERE
// project_id = ${escape(projectId)} AND
// created_at >= '${formatClickhouseDate(startDate)}' AND
// created_at <= '${formatClickhouseDate(endDate)}' AND
// name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
// GROUP BY session_id`;
// const profileIdsQuery = `WITH sessions AS (${innerSql})
// SELECT
// DISTINCT e.profile_id as id
// FROM sessions s
// JOIN ${TABLE_NAMES.events} e ON s.session_id = e.session_id
// WHERE
// s.level = ${step} AND
// e.project_id = ${escape(projectId)} AND
// e.created_at >= '${formatClickhouseDate(startDate)}' AND
// e.created_at <= '${formatClickhouseDate(endDate)}' AND
// name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
// ORDER BY e.created_at DESC
// LIMIT 500
// `;
// const res = await chQuery<{
// id: string;
// }>(profileIdsQuery);
// return getProfiles(
// res.map((r) => r.id),
// projectId,
// );
}
export async function getChartSerie(payload: IGetChartDataInput) {
async function getSeries() {
const result = await chQuery<ISerieDataItem>(getChartSql(payload));

23
packages/trpc/src/routers/chart.ts
View File

@@ -7,7 +7,6 @@ import {
chQuery,
createSqlBuilder,
db,
getFunnelData,
getSelectPropertyKey,
toDate,
} from '@openpanel/db';
@@ -32,6 +31,8 @@ import {
getChart,
getChartPrevStartEndDate,
getChartStartEndDate,
getFunnelData,
getFunnelStep,
} from './chart.helpers';
function utc(date: string | Date) {
@@ -86,12 +87,9 @@ export const chartRouter = createTRPCRouter({
.map((item) => item.replace(/\.([0-9]+)/g, '[*]'))
.map((item) => `properties.${item}`);
if (event === '*') {
properties.push('name');
}
properties.push(
'has_profile',
'name',
'path',
'origin',
'referrer',
@@ -186,9 +184,7 @@ export const chartRouter = createTRPCRouter({
const [current, previous] = await Promise.all([
getFunnelData({ ...input, ...currentPeriod }),
input.previous
? getFunnelData({ ...input, ...previousPeriod })
: Promise.resolve(null),
getFunnelData({ ...input, ...previousPeriod }),
]);
return {
@@ -197,6 +193,17 @@ export const chartRouter = createTRPCRouter({
};
}),
funnelStep: protectedProcedure
.input(
zChartInput.extend({
step: z.number(),
}),
)
.query(async ({ input }) => {
const currentPeriod = getChartStartEndDate(input);
return getFunnelStep({ ...input, ...currentPeriod });
}),
chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => {
if (ctx.session.userId) {
const access = await getProjectAccessCached({