feature(dashboard): add new retention chart type
This commit is contained in:
Carl-Gerhard Lindesvärd
committed by
Carl-Gerhard Lindesvärd
Carl-Gerhard Lindesvärd
parent
e2065da16e
commit
f977c5454a
@@ -1,4 +1,4 @@
|
||||
import { flatten, map, pipe, prop, sort, uniq } from 'ramda';
|
||||
import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';
|
||||
import { escape } from 'sqlstring';
|
||||
import { z } from 'zod';
|
||||
|
||||
@@ -7,12 +7,23 @@ import {
|
||||
chQuery,
|
||||
createSqlBuilder,
|
||||
db,
|
||||
formatClickhouseDate,
|
||||
getSelectPropertyKey,
|
||||
toDate,
|
||||
} from '@openpanel/db';
|
||||
import { zChartInput, zRange, zTimeInterval } from '@openpanel/validation';
|
||||
import {
|
||||
zChartInput,
|
||||
zCriteria,
|
||||
zRange,
|
||||
zTimeInterval,
|
||||
} from '@openpanel/validation';
|
||||
|
||||
import { round } from '@openpanel/common';
|
||||
import {
|
||||
differenceInDays,
|
||||
differenceInMonths,
|
||||
differenceInWeeks,
|
||||
formatISO,
|
||||
} from 'date-fns';
|
||||
import { getProjectAccessCached } from '../access';
|
||||
import { TRPCAccessError } from '../errors';
|
||||
import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
|
||||
@@ -24,21 +35,23 @@ import {
|
||||
getFunnelStep,
|
||||
} from './chart.helpers';
|
||||
|
||||
function utc(date: string | Date) {
|
||||
if (typeof date === 'string') {
|
||||
return date.replace('T', ' ').slice(0, 19);
|
||||
}
|
||||
return formatISO(date).replace('T', ' ').slice(0, 19);
|
||||
}
|
||||
|
||||
export const chartRouter = createTRPCRouter({
|
||||
events: protectedProcedure
|
||||
.input(
|
||||
z.object({
|
||||
projectId: z.string(),
|
||||
range: zRange,
|
||||
interval: zTimeInterval,
|
||||
startDate: z.string().nullish(),
|
||||
endDate: z.string().nullish(),
|
||||
}),
|
||||
)
|
||||
.query(async ({ input: { projectId, ...input } }) => {
|
||||
const { startDate, endDate } = getChartStartEndDate(input);
|
||||
.query(async ({ input: { projectId } }) => {
|
||||
const events = await chQuery<{ name: string }>(
|
||||
`SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`,
|
||||
`SELECT DISTINCT name FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${escape(projectId)}`,
|
||||
);
|
||||
|
||||
return [
|
||||
@@ -54,23 +67,22 @@ export const chartRouter = createTRPCRouter({
|
||||
z.object({
|
||||
event: z.string().optional(),
|
||||
projectId: z.string(),
|
||||
range: zRange,
|
||||
interval: zTimeInterval,
|
||||
startDate: z.string().nullish(),
|
||||
endDate: z.string().nullish(),
|
||||
}),
|
||||
)
|
||||
.query(async ({ input: { projectId, event, ...input } }) => {
|
||||
const { startDate, endDate } = getChartStartEndDate(input);
|
||||
const events = await chQuery<{ keys: string[] }>(
|
||||
`SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${
|
||||
event && event !== '*' ? `name = ${escape(event)} AND ` : ''
|
||||
} project_id = ${escape(projectId)} AND
|
||||
${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`,
|
||||
.query(async ({ input: { projectId, event } }) => {
|
||||
const res = await chQuery<{ property_key: string; created_at: string }>(
|
||||
`SELECT
|
||||
distinct property_key,
|
||||
max(created_at) as created_at
|
||||
FROM ${TABLE_NAMES.event_property_values_mv}
|
||||
WHERE project_id = ${escape(projectId)}
|
||||
${event && event !== '*' ? `AND name = ${escape(event)}` : ''}
|
||||
GROUP BY property_key
|
||||
ORDER BY created_at DESC`,
|
||||
);
|
||||
|
||||
const properties = events
|
||||
.flatMap((event) => event.keys)
|
||||
const properties = res
|
||||
.map((item) => item.property_key)
|
||||
.map((item) => item.replace(/\.([0-9]+)\./g, '.*.'))
|
||||
.map((item) => item.replace(/\.([0-9]+)/g, '[*]'))
|
||||
.map((item) => `properties.${item}`);
|
||||
@@ -108,36 +120,55 @@ export const chartRouter = createTRPCRouter({
|
||||
event: z.string(),
|
||||
property: z.string(),
|
||||
projectId: z.string(),
|
||||
range: zRange,
|
||||
interval: zTimeInterval,
|
||||
startDate: z.string().nullish(),
|
||||
endDate: z.string().nullish(),
|
||||
}),
|
||||
)
|
||||
.query(async ({ input: { event, property, projectId, ...input } }) => {
|
||||
const { startDate, endDate } = getChartStartEndDate(input);
|
||||
if (property === 'has_profile') {
|
||||
return {
|
||||
values: ['true', 'false'],
|
||||
};
|
||||
}
|
||||
|
||||
const { sb, getSql } = createSqlBuilder();
|
||||
sb.where.project_id = `project_id = ${escape(projectId)}`;
|
||||
if (event !== '*') {
|
||||
sb.where.event = `name = ${escape(event)}`;
|
||||
const values: string[] = [];
|
||||
|
||||
if (property.startsWith('properties.')) {
|
||||
const propertyKey = property.replace(/^properties\./, '');
|
||||
|
||||
const res = await chQuery<{
|
||||
property_value: string;
|
||||
created_at: string;
|
||||
}>(
|
||||
`SELECT
|
||||
distinct property_value,
|
||||
max(created_at) as created_at
|
||||
FROM ${TABLE_NAMES.event_property_values_mv}
|
||||
WHERE project_id = ${escape(projectId)}
|
||||
AND property_key = ${escape(propertyKey)}
|
||||
${event && event !== '*' ? `AND name = ${escape(event)}` : ''}
|
||||
GROUP BY property_value
|
||||
ORDER BY created_at DESC`,
|
||||
);
|
||||
|
||||
values.push(...res.map((e) => e.property_value));
|
||||
} else {
|
||||
const { sb, getSql } = createSqlBuilder();
|
||||
sb.where.project_id = `project_id = ${escape(projectId)}`;
|
||||
if (event !== '*') {
|
||||
sb.where.event = `name = ${escape(event)}`;
|
||||
}
|
||||
sb.select.values = `distinct ${getSelectPropertyKey(property)} as values`;
|
||||
sb.where.date = `${toDate('created_at', 'month')} > now() - INTERVAL 6 MONTH`;
|
||||
const events = await chQuery<{ values: string[] }>(getSql());
|
||||
|
||||
values.push(
|
||||
...pipe(
|
||||
(data: typeof events) => map(prop('values'), data),
|
||||
flatten,
|
||||
uniq,
|
||||
sort((a, b) => a.length - b.length),
|
||||
)(events),
|
||||
);
|
||||
}
|
||||
sb.select.values = `distinct ${getSelectPropertyKey(property)} as values`;
|
||||
sb.where.date = `${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`;
|
||||
|
||||
const events = await chQuery<{ values: string[] }>(getSql());
|
||||
|
||||
const values = pipe(
|
||||
(data: typeof events) => map(prop('values'), data),
|
||||
flatten,
|
||||
uniq,
|
||||
sort((a, b) => a.length - b.length),
|
||||
)(events);
|
||||
|
||||
return {
|
||||
values,
|
||||
@@ -204,4 +235,208 @@ export const chartRouter = createTRPCRouter({
|
||||
|
||||
return getChart(input);
|
||||
}),
|
||||
cohort: protectedProcedure
|
||||
.input(
|
||||
z.object({
|
||||
projectId: z.string(),
|
||||
firstEvent: z.array(z.string()).min(1),
|
||||
secondEvent: z.array(z.string()).min(1),
|
||||
criteria: zCriteria.default('on_or_after'),
|
||||
startDate: z.string().nullish(),
|
||||
endDate: z.string().nullish(),
|
||||
interval: zTimeInterval.default('day'),
|
||||
range: zRange,
|
||||
}),
|
||||
)
|
||||
.query(async ({ input }) => {
|
||||
const { projectId, firstEvent, secondEvent } = input;
|
||||
const dates = getChartStartEndDate(input);
|
||||
const diffInterval = {
|
||||
minute: () => differenceInDays(dates.endDate, dates.startDate),
|
||||
hour: () => differenceInDays(dates.endDate, dates.startDate),
|
||||
day: () => differenceInDays(dates.endDate, dates.startDate),
|
||||
week: () => differenceInWeeks(dates.endDate, dates.startDate),
|
||||
month: () => differenceInMonths(dates.endDate, dates.startDate),
|
||||
}[input.interval]();
|
||||
const sqlInterval = {
|
||||
minute: 'DAY',
|
||||
hour: 'DAY',
|
||||
day: 'DAY',
|
||||
week: 'WEEK',
|
||||
month: 'MONTH',
|
||||
}[input.interval];
|
||||
|
||||
const sqlToStartOf = {
|
||||
minute: 'toDate',
|
||||
hour: 'toDate',
|
||||
day: 'toDate',
|
||||
week: 'toStartOfWeek',
|
||||
month: 'toStartOfMonth',
|
||||
}[input.interval];
|
||||
|
||||
const countCriteria = input.criteria === 'on_or_after' ? '>=' : '=';
|
||||
|
||||
const usersSelect = range(0, diffInterval + 1)
|
||||
.map(
|
||||
(index) =>
|
||||
`groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`,
|
||||
)
|
||||
.join(',\n');
|
||||
|
||||
const countsSelect = range(0, diffInterval + 1)
|
||||
.map(
|
||||
(index) =>
|
||||
`length(interval_${index}_users) AS interval_${index}_user_count`,
|
||||
)
|
||||
.join(',\n');
|
||||
|
||||
const whereEventNameIs = (event: string[]) => {
|
||||
if (event.length === 1) {
|
||||
return `name = ${escape(event[0])}`;
|
||||
}
|
||||
return `name IN (${event.map((e) => escape(e)).join(',')})`;
|
||||
};
|
||||
|
||||
// const dropoffsSelect = range(1, diffInterval + 1)
|
||||
// .map(
|
||||
// (index) =>
|
||||
// `arrayFilter(x -> NOT has(interval_${index}_users, x), interval_${index - 1}_users) AS interval_${index}_dropoffs`,
|
||||
// )
|
||||
// .join(',\n');
|
||||
|
||||
// const dropoffCountsSelect = range(1, diffInterval + 1)
|
||||
// .map(
|
||||
// (index) =>
|
||||
// `length(interval_${index}_dropoffs) AS interval_${index}_dropoff_count`,
|
||||
// )
|
||||
// .join(',\n');
|
||||
|
||||
// SELECT
|
||||
// project_id,
|
||||
// profile_id AS userID,
|
||||
// name,
|
||||
// toDate(created_at) AS cohort_interval
|
||||
// FROM events_v2
|
||||
// WHERE profile_id != device_id
|
||||
// AND ${whereEventNameIs(firstEvent)}
|
||||
// AND project_id = ${escape(projectId)}
|
||||
// AND created_at BETWEEN toDate('${utc(dates.startDate)}') AND toDate('${utc(dates.endDate)}')
|
||||
// GROUP BY project_id, name, cohort_interval, userID
|
||||
|
||||
const cohortQuery = `
|
||||
WITH
|
||||
cohort_users AS (
|
||||
SELECT
|
||||
profile_id AS userID,
|
||||
project_id,
|
||||
${sqlToStartOf}(created_at) AS cohort_interval
|
||||
FROM ${TABLE_NAMES.cohort_events_mv}
|
||||
WHERE ${whereEventNameIs(firstEvent)}
|
||||
AND project_id = ${escape(projectId)}
|
||||
AND created_at BETWEEN toDate('${utc(dates.startDate)}') AND toDate('${utc(dates.endDate)}')
|
||||
),
|
||||
retention_matrix AS (
|
||||
SELECT
|
||||
c.cohort_interval,
|
||||
e.profile_id,
|
||||
dateDiff('${sqlInterval}', c.cohort_interval, ${sqlToStartOf}(e.created_at)) AS x_after_cohort
|
||||
FROM cohort_users AS c
|
||||
INNER JOIN ${TABLE_NAMES.cohort_events_mv} AS e ON c.userID = e.profile_id
|
||||
WHERE (${whereEventNameIs(secondEvent)}) AND (e.project_id = ${escape(projectId)})
|
||||
AND ((e.created_at >= c.cohort_interval) AND (e.created_at <= (c.cohort_interval + INTERVAL ${diffInterval} ${sqlInterval})))
|
||||
),
|
||||
interval_users AS (
|
||||
SELECT
|
||||
cohort_interval,
|
||||
${usersSelect}
|
||||
FROM retention_matrix
|
||||
GROUP BY cohort_interval
|
||||
),
|
||||
cohort_sizes AS (
|
||||
SELECT
|
||||
cohort_interval,
|
||||
COUNT(DISTINCT userID) AS total_first_event_count
|
||||
FROM cohort_users
|
||||
GROUP BY cohort_interval
|
||||
)
|
||||
SELECT
|
||||
cohort_interval,
|
||||
cohort_sizes.total_first_event_count,
|
||||
${countsSelect}
|
||||
FROM interval_users
|
||||
LEFT JOIN cohort_sizes AS cs ON cohort_interval = cs.cohort_interval
|
||||
ORDER BY cohort_interval ASC
|
||||
`;
|
||||
|
||||
const cohortData = await chQuery<{
|
||||
cohort_interval: string;
|
||||
total_first_event_count: number;
|
||||
[key: string]: any;
|
||||
}>(cohortQuery);
|
||||
|
||||
return processCohortData(cohortData, diffInterval);
|
||||
}),
|
||||
});
|
||||
|
||||
function processCohortData(
|
||||
data: Array<{
|
||||
cohort_interval: string;
|
||||
total_first_event_count: number;
|
||||
[key: string]: any;
|
||||
}>,
|
||||
diffInterval: number,
|
||||
) {
|
||||
if (data.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const processed = data.map((row) => {
|
||||
const sum = row.total_first_event_count;
|
||||
const values = range(0, diffInterval + 1).map(
|
||||
(index) => (row[`interval_${index}_user_count`] || 0) as number,
|
||||
);
|
||||
|
||||
return {
|
||||
cohort_interval: row.cohort_interval,
|
||||
sum,
|
||||
values: values,
|
||||
percentages: values.map((value) =>
|
||||
sum > 0 ? round((value / sum) * 100, 2) : 0,
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
// Initialize aggregation for averages
|
||||
const averageData: {
|
||||
sum: number;
|
||||
values: Array<number>;
|
||||
percentages: Array<number>;
|
||||
} = {
|
||||
sum: 0,
|
||||
values: range(0, diffInterval + 1).map(() => 0),
|
||||
percentages: range(0, diffInterval + 1).map(() => 0),
|
||||
};
|
||||
|
||||
// Aggregate data for averages
|
||||
processed.forEach((row) => {
|
||||
averageData.sum += row.sum;
|
||||
row.values.forEach((value, index) => {
|
||||
averageData.values[index] += value;
|
||||
averageData.percentages[index] += row.percentages[index]!;
|
||||
});
|
||||
});
|
||||
|
||||
const cohortCount = processed.length;
|
||||
|
||||
// Calculate average values
|
||||
const averageRow = {
|
||||
cohort_interval: 'Average',
|
||||
sum: cohortCount > 0 ? round(averageData.sum / cohortCount, 0) : 0,
|
||||
percentages: averageData.percentages.map((item) =>
|
||||
round(item / cohortCount, 2),
|
||||
),
|
||||
values: averageData.values.map((item) => round(item / cohortCount, 0)),
|
||||
};
|
||||
|
||||
return [averageRow, ...processed];
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user