zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

well deserved clean up (#1)

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-03-18 21:53:07 +01:00
parent 3a8404f704
commit b7513f24d5
106 changed files with 453 additions and 1275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/dashboard/src/server/api/root.ts
View File

@@ -11,7 +11,6 @@ import { projectRouter } from './routers/project';
import { referenceRouter } from './routers/reference';
import { reportRouter } from './routers/report';
import { shareRouter } from './routers/share';
import { uiRouter } from './routers/ui';
import { userRouter } from './routers/user';
/**
@@ -29,7 +28,6 @@ export const appRouter = createTRPCRouter({
client: clientRouter,
event: eventRouter,
profile: profileRouter,
ui: uiRouter,
share: shareRouter,
onboarding: onboardingRouter,
reference: referenceRouter,

216
apps/dashboard/src/server/api/routers/chart.helpers.ts
View File

@@ -1,10 +1,17 @@
import { getDaysOldDate } from '@/utils/date';
import { round } from '@/utils/math';
import { subDays } from 'date-fns';
import * as mathjs from 'mathjs';
import { sort } from 'ramda';
import { repeat, reverse, sort } from 'ramda';
import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';
import { chQuery, convertClickhouseDateToJs, getChartSql } from '@openpanel/db';
import {
chQuery,
convertClickhouseDateToJs,
createSqlBuilder,
formatClickhouseDate,
getChartSql,
getEventFiltersWhereClause,
} from '@openpanel/db';
import type {
IChartEvent,
IChartInput,
@@ -317,7 +324,7 @@ export function getDatesFromRange(range: IChartRange) {
let days = 1;
if (range === '24h') {
const startDate = getDaysOldDate(days);
const startDate = subDays(new Date(), days);
const endDate = new Date();
return {
startDate: startDate.toUTCString(),
@@ -337,7 +344,7 @@ export function getDatesFromRange(range: IChartRange) {
days = 365;
}
const startDate = getDaysOldDate(days);
const startDate = subDays(new Date(), days);
startDate.setUTCHours(0, 0, 0, 0);
const endDate = new Date();
endDate.setUTCHours(23, 59, 59, 999);
@@ -347,10 +354,197 @@ export function getDatesFromRange(range: IChartRange) {
};
}
export function getChartStartEndDate(
input: Pick<IChartInput, 'endDate' | 'startDate' | 'range'>
) {
return input.startDate && input.endDate
? { startDate: input.startDate, endDate: input.endDate }
: getDatesFromRange(input.range);
export function getChartStartEndDate({
startDate,
endDate,
range,
}: Pick<IChartInput, 'endDate' | 'startDate' | 'range'>) {
return startDate && endDate
? { startDate: startDate, endDate: endDate }
: getDatesFromRange(range);
}
export function getChartPrevStartEndDate({
startDate,
endDate,
range,
}: {
startDate: string;
endDate: string;
range: IChartRange;
}) {
let diff = 0;
switch (range) {
case '30min': {
diff = 1000 * 60 * 30;
break;
}
case '1h': {
diff = 1000 * 60 * 60;
break;
}
case '24h':
case 'today': {
diff = 1000 * 60 * 60 * 24;
break;
}
case '7d': {
diff = 1000 * 60 * 60 * 24 * 7;
break;
}
case '14d': {
diff = 1000 * 60 * 60 * 24 * 14;
break;
}
case '1m': {
diff = 1000 * 60 * 60 * 24 * 30;
break;
}
case '3m': {
diff = 1000 * 60 * 60 * 24 * 90;
break;
}
case '6m': {
diff = 1000 * 60 * 60 * 24 * 180;
break;
}
}
return {
startDate: new Date(new Date(startDate).getTime() - diff).toISOString(),
endDate: new Date(new Date(endDate).getTime() - diff).toISOString(),
};
}
export async function getFunnelData({ projectId, ...payload }: IChartInput) {
const { startDate, endDate } = getChartStartEndDate(payload);
if (payload.events.length === 0) {
return {
totalSessions: 0,
steps: [],
};
}
const funnels = payload.events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = '${event.name}'`;
return getWhere().replace('WHERE ', '');
});
const innerSql = `SELECT
session_id,
windowFunnel(6048000000000000,'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
FROM events
WHERE (project_id = '${projectId}' AND created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')
GROUP BY session_id`;
const sql = `SELECT level, count() AS count FROM (${innerSql}) GROUP BY level ORDER BY level DESC`;
const [funnelRes, sessionRes] = await Promise.all([
chQuery<{ level: number; count: number }>(sql),
chQuery<{ count: number }>(
`SELECT count(name) as count FROM events WHERE project_id = '${projectId}' AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')`
),
]);
if (funnelRes[0]?.level !== payload.events.length) {
funnelRes.unshift({
level: payload.events.length,
count: 0,
});
}
const totalSessions = sessionRes[0]?.count ?? 0;
const filledFunnelRes = funnelRes.reduce(
(acc, item, index) => {
const diff =
index !== 0 ? (acc[acc.length - 1]?.level ?? 0) - item.level : 1;
if (diff > 1) {
acc.push(
...reverse(
repeat({}, diff - 1).map((_, index) => ({
count: acc[acc.length - 1]?.count ?? 0,
level: item.level + index + 1,
}))
)
);
}
return [
...acc,
{
count: item.count + (acc[acc.length - 1]?.count ?? 0),
level: item.level,
},
];
},
[] as typeof funnelRes
);
const steps = reverse(filledFunnelRes)
.filter((item) => item.level !== 0)
.reduce(
(acc, item, index, list) => {
const prev = list[index - 1] ?? { count: totalSessions };
return [
...acc,
{
event: payload.events[item.level - 1]!,
before: prev.count,
current: item.count,
dropoff: {
count: prev.count - item.count,
percent: 100 - (item.count / prev.count) * 100,
},
percent: (item.count / totalSessions) * 100,
prevPercent: (prev.count / totalSessions) * 100,
},
];
},
[] as {
event: IChartEvent;
before: number;
current: number;
dropoff: {
count: number;
percent: number;
};
percent: number;
prevPercent: number;
}[]
);
return {
totalSessions,
steps,
};
}
export async function getSeriesFromEvents(input: IChartInput) {
const { startDate, endDate } =
input.startDate && input.endDate
? {
startDate: input.startDate,
endDate: input.endDate,
}
: getDatesFromRange(input.range);
const series = (
await Promise.all(
input.events.map(async (event) =>
getChartData({
...input,
startDate,
endDate,
event,
})
)
)
).flat();
return withFormula(input, series);
}

217
apps/dashboard/src/server/api/routers/chart.ts
View File

@@ -4,132 +4,20 @@ import {
publicProcedure,
} from '@/server/api/trpc';
import { average, max, min, round, sum } from '@/utils/math';
import { flatten, map, pipe, prop, repeat, reverse, sort, uniq } from 'ramda';
import { flatten, map, pipe, prop, sort, uniq } from 'ramda';
import { z } from 'zod';
import {
chQuery,
createSqlBuilder,
formatClickhouseDate,
getEventFiltersWhereClause,
} from '@openpanel/db';
import { chQuery, createSqlBuilder } from '@openpanel/db';
import { zChartInput } from '@openpanel/validation';
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import {
getChartData,
getChartPrevStartEndDate,
getChartStartEndDate,
getDatesFromRange,
withFormula,
getFunnelData,
getSeriesFromEvents,
} from './chart.helpers';
async function getFunnelData({ projectId, ...payload }: IChartInput) {
const { startDate, endDate } = getChartStartEndDate(payload);
if (payload.events.length === 0) {
return {
totalSessions: 0,
steps: [],
};
}
const funnels = payload.events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = '${event.name}'`;
return getWhere().replace('WHERE ', '');
});
const innerSql = `SELECT
session_id,
windowFunnel(6048000000000000,'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
FROM events
WHERE (project_id = '${projectId}' AND created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')
GROUP BY session_id`;
const sql = `SELECT level, count() AS count FROM (${innerSql}) GROUP BY level ORDER BY level DESC`;
const [funnelRes, sessionRes] = await Promise.all([
chQuery<{ level: number; count: number }>(sql),
chQuery<{ count: number }>(
`SELECT count(name) as count FROM events WHERE project_id = '${projectId}' AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')`
),
]);
if (funnelRes[0]?.level !== payload.events.length) {
funnelRes.unshift({
level: payload.events.length,
count: 0,
});
}
const totalSessions = sessionRes[0]?.count ?? 0;
const filledFunnelRes = funnelRes.reduce(
(acc, item, index) => {
const diff =
index !== 0 ? (acc[acc.length - 1]?.level ?? 0) - item.level : 1;
if (diff > 1) {
acc.push(
...reverse(
repeat({}, diff - 1).map((_, index) => ({
count: acc[acc.length - 1]?.count ?? 0,
level: item.level + index + 1,
}))
)
);
}
return [
...acc,
{
count: item.count + (acc[acc.length - 1]?.count ?? 0),
level: item.level,
},
];
},
[] as typeof funnelRes
);
const steps = reverse(filledFunnelRes)
.filter((item) => item.level !== 0)
.reduce(
(acc, item, index, list) => {
const prev = list[index - 1] ?? { count: totalSessions };
return [
...acc,
{
event: payload.events[item.level - 1]!,
before: prev.count,
current: item.count,
dropoff: {
count: prev.count - item.count,
percent: 100 - (item.count / prev.count) * 100,
},
percent: (item.count / totalSessions) * 100,
prevPercent: (prev.count / totalSessions) * 100,
},
];
},
[] as {
event: IChartEvent;
before: number;
current: number;
dropoff: {
count: number;
percent: number;
};
percent: number;
prevPercent: number;
}[]
);
return {
totalSessions,
steps,
};
}
type PreviousValue = {
value: number;
diff: number | null;
@@ -243,7 +131,7 @@ export const chartRouter = createTRPCRouter({
.replace(/^properties\./, '')
.replace('.*.', '.%.')}')) as values`;
} else {
sb.select.values = `${property} as values`;
sb.select.values = `distinct ${property} as values`;
}
const events = await chQuery<{ values: string[] }>(getSql());
@@ -266,57 +154,19 @@ export const chartRouter = createTRPCRouter({
// TODO: Make this private
chart: publicProcedure.input(zChartInput).query(async ({ input }) => {
const { startDate, endDate } = getChartStartEndDate(input);
let diff = 0;
const currentPeriod = getChartStartEndDate(input);
const previousPeriod = getChartPrevStartEndDate({
range: input.range,
...currentPeriod,
});
switch (input.range) {
case '30min': {
diff = 1000 * 60 * 30;
break;
}
case '1h': {
diff = 1000 * 60 * 60;
break;
}
case '24h':
case 'today': {
diff = 1000 * 60 * 60 * 24;
break;
}
case '7d': {
diff = 1000 * 60 * 60 * 24 * 7;
break;
}
case '14d': {
diff = 1000 * 60 * 60 * 24 * 14;
break;
}
case '1m': {
diff = 1000 * 60 * 60 * 24 * 30;
break;
}
case '3m': {
diff = 1000 * 60 * 60 * 24 * 90;
break;
}
case '6m': {
diff = 1000 * 60 * 60 * 24 * 180;
break;
}
}
const promises = [getSeriesFromEvents(input)];
const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];
if (input.previous) {
promises.push(
getSeriesFromEvents({
...input,
...{
startDate: new Date(
new Date(startDate).getTime() - diff
).toISOString(),
endDate: new Date(new Date(endDate).getTime() - diff).toISOString(),
},
...previousPeriod,
})
);
}
@@ -407,11 +257,11 @@ export const chartRouter = createTRPCRouter({
final.metrics.max = max(final.series.map((item) => item.metrics.max));
final.metrics.previous = {
sum: getPreviousMetric(
sum(final.series.map((item) => item.metrics.sum)),
final.metrics.sum,
sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))
),
average: getPreviousMetric(
round(average(final.series.map((item) => item.metrics.average)), 2),
final.metrics.average,
round(
average(
final.series.map(
@@ -422,15 +272,16 @@ export const chartRouter = createTRPCRouter({
)
),
min: getPreviousMetric(
min(final.series.map((item) => item.metrics.min)),
final.metrics.min,
min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))
),
max: getPreviousMetric(
max(final.series.map((item) => item.metrics.max)),
final.metrics.max,
max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))
),
};
// Sort by sum
final.series = final.series.sort((a, b) => {
if (input.chartType === 'linear') {
const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
@@ -441,16 +292,11 @@ export const chartRouter = createTRPCRouter({
}
});
// await new Promise((res) => {
// setTimeout(() => {
// res();
// }, 100);
// });
return final;
}),
});
function getPreviousMetric(
export function getPreviousMetric(
current: number,
previous: number | null
): PreviousValue {
@@ -483,28 +329,3 @@ function getPreviousMetric(
value: previous,
};
}
async function getSeriesFromEvents(input: IChartInput) {
const { startDate, endDate } =
input.startDate && input.endDate
? {
startDate: input.startDate,
endDate: input.endDate,
}
: getDatesFromRange(input.range);
const series = (
await Promise.all(
input.events.map(async (event) =>
getChartData({
...input,
startDate,
endDate,
event,
})
)
)
).flat();
return withFormula(input, series);
}

4
apps/dashboard/src/server/api/routers/client.ts
View File

@@ -1,9 +1,9 @@
import { randomUUID } from 'crypto';
import { createTRPCRouter, protectedProcedure } from '@/server/api/trpc';
import { db } from '@/server/db';
import { z } from 'zod';
import { hashPassword, stripTrailingSlash } from '@openpanel/common';
import { db, transformClient } from '@openpanel/db';
export const clientRouter = createTRPCRouter({
list: protectedProcedure
@@ -76,7 +76,7 @@ export const clientRouter = createTRPCRouter({
});
return {
...client,
...transformClient(client),
secret: input.cors ? null : secret,
};
}),

52
apps/dashboard/src/server/api/routers/dashboard.ts
View File

@@ -1,60 +1,20 @@
import { createTRPCRouter, protectedProcedure } from '@/server/api/trpc';
import { db, getId } from '@/server/db';
import { getId } from '@/utils/getDbId';
import { PrismaError } from 'prisma-error-enum';
import { z } from 'zod';
import { db, getDashboardsByProjectId } from '@openpanel/db';
import type { Prisma } from '@openpanel/db';
export const dashboardRouter = createTRPCRouter({
get: protectedProcedure
.input(z.object({ id: z.string() }))
.query(async ({ input }) => {
return db.dashboard.findUnique({
where: {
id: input.id,
},
});
}),
list: protectedProcedure
.input(
z
.object({
projectId: z.string(),
})
.or(
z.object({
organizationId: z.string(),
})
)
z.object({
projectId: z.string(),
})
)
.query(async ({ input }) => {
if ('projectId' in input) {
return db.dashboard.findMany({
where: {
project_id: input.projectId,
},
orderBy: {
createdAt: 'desc',
},
include: {
project: true,
},
});
} else {
return db.dashboard.findMany({
where: {
project: {
organization_slug: input.organizationId,
},
},
include: {
project: true,
},
orderBy: {
createdAt: 'desc',
},
});
}
return getDashboardsByProjectId(input.projectId);
}),
create: protectedProcedure
.input(

15
apps/dashboard/src/server/api/routers/onboarding.ts
View File

@@ -4,7 +4,12 @@ import { clerkClient } from '@clerk/nextjs';
import { z } from 'zod';
import { hashPassword, stripTrailingSlash } from '@openpanel/common';
import { db } from '@openpanel/db';
import {
db,
transformClient,
transformOrganization,
transformProject,
} from '@openpanel/db';
export const onboardingRouter = createTRPCRouter({
organziation: protectedProcedure
@@ -41,12 +46,12 @@ export const onboardingRouter = createTRPCRouter({
});
return {
client: {
client: transformClient({
...client,
secret: input.cors ? null : secret,
},
project,
organization: org,
}),
project: transformProject(project),
organization: transformOrganization(org),
};
}

1
apps/dashboard/src/server/api/routers/organization.ts
View File

@@ -9,7 +9,6 @@ export const organizationRouter = createTRPCRouter({
list: protectedProcedure.query(() => {
return clerkClient.organizations.getOrganizationList();
}),
// first: protectedProcedure.query(() => getCurrentOrganization()),
get: protectedProcedure
.input(
z.object({

60
apps/dashboard/src/server/api/routers/profile.ts
View File

@@ -3,71 +3,12 @@ import {
protectedProcedure,
publicProcedure,
} from '@/server/api/trpc';
import { db } from '@/server/db';
import { flatten, map, pipe, prop, sort, uniq } from 'ramda';
import { z } from 'zod';
import { chQuery, createSqlBuilder } from '@openpanel/db';
export const profileRouter = createTRPCRouter({
list: protectedProcedure
.input(
z.object({
query: z.string().nullable(),
projectId: z.string(),
take: z.number().default(100),
skip: z.number().default(0),
})
)
.query(async ({ input: { take, skip, projectId, query } }) => {
return db.profile.findMany({
take,
skip,
where: {
project_id: projectId,
...(query
? {
OR: [
{
first_name: {
contains: query,
mode: 'insensitive',
},
},
{
last_name: {
contains: query,
mode: 'insensitive',
},
},
{
email: {
contains: query,
mode: 'insensitive',
},
},
],
}
: {}),
},
orderBy: {
createdAt: 'desc',
},
});
}),
get: protectedProcedure
.input(
z.object({
id: z.string(),
})
)
.query(async ({ input: { id } }) => {
return db.profile.findUniqueOrThrow({
where: {
id,
},
});
}),
properties: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ input: { projectId } }) => {
@@ -88,6 +29,7 @@ export const profileRouter = createTRPCRouter({
uniq
)(properties);
}),
values: publicProcedure
.input(
z.object({

26
apps/dashboard/src/server/api/routers/project.ts
View File

@@ -1,8 +1,9 @@
import { createTRPCRouter, protectedProcedure } from '@/server/api/trpc';
import { db, getId } from '@/server/db';
import { slug } from '@/utils/slug';
import { getId } from '@/utils/getDbId';
import { z } from 'zod';
import { db, getProjectsByOrganizationSlug } from '@openpanel/db';
export const projectRouter = createTRPCRouter({
list: protectedProcedure
.input(
@@ -12,26 +13,9 @@ export const projectRouter = createTRPCRouter({
)
.query(async ({ input: { organizationId } }) => {
if (organizationId === null) return [];
return getProjectsByOrganizationSlug(organizationId);
}),
return db.project.findMany({
where: {
organization_slug: organizationId,
},
});
}),
get: protectedProcedure
.input(
z.object({
id: z.string(),
})
)
.query(({ input: { id } }) => {
return db.project.findUniqueOrThrow({
where: {
id,
},
});
}),
update: protectedProcedure
.input(
z.object({

50
apps/dashboard/src/server/api/routers/report.ts
View File

@@ -1,57 +1,11 @@
import { createTRPCRouter, protectedProcedure } from '@/server/api/trpc';
import { db } from '@/server/db';
import { z } from 'zod';
import { transformReport } from '@openpanel/db';
import { db } from '@openpanel/db';
import { zChartInput } from '@openpanel/validation';
export const reportRouter = createTRPCRouter({
get: protectedProcedure
.input(
z.object({
id: z.string(),
})
)
.query(({ input: { id } }) => {
return db.report
.findUniqueOrThrow({
where: {
id,
},
})
.then(transformReport);
}),
list: protectedProcedure
.input(
z.object({
projectId: z.string(),
dashboardId: z.string(),
})
)
.query(async ({ input: { projectId, dashboardId } }) => {
const [dashboard, reports] = await db.$transaction([
db.dashboard.findUniqueOrThrow({
where: {
id: dashboardId,
},
}),
db.report.findMany({
where: {
project_id: projectId,
dashboard_id: dashboardId,
},
orderBy: {
createdAt: 'desc',
},
}),
]);
return {
reports: reports.map(transformReport),
dashboard,
};
}),
save: protectedProcedure
create: protectedProcedure
.input(
z.object({
report: zChartInput.omit({ projectId: true }),

2
apps/dashboard/src/server/api/routers/share.ts
View File

@@ -1,7 +1,7 @@
import { createTRPCRouter, protectedProcedure } from '@/server/api/trpc';
import { db } from '@/server/db';
import ShortUniqueId from 'short-unique-id';
import { db } from '@openpanel/db';
import { zShareOverview } from '@openpanel/validation';
const uid = new ShortUniqueId({ length: 6 });

15
apps/dashboard/src/server/api/routers/ui.ts
View File

@@ -1,15 +0,0 @@
import { createTRPCRouter, protectedProcedure } from '@/server/api/trpc';
import { z } from 'zod';
export const uiRouter = createTRPCRouter({
breadcrumbs: protectedProcedure
.input(
z.object({
url: z.string(),
})
)
.query(({ input: { url } }) => {
const parts = url.split('/').filter(Boolean);
return parts;
}),
});