From f20cca6e15c657f3550e70d504ee5bd23d964520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl-Gerhard=20Lindesva=CC=88rd?= Date: Wed, 12 Jun 2024 22:05:21 +0200 Subject: [PATCH] add charts to export api --- apps/api/package.json | 6 +- apps/api/src/controllers/export.controller.ts | 118 +++++-- apps/api/src/routes/export.router.ts | 6 + apps/api/src/utils/parse-zod-query-string.ts | 19 ++ packages/trpc/src/routers/chart.helpers.ts | 17 +- packages/trpc/src/routers/chart.ts | 292 +++++++++--------- packages/validation/src/index.ts | 6 +- pnpm-lock.yaml | 6 + 8 files changed, 289 insertions(+), 181 deletions(-) create mode 100644 apps/api/src/utils/parse-zod-query-string.ts diff --git a/apps/api/package.json b/apps/api/package.json index 9ef23fa9..c8612c7b 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -22,6 +22,7 @@ "@openpanel/queue": "workspace:*", "@openpanel/redis": "workspace:*", "@openpanel/trpc": "workspace:*", + "@openpanel/validation": "workspace:*", "@trpc/server": "^10.45.1", "fastify": "^4.25.2", "fastify-metrics": "^11.0.0", @@ -35,7 +36,8 @@ "sqlstring": "^2.3.3", "superjson": "^1.13.3", "ua-parser-js": "^1.0.37", - "url-metadata": "^4.1.0" + "url-metadata": "^4.1.0", + "zod": "^3.22.4" }, "devDependencies": { "@openpanel/eslint-config": "workspace:*", @@ -61,4 +63,4 @@ ] }, "prettier": "@openpanel/prettier-config" -} +} \ No newline at end of file diff --git a/apps/api/src/controllers/export.controller.ts b/apps/api/src/controllers/export.controller.ts index 0e442b89..ccb9f647 100644 --- a/apps/api/src/controllers/export.controller.ts +++ b/apps/api/src/controllers/export.controller.ts @@ -1,42 +1,39 @@ +import { parseQueryString } from '@/utils/parse-zod-query-string'; import type { FastifyReply, FastifyRequest } from 'fastify'; +import { z } from 'zod'; import type { GetEventListOptions } from '@openpanel/db'; import { ClientType, db, getEventList, getEventsCount } from '@openpanel/db'; +import { getChart } from '@openpanel/trpc/src/routers/chart'; +import { zChartInput } from '@openpanel/validation'; -type EventsQuery = { - project_id?: string; - event?: string | string[]; - start?: string; - end?: string; - page?: string; - limit?: string; -}; -export async function events( +async function getProjectId( request: FastifyRequest<{ - Querystring: EventsQuery; + Querystring: { + project_id?: string; + projectId?: string; + }; }>, reply: FastifyReply ) { - const query = request.query; - const limit = parseInt(query.limit || '50', 10); - const page = parseInt(query.page || '1', 10); + let projectId = request.query.projectId || request.query.project_id; - if (query.project_id) { + if (projectId) { if ( request.client?.type === ClientType.read && - request.client?.projectId !== query.project_id + request.client?.projectId !== projectId ) { reply.status(403).send({ error: 'Forbidden', message: 'You do not have access to this project', }); - return; + return ''; } const project = await db.project.findUnique({ where: { organizationSlug: request.client?.organizationSlug, - id: query.project_id, + id: projectId, }, }); @@ -45,29 +42,64 @@ export async function events( error: 'Not Found', message: 'Project not found', }); - return; + return ''; } } - const projectId = query.project_id ?? request.client?.projectId; + if (!projectId && request.client?.projectId) { + projectId = request.client?.projectId; + } if (!projectId) { reply.status(400).send({ error: 'Bad Request', message: 'project_id is required', }); - return; + return ''; } + return projectId; +} + +const eventsScheme = z.object({ + project_id: z.string().optional(), + projectId: z.string().optional(), + event: z.union([z.string(), z.array(z.string())]).optional(), + start: z.coerce.string().optional(), + end: z.coerce.string().optional(), + page: z.coerce.number().optional().default(1), + limit: z.coerce.number().optional().default(50), +}); + +export async function events( + request: FastifyRequest<{ + Querystring: z.infer; + }>, + reply: FastifyReply +) { + const query = eventsScheme.safeParse(request.query); + + if (query.success === false) { + return reply.status(400).send({ + error: 'Bad Request', + message: 'Invalid query parameters', + details: query.error.errors, + }); + } + + const projectId = await getProjectId(request, reply); + const limit = query.data.limit; + const page = Math.max(query.data.page, 1); const take = Math.max(Math.min(limit, 50), 1); const cursor = page - 1; const options: GetEventListOptions = { projectId, - events: (Array.isArray(query.event) ? query.event : [query.event]).filter( - (s): s is string => typeof s === 'string' - ), - startDate: query.start ? new Date(query.start) : undefined, - endDate: query.end ? new Date(query.end) : undefined, + events: (Array.isArray(query.data.event) + ? query.data.event + : [query.data.event] + ).filter((s): s is string => typeof s === 'string'), + startDate: query.data.start ? new Date(query.data.start) : undefined, + endDate: query.data.end ? new Date(query.data.end) : undefined, cursor, take, meta: false, @@ -89,3 +121,39 @@ export async function events( data, }); } + +const chartSchemeFull = zChartInput.pick({ + events: true, + breakdowns: true, + projectId: true, + interval: true, + range: true, + previous: true, + startDate: true, + endDate: true, +}); + +export async function charts( + request: FastifyRequest<{ + Querystring: Record; + }>, + reply: FastifyReply +) { + const query = chartSchemeFull.safeParse(parseQueryString(request.query)); + + if (query.success === false) { + return reply.status(400).send({ + error: 'Bad Request', + message: 'Invalid query parameters', + details: query.error.errors, + }); + } + + return getChart({ + ...query.data, + name: 'export-api', + metric: 'sum', + lineType: 'monotone', + chartType: 'linear', + }); +} diff --git a/apps/api/src/routes/export.router.ts b/apps/api/src/routes/export.router.ts index 03ed838b..f1f61d72 100644 --- a/apps/api/src/routes/export.router.ts +++ b/apps/api/src/routes/export.router.ts @@ -31,6 +31,12 @@ const eventRouter: FastifyPluginCallback = (fastify, opts, done) => { url: '/events', handler: controller.events, }); + + fastify.route({ + method: 'GET', + url: '/charts', + handler: controller.charts, + }); done(); }; diff --git a/apps/api/src/utils/parse-zod-query-string.ts b/apps/api/src/utils/parse-zod-query-string.ts new file mode 100644 index 00000000..e492e8ac --- /dev/null +++ b/apps/api/src/utils/parse-zod-query-string.ts @@ -0,0 +1,19 @@ +import { getSafeJson } from '@openpanel/common'; + +export const parseQueryString = (obj: Record): any => { + return Object.fromEntries( + Object.entries(obj).map(([k, v]) => { + if (typeof v === 'object') return [k, parseQueryString(v)]; + if (!isNaN(parseFloat(v))) return [k, parseFloat(v)]; + if (v === 'true') return [k, true]; + if (v === 'false') return [k, false]; + if (typeof v === 'string') { + if (getSafeJson(v) !== null) { + return [k, getSafeJson(v)]; + } + return [k, v]; + } + return [k, null]; + }) + ); +}; diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts index 240150f3..43bb0b62 100644 --- a/packages/trpc/src/routers/chart.helpers.ts +++ b/packages/trpc/src/routers/chart.helpers.ts @@ -1,10 +1,8 @@ import { differenceInMilliseconds, - endOfDay, endOfMonth, endOfYear, formatISO, - startOfDay, startOfMonth, startOfYear, subDays, @@ -14,14 +12,13 @@ import { subYears, } from 'date-fns'; import * as mathjs from 'mathjs'; -import { repeat, reverse, sort } from 'ramda'; +import { repeat, reverse } from 'ramda'; import { escape } from 'sqlstring'; import { completeTimeline, round } from '@openpanel/common'; import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants'; import { chQuery, - convertClickhouseDateToJs, createSqlBuilder, formatClickhouseDate, getChartSql, @@ -44,7 +41,7 @@ export interface ResultItem { } function getEventLegend(event: IChartEvent) { - return event.displayName ?? `${event.name} (${event.id})`; + return event.displayName ?? event.name; } export function withFormula( @@ -68,12 +65,16 @@ export function withFormula( if (events.length === 1) { return series.map((serie) => { + if (!serie.event.id) { + return serie; + } + return { ...serie, data: serie.data.map((item) => { serie.event.id; const scope = { - [serie.event.id]: item?.count ?? 0, + [serie.event.id ?? '']: item?.count ?? 0, }; const count = mathjs .parse(formula) @@ -97,6 +98,10 @@ export function withFormula( ...series[0], data: series[0].data.map((item, dIndex) => { const scope = series.reduce((acc, item) => { + if (!item.event.id) { + return acc; + } + return { ...acc, [item.event.id]: item.data[dIndex]?.count ?? 0, diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index 214e17a2..b950bdad 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -205,154 +205,156 @@ export const chartRouter = createTRPCRouter({ } } - const currentPeriod = getChartStartEndDate(input); - const previousPeriod = getChartPrevStartEndDate({ - range: input.range, - ...currentPeriod, - }); - - const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })]; - - if (input.previous) { - promises.push( - getSeriesFromEvents({ - ...input, - ...previousPeriod, - }) - ); - } - - const result = await Promise.all(promises); - const series = result[0]!; - const previousSeries = result[1]; - - const final: FinalChart = { - events: input.events, - series: series.map((serie, index) => { - const previousSerie = previousSeries?.find( - (item) => item.name === serie.name - ); - const metrics = { - sum: sum(serie.data.map((item) => item.count)), - average: round(average(serie.data.map((item) => item.count)), 2), - min: min(serie.data.map((item) => item.count)), - max: max(serie.data.map((item) => item.count)), - }; - - return { - id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend - name: serie.name, - event: { - ...serie.event, - displayName: serie.event.displayName ?? serie.event.name, - }, - metrics: { - ...metrics, - previous: { - sum: getPreviousMetric( - metrics.sum, - previousSerie - ? sum(previousSerie?.data.map((item) => item.count)) - : null - ), - average: getPreviousMetric( - metrics.average, - previousSerie - ? round( - average(previousSerie?.data.map((item) => item.count)), - 2 - ) - : null - ), - min: getPreviousMetric( - metrics.sum, - previousSerie - ? min(previousSerie?.data.map((item) => item.count)) - : null - ), - max: getPreviousMetric( - metrics.sum, - previousSerie - ? max(previousSerie?.data.map((item) => item.count)) - : null - ), - }, - }, - data: serie.data.map((item, index) => ({ - date: item.date, - count: item.count ?? 0, - label: item.label, - previous: previousSerie?.data[index] - ? getPreviousMetric( - item.count ?? 0, - previousSerie?.data[index]?.count ?? null - ) - : null, - })), - }; - }), - metrics: { - sum: 0, - average: 0, - min: 0, - max: 0, - previous: { - sum: null, - average: null, - min: null, - max: null, - }, - }, - }; - - final.metrics.sum = sum(final.series.map((item) => item.metrics.sum)); - final.metrics.average = round( - average(final.series.map((item) => item.metrics.average)), - 2 - ); - final.metrics.min = min(final.series.map((item) => item.metrics.min)); - final.metrics.max = max(final.series.map((item) => item.metrics.max)); - final.metrics.previous = { - sum: getPreviousMetric( - final.metrics.sum, - sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0)) - ), - average: getPreviousMetric( - final.metrics.average, - round( - average( - final.series.map( - (item) => item.metrics.previous.average?.value ?? 0 - ) - ), - 2 - ) - ), - min: getPreviousMetric( - final.metrics.min, - min(final.series.map((item) => item.metrics.previous.min?.value ?? 0)) - ), - max: getPreviousMetric( - final.metrics.max, - max(final.series.map((item) => item.metrics.previous.max?.value ?? 0)) - ), - }; - - // Sort by sum - final.series = final.series.sort((a, b) => { - if (input.chartType === 'linear') { - const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0); - const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0); - return sumB - sumA; - } else { - return b.metrics[input.metric] - a.metrics[input.metric]; - } - }); - - return final; + return getChart(input); }), }); +export async function getChart(input: IChartInput) { + const currentPeriod = getChartStartEndDate(input); + const previousPeriod = getChartPrevStartEndDate({ + range: input.range, + ...currentPeriod, + }); + + const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })]; + + if (input.previous) { + promises.push( + getSeriesFromEvents({ + ...input, + ...previousPeriod, + }) + ); + } + + const result = await Promise.all(promises); + const series = result[0]!; + const previousSeries = result[1]; + + const final: FinalChart = { + events: input.events, + series: series.map((serie) => { + const previousSerie = previousSeries?.find( + (item) => item.name === serie.name + ); + const metrics = { + sum: sum(serie.data.map((item) => item.count)), + average: round(average(serie.data.map((item) => item.count)), 2), + min: min(serie.data.map((item) => item.count)), + max: max(serie.data.map((item) => item.count)), + }; + + return { + id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend + name: serie.name, + event: { + ...serie.event, + displayName: serie.event.displayName ?? serie.event.name, + }, + metrics: { + ...metrics, + previous: { + sum: getPreviousMetric( + metrics.sum, + previousSerie + ? sum(previousSerie?.data.map((item) => item.count)) + : null + ), + average: getPreviousMetric( + metrics.average, + previousSerie + ? round( + average(previousSerie?.data.map((item) => item.count)), + 2 + ) + : null + ), + min: getPreviousMetric( + metrics.sum, + previousSerie + ? min(previousSerie?.data.map((item) => item.count)) + : null + ), + max: getPreviousMetric( + metrics.sum, + previousSerie + ? max(previousSerie?.data.map((item) => item.count)) + : null + ), + }, + }, + data: serie.data.map((item, index) => ({ + date: item.date, + count: item.count ?? 0, + label: item.label, + previous: previousSerie?.data[index] + ? getPreviousMetric( + item.count ?? 0, + previousSerie?.data[index]?.count ?? null + ) + : null, + })), + }; + }), + metrics: { + sum: 0, + average: 0, + min: 0, + max: 0, + previous: { + sum: null, + average: null, + min: null, + max: null, + }, + }, + }; + + final.metrics.sum = sum(final.series.map((item) => item.metrics.sum)); + final.metrics.average = round( + average(final.series.map((item) => item.metrics.average)), + 2 + ); + final.metrics.min = min(final.series.map((item) => item.metrics.min)); + final.metrics.max = max(final.series.map((item) => item.metrics.max)); + final.metrics.previous = { + sum: getPreviousMetric( + final.metrics.sum, + sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0)) + ), + average: getPreviousMetric( + final.metrics.average, + round( + average( + final.series.map((item) => item.metrics.previous.average?.value ?? 0) + ), + 2 + ) + ), + min: getPreviousMetric( + final.metrics.min, + min(final.series.map((item) => item.metrics.previous.min?.value ?? 0)) + ), + max: getPreviousMetric( + final.metrics.max, + max(final.series.map((item) => item.metrics.previous.max?.value ?? 0)) + ), + }; + + // Sort by sum + final.series = final.series.sort((a, b) => { + if (input.chartType === 'linear') { + const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0); + const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0); + return sumB - sumA; + } else { + return b.metrics[input.metric] - a.metrics[input.metric]; + } + }); + + return final; +} + export function getPreviousMetric( current: number, previous: number | null diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts index 029023c4..ba1584a0 100644 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -19,7 +19,7 @@ export function objectToZodEnums( export const mapKeys = objectToZodEnums; export const zChartEvent = z.object({ - id: z.string(), + id: z.string().optional(), name: z.string(), displayName: z.string().optional(), property: z.string().optional(), @@ -34,7 +34,7 @@ export const zChartEvent = z.object({ ]), filters: z.array( z.object({ - id: z.string(), + id: z.string().optional(), name: z.string(), operator: z.enum(objectToZodEnums(operators)), value: z.array(z.string().or(z.number()).or(z.boolean()).or(z.null())), @@ -42,7 +42,7 @@ export const zChartEvent = z.object({ ), }); export const zChartBreakdown = z.object({ - id: z.string(), + id: z.string().optional(), name: z.string(), }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 77272f35..601f6e68 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: '@openpanel/trpc': specifier: workspace:* version: link:../../packages/trpc + '@openpanel/validation': + specifier: workspace:* + version: link:../../packages/validation '@trpc/server': specifier: ^10.45.1 version: 10.45.1 @@ -101,6 +104,9 @@ importers: url-metadata: specifier: ^4.1.0 version: 4.1.0 + zod: + specifier: ^3.22.4 + version: 3.22.4 devDependencies: '@openpanel/eslint-config': specifier: workspace:*