add charts to export api

2024-06-12 22:05:21 +02:00
parent 3af1882d1e
commit f20cca6e15
8 changed files with 289 additions and 181 deletions
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -22,6 +22,7 @@
    "@openpanel/queue": "workspace:*",
    "@openpanel/redis": "workspace:*",
    "@openpanel/trpc": "workspace:*",
+    "@openpanel/validation": "workspace:*",
    "@trpc/server": "^10.45.1",
    "fastify": "^4.25.2",
    "fastify-metrics": "^11.0.0",
@@ -35,7 +36,8 @@
    "sqlstring": "^2.3.3",
    "superjson": "^1.13.3",
    "ua-parser-js": "^1.0.37",
-    "url-metadata": "^4.1.0"
+    "url-metadata": "^4.1.0",
+    "zod": "^3.22.4"
  },
  "devDependencies": {
    "@openpanel/eslint-config": "workspace:*",
@@ -61,4 +63,4 @@
    ]
  },
  "prettier": "@openpanel/prettier-config"
-}
+}
--- a/apps/api/src/controllers/export.controller.ts
+++ b/apps/api/src/controllers/export.controller.ts
@@ -1,42 +1,39 @@
+import { parseQueryString } from '@/utils/parse-zod-query-string';
 import type { FastifyReply, FastifyRequest } from 'fastify';
+import { z } from 'zod';

 import type { GetEventListOptions } from '@openpanel/db';
 import { ClientType, db, getEventList, getEventsCount } from '@openpanel/db';
+import { getChart } from '@openpanel/trpc/src/routers/chart';
+import { zChartInput } from '@openpanel/validation';

-type EventsQuery = {
-  project_id?: string;
-  event?: string | string[];
-  start?: string;
-  end?: string;
-  page?: string;
-  limit?: string;
-};
-export async function events(
+async function getProjectId(
  request: FastifyRequest<{
-    Querystring: EventsQuery;
+    Querystring: {
+      project_id?: string;
+      projectId?: string;
+    };
  }>,
  reply: FastifyReply
 ) {
-  const query = request.query;
-  const limit = parseInt(query.limit || '50', 10);
-  const page = parseInt(query.page || '1', 10);
+  let projectId = request.query.projectId || request.query.project_id;

-  if (query.project_id) {
+  if (projectId) {
    if (
      request.client?.type === ClientType.read &&
-      request.client?.projectId !== query.project_id
+      request.client?.projectId !== projectId
    ) {
      reply.status(403).send({
        error: 'Forbidden',
        message: 'You do not have access to this project',
      });
-      return;
+      return '';
    }

    const project = await db.project.findUnique({
      where: {
        organizationSlug: request.client?.organizationSlug,
-        id: query.project_id,
+        id: projectId,
      },
    });

@@ -45,29 +42,64 @@ export async function events(
        error: 'Not Found',
        message: 'Project not found',
      });
-      return;
+      return '';
    }
  }

-  const projectId = query.project_id ?? request.client?.projectId;
+  if (!projectId && request.client?.projectId) {
+    projectId = request.client?.projectId;
+  }

  if (!projectId) {
    reply.status(400).send({
      error: 'Bad Request',
      message: 'project_id is required',
    });
-    return;
+    return '';
  }

+  return projectId;
+}
+
+const eventsScheme = z.object({
+  project_id: z.string().optional(),
+  projectId: z.string().optional(),
+  event: z.union([z.string(), z.array(z.string())]).optional(),
+  start: z.coerce.string().optional(),
+  end: z.coerce.string().optional(),
+  page: z.coerce.number().optional().default(1),
+  limit: z.coerce.number().optional().default(50),
+});
+
+export async function events(
+  request: FastifyRequest<{
+    Querystring: z.infer<typeof eventsScheme>;
+  }>,
+  reply: FastifyReply
+) {
+  const query = eventsScheme.safeParse(request.query);
+
+  if (query.success === false) {
+    return reply.status(400).send({
+      error: 'Bad Request',
+      message: 'Invalid query parameters',
+      details: query.error.errors,
+    });
+  }
+
+  const projectId = await getProjectId(request, reply);
+  const limit = query.data.limit;
+  const page = Math.max(query.data.page, 1);
  const take = Math.max(Math.min(limit, 50), 1);
  const cursor = page - 1;
  const options: GetEventListOptions = {
    projectId,
-    events: (Array.isArray(query.event) ? query.event : [query.event]).filter(
-      (s): s is string => typeof s === 'string'
-    ),
-    startDate: query.start ? new Date(query.start) : undefined,
-    endDate: query.end ? new Date(query.end) : undefined,
+    events: (Array.isArray(query.data.event)
+      ? query.data.event
+      : [query.data.event]
+    ).filter((s): s is string => typeof s === 'string'),
+    startDate: query.data.start ? new Date(query.data.start) : undefined,
+    endDate: query.data.end ? new Date(query.data.end) : undefined,
    cursor,
    take,
    meta: false,
@@ -89,3 +121,39 @@ export async function events(
    data,
  });
 }
+
+const chartSchemeFull = zChartInput.pick({
+  events: true,
+  breakdowns: true,
+  projectId: true,
+  interval: true,
+  range: true,
+  previous: true,
+  startDate: true,
+  endDate: true,
+});
+
+export async function charts(
+  request: FastifyRequest<{
+    Querystring: Record<string, string>;
+  }>,
+  reply: FastifyReply
+) {
+  const query = chartSchemeFull.safeParse(parseQueryString(request.query));
+
+  if (query.success === false) {
+    return reply.status(400).send({
+      error: 'Bad Request',
+      message: 'Invalid query parameters',
+      details: query.error.errors,
+    });
+  }
+
+  return getChart({
+    ...query.data,
+    name: 'export-api',
+    metric: 'sum',
+    lineType: 'monotone',
+    chartType: 'linear',
+  });
+}
--- a/apps/api/src/routes/export.router.ts
+++ b/apps/api/src/routes/export.router.ts
@@ -31,6 +31,12 @@ const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
    url: '/events',
    handler: controller.events,
  });
+
+  fastify.route({
+    method: 'GET',
+    url: '/charts',
+    handler: controller.charts,
+  });
  done();
 };

--- a/apps/api/src/utils/parse-zod-query-string.ts
+++ b/apps/api/src/utils/parse-zod-query-string.ts
@@ -0,0 +1,19 @@
+import { getSafeJson } from '@openpanel/common';
+
+export const parseQueryString = (obj: Record<string, any>): any => {
+  return Object.fromEntries(
+    Object.entries(obj).map(([k, v]) => {
+      if (typeof v === 'object') return [k, parseQueryString(v)];
+      if (!isNaN(parseFloat(v))) return [k, parseFloat(v)];
+      if (v === 'true') return [k, true];
+      if (v === 'false') return [k, false];
+      if (typeof v === 'string') {
+        if (getSafeJson(v) !== null) {
+          return [k, getSafeJson(v)];
+        }
+        return [k, v];
+      }
+      return [k, null];
+    })
+  );
+};
--- a/packages/trpc/src/routers/chart.helpers.ts
+++ b/packages/trpc/src/routers/chart.helpers.ts
@@ -1,10 +1,8 @@
 import {
  differenceInMilliseconds,
-  endOfDay,
  endOfMonth,
  endOfYear,
  formatISO,
-  startOfDay,
  startOfMonth,
  startOfYear,
  subDays,
@@ -14,14 +12,13 @@ import {
  subYears,
 } from 'date-fns';
 import * as mathjs from 'mathjs';
-import { repeat, reverse, sort } from 'ramda';
+import { repeat, reverse } from 'ramda';
 import { escape } from 'sqlstring';

 import { completeTimeline, round } from '@openpanel/common';
 import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';
 import {
  chQuery,
-  convertClickhouseDateToJs,
  createSqlBuilder,
  formatClickhouseDate,
  getChartSql,
@@ -44,7 +41,7 @@ export interface ResultItem {
 }

 function getEventLegend(event: IChartEvent) {
-  return event.displayName ?? `${event.name} (${event.id})`;
+  return event.displayName ?? event.name;
 }

 export function withFormula(
@@ -68,12 +65,16 @@ export function withFormula(

  if (events.length === 1) {
    return series.map((serie) => {
+      if (!serie.event.id) {
+        return serie;
+      }
+
      return {
        ...serie,
        data: serie.data.map((item) => {
          serie.event.id;
          const scope = {
-            [serie.event.id]: item?.count ?? 0,
+            [serie.event.id ?? '']: item?.count ?? 0,
          };
          const count = mathjs
            .parse(formula)
@@ -97,6 +98,10 @@ export function withFormula(
      ...series[0],
      data: series[0].data.map((item, dIndex) => {
        const scope = series.reduce((acc, item) => {
+          if (!item.event.id) {
+            return acc;
+          }
+
          return {
            ...acc,
            [item.event.id]: item.data[dIndex]?.count ?? 0,
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -205,154 +205,156 @@ export const chartRouter = createTRPCRouter({
      }
    }

-    const currentPeriod = getChartStartEndDate(input);
-    const previousPeriod = getChartPrevStartEndDate({
-      range: input.range,
-      ...currentPeriod,
-    });
-
-    const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];
-
-    if (input.previous) {
-      promises.push(
-        getSeriesFromEvents({
-          ...input,
-          ...previousPeriod,
-        })
-      );
-    }
-
-    const result = await Promise.all(promises);
-    const series = result[0]!;
-    const previousSeries = result[1];
-
-    const final: FinalChart = {
-      events: input.events,
-      series: series.map((serie, index) => {
-        const previousSerie = previousSeries?.find(
-          (item) => item.name === serie.name
-        );
-        const metrics = {
-          sum: sum(serie.data.map((item) => item.count)),
-          average: round(average(serie.data.map((item) => item.count)), 2),
-          min: min(serie.data.map((item) => item.count)),
-          max: max(serie.data.map((item) => item.count)),
-        };
-
-        return {
-          id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend
-          name: serie.name,
-          event: {
-            ...serie.event,
-            displayName: serie.event.displayName ?? serie.event.name,
-          },
-          metrics: {
-            ...metrics,
-            previous: {
-              sum: getPreviousMetric(
-                metrics.sum,
-                previousSerie
-                  ? sum(previousSerie?.data.map((item) => item.count))
-                  : null
-              ),
-              average: getPreviousMetric(
-                metrics.average,
-                previousSerie
-                  ? round(
-                      average(previousSerie?.data.map((item) => item.count)),
-                      2
-                    )
-                  : null
-              ),
-              min: getPreviousMetric(
-                metrics.sum,
-                previousSerie
-                  ? min(previousSerie?.data.map((item) => item.count))
-                  : null
-              ),
-              max: getPreviousMetric(
-                metrics.sum,
-                previousSerie
-                  ? max(previousSerie?.data.map((item) => item.count))
-                  : null
-              ),
-            },
-          },
-          data: serie.data.map((item, index) => ({
-            date: item.date,
-            count: item.count ?? 0,
-            label: item.label,
-            previous: previousSerie?.data[index]
-              ? getPreviousMetric(
-                  item.count ?? 0,
-                  previousSerie?.data[index]?.count ?? null
-                )
-              : null,
-          })),
-        };
-      }),
-      metrics: {
-        sum: 0,
-        average: 0,
-        min: 0,
-        max: 0,
-        previous: {
-          sum: null,
-          average: null,
-          min: null,
-          max: null,
-        },
-      },
-    };
-
-    final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
-    final.metrics.average = round(
-      average(final.series.map((item) => item.metrics.average)),
-      2
-    );
-    final.metrics.min = min(final.series.map((item) => item.metrics.min));
-    final.metrics.max = max(final.series.map((item) => item.metrics.max));
-    final.metrics.previous = {
-      sum: getPreviousMetric(
-        final.metrics.sum,
-        sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))
-      ),
-      average: getPreviousMetric(
-        final.metrics.average,
-        round(
-          average(
-            final.series.map(
-              (item) => item.metrics.previous.average?.value ?? 0
-            )
-          ),
-          2
-        )
-      ),
-      min: getPreviousMetric(
-        final.metrics.min,
-        min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))
-      ),
-      max: getPreviousMetric(
-        final.metrics.max,
-        max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))
-      ),
-    };
-
-    // Sort by sum
-    final.series = final.series.sort((a, b) => {
-      if (input.chartType === 'linear') {
-        const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
-        const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
-        return sumB - sumA;
-      } else {
-        return b.metrics[input.metric] - a.metrics[input.metric];
-      }
-    });
-
-    return final;
+    return getChart(input);
  }),
 });

+export async function getChart(input: IChartInput) {
+  const currentPeriod = getChartStartEndDate(input);
+  const previousPeriod = getChartPrevStartEndDate({
+    range: input.range,
+    ...currentPeriod,
+  });
+
+  const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];
+
+  if (input.previous) {
+    promises.push(
+      getSeriesFromEvents({
+        ...input,
+        ...previousPeriod,
+      })
+    );
+  }
+
+  const result = await Promise.all(promises);
+  const series = result[0]!;
+  const previousSeries = result[1];
+
+  const final: FinalChart = {
+    events: input.events,
+    series: series.map((serie) => {
+      const previousSerie = previousSeries?.find(
+        (item) => item.name === serie.name
+      );
+      const metrics = {
+        sum: sum(serie.data.map((item) => item.count)),
+        average: round(average(serie.data.map((item) => item.count)), 2),
+        min: min(serie.data.map((item) => item.count)),
+        max: max(serie.data.map((item) => item.count)),
+      };
+
+      return {
+        id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend
+        name: serie.name,
+        event: {
+          ...serie.event,
+          displayName: serie.event.displayName ?? serie.event.name,
+        },
+        metrics: {
+          ...metrics,
+          previous: {
+            sum: getPreviousMetric(
+              metrics.sum,
+              previousSerie
+                ? sum(previousSerie?.data.map((item) => item.count))
+                : null
+            ),
+            average: getPreviousMetric(
+              metrics.average,
+              previousSerie
+                ? round(
+                    average(previousSerie?.data.map((item) => item.count)),
+                    2
+                  )
+                : null
+            ),
+            min: getPreviousMetric(
+              metrics.sum,
+              previousSerie
+                ? min(previousSerie?.data.map((item) => item.count))
+                : null
+            ),
+            max: getPreviousMetric(
+              metrics.sum,
+              previousSerie
+                ? max(previousSerie?.data.map((item) => item.count))
+                : null
+            ),
+          },
+        },
+        data: serie.data.map((item, index) => ({
+          date: item.date,
+          count: item.count ?? 0,
+          label: item.label,
+          previous: previousSerie?.data[index]
+            ? getPreviousMetric(
+                item.count ?? 0,
+                previousSerie?.data[index]?.count ?? null
+              )
+            : null,
+        })),
+      };
+    }),
+    metrics: {
+      sum: 0,
+      average: 0,
+      min: 0,
+      max: 0,
+      previous: {
+        sum: null,
+        average: null,
+        min: null,
+        max: null,
+      },
+    },
+  };
+
+  final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
+  final.metrics.average = round(
+    average(final.series.map((item) => item.metrics.average)),
+    2
+  );
+  final.metrics.min = min(final.series.map((item) => item.metrics.min));
+  final.metrics.max = max(final.series.map((item) => item.metrics.max));
+  final.metrics.previous = {
+    sum: getPreviousMetric(
+      final.metrics.sum,
+      sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))
+    ),
+    average: getPreviousMetric(
+      final.metrics.average,
+      round(
+        average(
+          final.series.map((item) => item.metrics.previous.average?.value ?? 0)
+        ),
+        2
+      )
+    ),
+    min: getPreviousMetric(
+      final.metrics.min,
+      min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))
+    ),
+    max: getPreviousMetric(
+      final.metrics.max,
+      max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))
+    ),
+  };
+
+  // Sort by sum
+  final.series = final.series.sort((a, b) => {
+    if (input.chartType === 'linear') {
+      const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
+      const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
+      return sumB - sumA;
+    } else {
+      return b.metrics[input.metric] - a.metrics[input.metric];
+    }
+  });
+
+  return final;
+}
+
 export function getPreviousMetric(
  current: number,
  previous: number | null
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -19,7 +19,7 @@ export function objectToZodEnums<K extends string>(
 export const mapKeys = objectToZodEnums;

 export const zChartEvent = z.object({
-  id: z.string(),
+  id: z.string().optional(),
  name: z.string(),
  displayName: z.string().optional(),
  property: z.string().optional(),
@@ -34,7 +34,7 @@ export const zChartEvent = z.object({
  ]),
  filters: z.array(
    z.object({
-      id: z.string(),
+      id: z.string().optional(),
      name: z.string(),
      operator: z.enum(objectToZodEnums(operators)),
      value: z.array(z.string().or(z.number()).or(z.boolean()).or(z.null())),
@@ -42,7 +42,7 @@ export const zChartEvent = z.object({
  ),
 });
 export const zChartBreakdown = z.object({
-  id: z.string(),
+  id: z.string().optional(),
  name: z.string(),
 });

--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -59,6 +59,9 @@ importers:
      '@openpanel/trpc':
        specifier: workspace:*
        version: link:../../packages/trpc
+      '@openpanel/validation':
+        specifier: workspace:*
+        version: link:../../packages/validation
      '@trpc/server':
        specifier: ^10.45.1
        version: 10.45.1
@@ -101,6 +104,9 @@ importers:
      url-metadata:
        specifier: ^4.1.0
        version: 4.1.0
+      zod:
+        specifier: ^3.22.4
+        version: 3.22.4
    devDependencies:
      '@openpanel/eslint-config':
        specifier: workspace:*