feat:add otel logging

packages/trpc/src/root.ts

@@ -10,6 +10,7 @@ import { gscRouter } from './routers/gsc';
 import { importRouter } from './routers/import';
 import { insightRouter } from './routers/insight';
 import { integrationRouter } from './routers/integration';
+import { logRouter } from './routers/log';
 import { notificationRouter } from './routers/notification';
 import { onboardingRouter } from './routers/onboarding';
 import { organizationRouter } from './routers/organization';
@@ -57,6 +58,7 @@ export const appRouter = createTRPCRouter({
   email: emailRouter,
   gsc: gscRouter,
   group: groupRouter,
+  log: logRouter,
 });
 
 // export type definition of API

packages/trpc/src/routers/log.ts

@@ -0,0 +1,212 @@
+import { chQuery, convertClickhouseDateToJs } from '@openpanel/db';
+import { zSeverityText } from '@openpanel/validation';
+import sqlstring from 'sqlstring';
+import { z } from 'zod';
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export interface IServiceLog {
+  id: string;
+  projectId: string;
+  deviceId: string;
+  profileId: string;
+  sessionId: string;
+  timestamp: Date;
+  severityNumber: number;
+  severityText: string;
+  body: string;
+  traceId: string;
+  spanId: string;
+  traceFlags: number;
+  loggerName: string;
+  attributes: Record<string, string>;
+  resource: Record<string, string>;
+  sdkName: string;
+  sdkVersion: string;
+  country: string;
+  city: string;
+  region: string;
+  os: string;
+  osVersion: string;
+  browser: string;
+  browserVersion: string;
+  device: string;
+  brand: string;
+  model: string;
+}
+
+interface IClickhouseLog {
+  id: string;
+  project_id: string;
+  device_id: string;
+  profile_id: string;
+  session_id: string;
+  timestamp: string;
+  severity_number: number;
+  severity_text: string;
+  body: string;
+  trace_id: string;
+  span_id: string;
+  trace_flags: number;
+  logger_name: string;
+  attributes: Record<string, string>;
+  resource: Record<string, string>;
+  sdk_name: string;
+  sdk_version: string;
+  country: string;
+  city: string;
+  region: string;
+  os: string;
+  os_version: string;
+  browser: string;
+  browser_version: string;
+  device: string;
+  brand: string;
+  model: string;
+}
+
+function toServiceLog(row: IClickhouseLog): IServiceLog {
+  return {
+    id: row.id,
+    projectId: row.project_id,
+    deviceId: row.device_id,
+    profileId: row.profile_id,
+    sessionId: row.session_id,
+    timestamp: convertClickhouseDateToJs(row.timestamp),
+    severityNumber: row.severity_number,
+    severityText: row.severity_text,
+    body: row.body,
+    traceId: row.trace_id,
+    spanId: row.span_id,
+    traceFlags: row.trace_flags,
+    loggerName: row.logger_name,
+    attributes: row.attributes,
+    resource: row.resource,
+    sdkName: row.sdk_name,
+    sdkVersion: row.sdk_version,
+    country: row.country,
+    city: row.city,
+    region: row.region,
+    os: row.os,
+    osVersion: row.os_version,
+    browser: row.browser,
+    browserVersion: row.browser_version,
+    device: row.device,
+    brand: row.brand,
+    model: row.model,
+  };
+}
+
+export const logRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        cursor: z.string().nullish(),
+        severity: z.array(zSeverityText).optional(),
+        search: z.string().optional(),
+        loggerName: z.string().optional(),
+        startDate: z.date().optional(),
+        endDate: z.date().optional(),
+        take: z.number().default(50),
+      }),
+    )
+    .query(async ({ input }) => {
+      const { projectId, cursor, severity, search, loggerName, startDate, endDate, take } = input;
+
+      const conditions: string[] = [
+        `project_id = ${sqlstring.escape(projectId)}`,
+      ];
+
+      if (cursor) {
+        conditions.push(`timestamp < ${sqlstring.escape(cursor)}`);
+      }
+
+      if (severity && severity.length > 0) {
+        const escaped = severity.map((s) => sqlstring.escape(s)).join(', ');
+        conditions.push(`severity_text IN (${escaped})`);
+      }
+
+      if (search) {
+        conditions.push(`body ILIKE ${sqlstring.escape(`%${search}%`)}`);
+      }
+
+      if (loggerName) {
+        conditions.push(`logger_name = ${sqlstring.escape(loggerName)}`);
+      }
+
+      if (startDate) {
+        conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
+      }
+
+      if (endDate) {
+        conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
+      }
+
+      const where = conditions.join(' AND ');
+
+      const rows = await chQuery<IClickhouseLog>(
+        `SELECT
+          id, project_id, device_id, profile_id, session_id,
+          timestamp, severity_number, severity_text, body,
+          trace_id, span_id, trace_flags, logger_name,
+          attributes, resource,
+          sdk_name, sdk_version,
+          country, city, region, os, os_version,
+          browser, browser_version, device, brand, model
+        FROM logs
+        WHERE ${where}
+        ORDER BY timestamp DESC
+        LIMIT ${take + 1}`,
+      );
+
+      const hasMore = rows.length > take;
+      const data = rows.slice(0, take).map(toServiceLog);
+      const lastItem = data[data.length - 1];
+
+      return {
+        data,
+        meta: {
+          next: hasMore && lastItem ? lastItem.timestamp.toISOString() : null,
+        },
+      };
+    }),
+
+  severityCounts: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        startDate: z.date().optional(),
+        endDate: z.date().optional(),
+      }),
+    )
+    .query(async ({ input }) => {
+      const { projectId, startDate, endDate } = input;
+
+      const conditions: string[] = [
+        `project_id = ${sqlstring.escape(projectId)}`,
+      ];
+
+      if (startDate) {
+        conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
+      }
+
+      if (endDate) {
+        conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
+      }
+
+      const where = conditions.join(' AND ');
+
+      const rows = await chQuery<{ severity_text: string; count: number }>(
+        `SELECT severity_text, count() AS count
+         FROM logs
+         WHERE ${where}
+         GROUP BY severity_text
+         ORDER BY count DESC`,
+      );
+
+      return rows.reduce<Record<string, number>>((acc, row) => {
+        acc[row.severity_text] = row.count;
+        return acc;
+      }, {});
+    }),
+});