a lot

packages/db/clickhouse_tables.sql

@@ -0,0 +1,46 @@
+CREATE TABLE test.events (
+  `name` String,
+  `profile_id` String,
+  `project_id` String,
+  -- the route
+  `path` String,
+  `utm_source` String,
+  `utm_medium` String,
+  `utm_campaign` String,
+  `utm_term` String,
+  `utm_content` String,
+  `referrer` String,
+  `referrer_name` String,
+  `duration` UInt64,
+  `properties` Map(String, String),
+  `created_at` DateTime64(3),
+  `country` String,
+  `city` String,
+  `region` String,
+  `os` String,
+  `os_version` String,
+  `browser` String,
+  `browser_version` String,
+  -- device: mobile/desktop/tablet
+  `device` String,
+  -- brand: (Samsung, OnePlus)
+  `brand` String,
+  -- model: (Samsung Galaxy, iPhone X)
+  `model` String
+) ENGINE MergeTree
+ORDER BY
+  (project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
+
+CREATE TABLE test.profiles (
+  `id` String,
+  `external_id` String,
+  `first_name` String,
+  `last_name` String,
+  `email` String,
+  `avatar` String,
+  `properties` Map(String, String),
+  `project_id` String,
+  `created_at` DateTime
+) ENGINE = ReplacingMergeTree
+ORDER BY
+  (id) SETTINGS index_granularity = 8192;

packages/db/index.ts

@@ -1,15 +1,6 @@
-import { PrismaClient } from '@prisma/client';
-
-export * from '@prisma/client';
-
-const globalForPrisma = globalThis as unknown as {
-  prisma: PrismaClient | undefined;
-};
-
-export const db =
-  globalForPrisma.prisma ??
-  new PrismaClient({
-    log: ['error'],
-  });
-
-if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = db;
+export * from './src/prisma-client';
+export * from './src/prisma-types';
+export * from './src/clickhouse-client';
+export * from './src/sql-builder';
+export * from './src/services/salt';
+export * from './src/services/event.service';

packages/db/package.json

@@ -12,7 +12,10 @@
     "with-env": "dotenv -e ../../.env -c --"
   },
   "dependencies": {
-    "@prisma/client": "^5.1.1"
+    "@mixan/common": "workspace:*",
+    "@clickhouse/client": "^0.2.9",
+    "@prisma/client": "^5.1.1",
+    "ramda": "^0.29.1"
   },
   "devDependencies": {
     "@mixan/eslint-config": "workspace:*",
@@ -20,6 +23,7 @@
     "@mixan/tsconfig": "workspace:*",
     "@mixan/types": "workspace:*",
     "@types/node": "^18.16.0",
+    "@types/ramda": "^0.29.6",
     "eslint": "^8.48.0",
     "prettier": "^3.0.3",
     "prisma": "^5.1.1",

packages/db/prisma/migrations/20240122191559_remove_foreigen_key_profile_events/migration.sql

@@ -0,0 +1,17 @@
+/*
+  Warnings:
+
+  - The primary key for the `profiles` table will be changed. If it partially fails, the table could be left without primary key constraint.
+
+*/
+-- DropForeignKey
+ALTER TABLE "events" DROP CONSTRAINT "events_profile_id_fkey";
+
+-- AlterTable
+ALTER TABLE "events" ALTER COLUMN "profile_id" SET DATA TYPE TEXT;
+
+-- AlterTable
+ALTER TABLE "profiles" DROP CONSTRAINT "profiles_pkey",
+ALTER COLUMN "id" DROP DEFAULT,
+ALTER COLUMN "id" SET DATA TYPE TEXT,
+ADD CONSTRAINT "profiles_pkey" PRIMARY KEY ("id");

packages/db/prisma/migrations/20240129163925_add_salts/migration.sql

@@ -0,0 +1,8 @@
+-- CreateTable
+CREATE TABLE "salts" (
+    "salt" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "salts_pkey" PRIMARY KEY ("salt")
+);

packages/db/prisma/migrations/20240131204540_add_formula/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "reports" ADD COLUMN     "formula" TEXT;

packages/db/prisma/migrations/20240131212106_add_metric/migration.sql

@@ -0,0 +1,6 @@
+-- CreateEnum
+CREATE TYPE "Metric" AS ENUM ('sum', 'average', 'min', 'max');
+
+-- AlterTable
+ALTER TABLE "reports" ADD COLUMN     "metric" "Metric" NOT NULL DEFAULT 'sum',
+ADD COLUMN     "unit" TEXT;

packages/db/prisma/migrations/20240202175049_add_events_count_on_project/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN     "eventsCount" INTEGER NOT NULL DEFAULT 0;

packages/db/prisma/schema.prisma

@@ -30,6 +30,7 @@ model Project {
   organization_id String
   organization    Organization @relation(fields: [organization_id], references: [id])
   events          Event[]
+  eventsCount     Int          @default(0)
   profiles        Profile[]
   clients         Client[]
 
@@ -64,8 +65,7 @@ model Event {
   project_id String
   project    Project @relation(fields: [project_id], references: [id])
 
-  profile_id String?  @db.Uuid
-  profile    Profile? @relation(fields: [profile_id], references: [id])
+  profile_id String?
 
   createdAt DateTime @default(now())
   updatedAt DateTime @default(now()) @updatedAt
@@ -73,8 +73,16 @@ model Event {
   @@map("events")
 }
 
+model Salt {
+  salt      String   @id
+  createdAt DateTime @default(now())
+  updatedAt DateTime @default(now()) @updatedAt
+
+  @@map("salts")
+}
+
 model Profile {
-  id          String  @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+  id          String   @id
   external_id String?
   first_name  String?
   last_name   String?
@@ -82,11 +90,9 @@ model Profile {
   avatar      String?
   properties  Json
   project_id  String
-  project     Project @relation(fields: [project_id], references: [id])
-  events      Event[]
-
-  createdAt DateTime @default(now())
-  updatedAt DateTime @default(now()) @updatedAt
+  project     Project  @relation(fields: [project_id], references: [id])
+  createdAt   DateTime @default(now())
+  updatedAt   DateTime @default(now()) @updatedAt
 
   @@map("profiles")
 }
@@ -160,6 +166,13 @@ model Dashboard {
   @@map("dashboards")
 }
 
+enum Metric {
+  sum
+  average
+  min
+  max
+}
+
 model Report {
   id         String    @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
   name       String
@@ -169,6 +182,9 @@ model Report {
   line_type  String    @default("monotone")
   breakdowns Json
   events     Json
+  formula    String?
+  unit       String?
+  metric     Metric    @default(sum)
   project_id String
   project    Project   @relation(fields: [project_id], references: [id])
   previous   Boolean   @default(false)

packages/db/scripts/insert.ts

@@ -0,0 +1,163 @@
+// @ts-nocheck
+
+import { createEvent } from '@/services/event.service';
+import { last, omit } from 'ramda';
+
+import type { Event } from '../src/prisma-client';
+import { db } from '../src/prisma-client';
+
+async function push(event: Event) {
+  if (event.properties.ip && Number.isNaN(parseInt(event.properties.ip[0]))) {
+    return console.log('IGNORE', event.id);
+  }
+  await fetch('http://localhost:3030/api/event', {
+    method: 'POST',
+    body: JSON.stringify({
+      name: event.name,
+      timestamp: event.createdAt.toISOString(),
+      path: event.properties.path,
+      properties: omit(
+        [
+          'ip',
+          'os',
+          'ua',
+          'url',
+          'hash',
+          'host',
+          'path',
+          'device',
+          'screen',
+          'hostname',
+          'language',
+          'referrer',
+          'timezone',
+        ],
+        event.properties
+      ),
+      referrer: event.properties?.referrer?.host ?? undefined,
+    }),
+    headers: {
+      'Content-Type': 'application/json',
+      'User-Agent': event.properties.ua,
+      'X-Forwarded-For': event.properties.ip,
+      'mixan-client-id': 'c8b4962e-bc3d-4b23-8ea4-505c8fbdf09e',
+      origin: 'https://mixan.kiddo.se',
+    },
+  }).catch(() => {});
+}
+
+async function main() {
+  const events = await db.event.findMany({
+    where: {
+      project_id: '4e2798cb-e255-4e9d-960d-c9ad095aabd7',
+      name: 'screen_view',
+      createdAt: {
+        gte: new Date('2024-01-14'),
+        lt: new Date('2024-01-18'),
+      },
+    },
+    orderBy: {
+      createdAt: 'asc',
+    },
+  });
+
+  const grouped: Record<string, Event[]> = {};
+  let index = 0;
+  for (const event of events.slice()) {
+    console.log(index, event.name, event.createdAt.toISOString());
+    index++;
+
+    if (event.properties.ua?.includes('bot')) {
+      console.log('IGNORE', event.id);
+      continue;
+    }
+
+    if (grouped[event.profile_id]) {
+      grouped[event.profile_id].push(event);
+    } else {
+      grouped[event.profile_id] = [event];
+    }
+  }
+
+  for (const profile_id of Object.keys(grouped).slice(0, 10)) {
+    const events = grouped[profile_id];
+
+    if (events) {
+      console.log('new user...');
+      let eidx = -1;
+      for (const event of events) {
+        eidx++;
+        const lastEventAt = events[eidx - 1]?.createdAt;
+
+        const profileId: string | null = null;
+        const projectId = event.project_id;
+        const path = event.properties.path as string;
+        const ip = event.properties.ip as string;
+        const origin = 'https://mixan.kiddo.se';
+        const ua = event.properties.ua as string;
+        const uaInfo = parseUserAgent(ua);
+        const salts = await getSalts();
+
+        const [currentProfileId, previousProfileId, geo, eventsJobs] =
+          await Promise.all([
+            generateProfileId({
+              salt: salts.current,
+              origin,
+              ip,
+              ua,
+            }),
+            generateProfileId({
+              salt: salts.previous,
+              origin,
+              ip,
+              ua,
+            }),
+            parseIp(ip),
+            eventsQueue.getJobs(['delayed']),
+          ]);
+        const payload: IServiceCreateEventPayload = {
+          name: body.name,
+          profileId,
+          projectId,
+          properties: body.properties,
+          createdAt: body.timestamp,
+          country: geo.country,
+          city: geo.city,
+          region: geo.region,
+          continent: geo.continent,
+          os: uaInfo.os,
+          osVersion: uaInfo.osVersion,
+          browser: uaInfo.browser,
+          browserVersion: uaInfo.browserVersion,
+          device: uaInfo.device,
+          brand: uaInfo.brand,
+          model: uaInfo.model,
+          duration: 0,
+          path,
+          referrer: body.referrer, // TODO
+          referrerName: body.referrer, // TODO
+        };
+
+        if (!lastEventAt) {
+          createEvent({});
+          continue;
+        }
+
+        if (
+          event.createdAt.getTime() - lastEventAt.getTime() >
+          1000 * 60 * 30
+        ) {
+          console.log(
+            'new Session?',
+            event.createdAt.toISOString(),
+            event.properties.path
+          );
+        } else {
+          console.log('Same session?');
+        }
+      }
+    }
+  }
+}
+
+main();

packages/db/src/clickhouse-client.ts

@@ -0,0 +1,51 @@
+import { createClient } from '@clickhouse/client';
+
+export const ch = createClient({
+  host: process.env.CLICKHOUSE_URL,
+  username: process.env.CLICKHOUSE_USER,
+  password: process.env.CLICKHOUSE_PASSWORD,
+  database: process.env.CLICKHOUSE_DB,
+});
+
+interface ClickhouseJsonResponse<T> {
+  data: T[];
+  rows: number;
+  statistics: { elapsed: number; rows_read: number; bytes_read: number };
+  meta: { name: string; type: string }[];
+}
+
+export async function chQueryAll<T extends Record<string, any>>(
+  query: string
+): Promise<ClickhouseJsonResponse<T>> {
+  const res = await ch.query({
+    query,
+  });
+  const json = await res.json<ClickhouseJsonResponse<T>>();
+  return {
+    ...json,
+    data: json.data.map((item) => {
+      const keys = Object.keys(item);
+      return keys.reduce((acc, key) => {
+        const meta = json.meta.find((m) => m.name === key);
+        return {
+          ...acc,
+          [key]:
+            item[key] && meta?.type.includes('Int')
+              ? parseFloat(item[key] as string)
+              : item[key],
+        };
+      }, {} as T);
+    }),
+  };
+}
+
+export async function chQuery<T extends Record<string, any>>(
+  query: string
+): Promise<T[]> {
+  return (await chQueryAll<T>(query)).data;
+}
+
+export function formatClickhouseDate(_date: Date | string) {
+  const date = typeof _date === 'string' ? new Date(_date) : _date;
+  return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
+}

packages/db/src/prisma-client.ts

@@ -0,0 +1,15 @@
+import { PrismaClient } from '@prisma/client';
+
+export * from '@prisma/client';
+
+const globalForPrisma = globalThis as unknown as {
+  prisma: PrismaClient | undefined;
+};
+
+export const db =
+  globalForPrisma.prisma ??
+  new PrismaClient({
+    log: ['error'],
+  });
+
+if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = db;

packages/db/src/prisma-types.ts

@@ -0,0 +1,22 @@
+/* eslint-disable */
+
+export type IDBEvent = {
+  id: string;
+  name: string;
+  profile_id?: string;
+  project_id: string;
+  properties: Record<string, string>;
+  created_at: string;
+};
+
+export type IDBProfile = {
+  id: string;
+  external_id?: string;
+  first_name?: string;
+  last_name?: string;
+  email?: string;
+  avatar?: string;
+  properties: Record<string, string>;
+  project_id: String;
+  created_at: string;
+};

packages/db/src/services/event.service.ts

@@ -0,0 +1,113 @@
+import { omit } from 'ramda';
+
+import { toDots } from '@mixan/common';
+
+import { ch, chQuery, formatClickhouseDate } from '../clickhouse-client';
+
+export interface IClickhouseEvent {
+  name: string;
+  profile_id: string;
+  project_id: string;
+  path: string;
+  referrer: string;
+  referrer_name: string;
+  duration: number;
+  properties: Record<string, string>;
+  created_at: string;
+  country: string;
+  city: string;
+  region: string;
+  os: string;
+  os_version: string;
+  browser: string;
+  browser_version: string;
+  device: string;
+  brand: string;
+  model: string;
+}
+
+export function transformEvent(
+  event: IClickhouseEvent
+): IServiceCreateEventPayload {
+  return {
+    name: event.name,
+    profileId: event.profile_id,
+    projectId: event.project_id,
+    properties: event.properties,
+    createdAt: event.created_at,
+    country: event.country,
+    city: event.city,
+    region: event.region,
+    os: event.os,
+    osVersion: event.os_version,
+    browser: event.browser,
+    browserVersion: event.browser_version,
+    device: event.device,
+    brand: event.brand,
+    model: event.model,
+    duration: event.duration,
+    path: event.path,
+    referrer: event.referrer,
+    referrerName: event.referrer_name,
+  };
+}
+
+export interface IServiceCreateEventPayload {
+  name: string;
+  profileId: string;
+  projectId: string;
+  properties: Record<string, unknown>;
+  createdAt: string;
+  country?: string | undefined;
+  city?: string | undefined;
+  region?: string | undefined;
+  continent?: string | undefined;
+  os?: string | undefined;
+  osVersion?: string | undefined;
+  browser?: string | undefined;
+  browserVersion?: string | undefined;
+  device?: string | undefined;
+  brand?: string | undefined;
+  model?: string | undefined;
+  duration: number;
+  path: string;
+  referrer: string | undefined;
+  referrerName: string | undefined;
+}
+
+export function getEvents(sql: string) {
+  return chQuery<IClickhouseEvent>(sql).then((events) =>
+    events.map(transformEvent)
+  );
+}
+
+export async function createEvent(payload: IServiceCreateEventPayload) {
+  console.log(`create event ${payload.name} for ${payload.profileId}`);
+
+  return ch.insert({
+    table: 'events',
+    values: [
+      {
+        name: payload.name,
+        profile_id: payload.profileId,
+        project_id: payload.projectId,
+        properties: toDots(omit(['_path'], payload.properties)),
+        path: payload.path ?? '',
+        created_at: formatClickhouseDate(payload.createdAt),
+        country: payload.country ?? '',
+        city: payload.city ?? '',
+        region: payload.region ?? '',
+        os: payload.os ?? '',
+        os_version: payload.osVersion ?? '',
+        browser: payload.browser ?? '',
+        browser_version: payload.browserVersion ?? '',
+        device: payload.device ?? '',
+        brand: payload.brand ?? '',
+        model: payload.model ?? '',
+        duration: payload.duration,
+        referrer: payload.referrer ?? '',
+      },
+    ],
+    format: 'JSONEachRow',
+  });
+}

packages/db/src/services/salt.ts

@@ -0,0 +1,37 @@
+import { db } from '../prisma-client';
+
+export async function getCurrentSalt() {
+  const salt = await db.salt.findFirst({
+    orderBy: {
+      createdAt: 'desc',
+    },
+  });
+
+  if (!salt) {
+    throw new Error('No salt found');
+  }
+
+  return salt.salt;
+}
+
+export async function getSalts() {
+  const [curr, prev] = await db.salt.findMany({
+    orderBy: {
+      createdAt: 'desc',
+    },
+    take: 2,
+  });
+
+  if (!curr) {
+    throw new Error('No salt found');
+  }
+
+  if (!prev) {
+    throw new Error('No previous salt found');
+  }
+
+  return {
+    current: curr.salt,
+    previous: prev.salt,
+  };
+}

packages/db/src/sql-builder.ts

@@ -0,0 +1,56 @@
+export function createSqlBuilder() {
+  const join = (obj: Record<string, string> | string[], joiner: string) =>
+    Object.values(obj).filter(Boolean).join(joiner);
+
+  const sb: {
+    where: Record<string, string>;
+    select: Record<string, string>;
+    groupBy: Record<string, string>;
+    orderBy: Record<string, string>;
+    from: string;
+    limit: number | undefined;
+    offset: number | undefined;
+  } = {
+    where: {},
+    from: 'events',
+    select: {},
+    groupBy: {},
+    orderBy: {},
+    limit: undefined,
+    offset: undefined,
+  };
+
+  const getWhere = () =>
+    Object.keys(sb.where).length ? 'WHERE ' + join(sb.where, ' AND ') : '';
+  const getFrom = () => `FROM ${sb.from}`;
+  const getSelect = () =>
+    'SELECT ' + (Object.keys(sb.select).length ? join(sb.select, ', ') : '*');
+  const getGroupBy = () =>
+    Object.keys(sb.groupBy).length ? 'GROUP BY ' + join(sb.groupBy, ', ') : '';
+  const getOrderBy = () =>
+    Object.keys(sb.orderBy).length ? 'ORDER BY ' + join(sb.orderBy, ', ') : '';
+  const getLimit = () => (sb.limit ? `LIMIT ${sb.limit}` : '');
+  const getOffset = () => (sb.offset ? `OFFSET ${sb.offset}` : '');
+
+  return {
+    sb,
+    join,
+    getWhere,
+    getFrom,
+    getSelect,
+    getGroupBy,
+    getOrderBy,
+    getSql: () =>
+      [
+        getSelect(),
+        getFrom(),
+        getWhere(),
+        getGroupBy(),
+        getOrderBy(),
+        getLimit(),
+        getOffset(),
+      ]
+        .filter(Boolean)
+        .join(' '),
+  };
+}