a lot

packages/common/index.ts

@@ -0,0 +1,4 @@
+export * from './src/crypto';
+export * from './src/profileId';
+export * from './src/date';
+export * from './src/object';

packages/common/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@mixan/common",
+  "version": "0.0.1",
+  "main": "index.ts",
+  "scripts": {
+    "lint": "eslint .",
+    "format": "prettier --check \"**/*.{mjs,ts,md,json}\"",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "ramda": "^0.29.1"
+  },
+  "devDependencies": {
+    "@mixan/eslint-config": "workspace:*",
+    "@mixan/prettier-config": "workspace:*",
+    "@mixan/tsconfig": "workspace:*",
+    "@mixan/types": "workspace:*",
+    "@types/node": "^18.16.0",
+    "@types/ramda": "^0.29.6",
+    "eslint": "^8.48.0",
+    "prettier": "^3.0.3",
+    "prisma": "^5.1.1",
+    "typescript": "^5.2.2"
+  },
+  "eslintConfig": {
+    "root": true,
+    "extends": [
+      "@mixan/eslint-config/base"
+    ]
+  },
+  "prettier": "@mixan/prettier-config"
+}

packages/common/src/crypto.ts

@@ -0,0 +1,51 @@
+import { randomBytes, scrypt, timingSafeEqual } from 'crypto';
+
+export function generateSalt() {
+  return randomBytes(16).toString('hex');
+}
+
+/**
+ * Has a password or a secret with a password hashing algorithm (scrypt)
+ * @param {string} password
+ * @returns {string} The salt+hash
+ */
+export async function hashPassword(
+  password: string,
+  _salt?: string,
+  keyLength = 32
+): Promise<string> {
+  return new Promise((resolve, reject) => {
+    // generate random 16 bytes long salt - recommended by NodeJS Docs
+    const salt = _salt || generateSalt();
+    scrypt(password, salt, keyLength, (err, derivedKey) => {
+      if (err) reject(err);
+      // derivedKey is of type Buffer
+      resolve(`${salt}.${derivedKey.toString('hex')}`);
+    });
+  });
+}
+
+/**
+ * Compare a plain text password with a salt+hash password
+ * @param {string} password The plain text password
+ * @param {string} hash The hash+salt to check against
+ * @returns {boolean}
+ */
+export async function verifyPassword(
+  password: string,
+  hash: string,
+  keyLength = 32
+): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const [salt, hashKey] = hash.split('.');
+    // we need to pass buffer values to timingSafeEqual
+    const hashKeyBuff = Buffer.from(hashKey!, 'hex');
+    scrypt(password, salt!, keyLength, (err, derivedKey) => {
+      if (err) {
+        reject(err);
+      }
+      // compare the new supplied password with the hashed password using timeSafeEqual
+      resolve(timingSafeEqual(hashKeyBuff, derivedKey));
+    });
+  });
+}

packages/common/src/date.ts

@@ -0,0 +1,7 @@
+export function getTime(date: string | number) {
+  return new Date(date).getTime();
+}
+
+export function toISOString(date: string | number) {
+  return new Date(date).toISOString();
+}

packages/common/src/object.ts

@@ -0,0 +1,22 @@
+import { anyPass, isEmpty, isNil, reject } from 'ramda';
+
+export function toDots(
+  obj: Record<string, unknown>,
+  path = ''
+): Record<string, number | string | boolean> {
+  return Object.entries(obj).reduce((acc, [key, value]) => {
+    if (typeof value === 'object' && value !== null) {
+      return {
+        ...acc,
+        ...toDots(value as Record<string, unknown>, `${path}${key}.`),
+      };
+    }
+
+    return {
+      ...acc,
+      [`${path}${key}`]: value,
+    };
+  }, {});
+}
+
+export const strip = reject(anyPass([isEmpty, isNil]));

packages/common/src/profileId.ts

@@ -0,0 +1,17 @@
+import { hashPassword } from './crypto';
+
+interface GenerateProfileIdOptions {
+  salt: string;
+  ua: string;
+  ip: string;
+  origin: string;
+}
+
+export async function generateProfileId({
+  salt,
+  ua,
+  ip,
+  origin,
+}: GenerateProfileIdOptions) {
+  return await hashPassword(`${ua}:${ip}:${origin}`, salt, 8);
+}

packages/common/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "@mixan/tsconfig/base.json",
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./src/*"]
+    },
+    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+  },
+  "include": ["."],
+  "exclude": ["node_modules"]
+}

packages/db/clickhouse_tables.sql

@@ -0,0 +1,46 @@
+CREATE TABLE test.events (
+  `name` String,
+  `profile_id` String,
+  `project_id` String,
+  -- the route
+  `path` String,
+  `utm_source` String,
+  `utm_medium` String,
+  `utm_campaign` String,
+  `utm_term` String,
+  `utm_content` String,
+  `referrer` String,
+  `referrer_name` String,
+  `duration` UInt64,
+  `properties` Map(String, String),
+  `created_at` DateTime64(3),
+  `country` String,
+  `city` String,
+  `region` String,
+  `os` String,
+  `os_version` String,
+  `browser` String,
+  `browser_version` String,
+  -- device: mobile/desktop/tablet
+  `device` String,
+  -- brand: (Samsung, OnePlus)
+  `brand` String,
+  -- model: (Samsung Galaxy, iPhone X)
+  `model` String
+) ENGINE MergeTree
+ORDER BY
+  (project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
+
+CREATE TABLE test.profiles (
+  `id` String,
+  `external_id` String,
+  `first_name` String,
+  `last_name` String,
+  `email` String,
+  `avatar` String,
+  `properties` Map(String, String),
+  `project_id` String,
+  `created_at` DateTime
+) ENGINE = ReplacingMergeTree
+ORDER BY
+  (id) SETTINGS index_granularity = 8192;

packages/db/index.ts

@@ -1,15 +1,6 @@
-import { PrismaClient } from '@prisma/client';
-
-export * from '@prisma/client';
-
-const globalForPrisma = globalThis as unknown as {
-  prisma: PrismaClient | undefined;
-};
-
-export const db =
-  globalForPrisma.prisma ??
-  new PrismaClient({
-    log: ['error'],
-  });
-
-if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = db;
+export * from './src/prisma-client';
+export * from './src/prisma-types';
+export * from './src/clickhouse-client';
+export * from './src/sql-builder';
+export * from './src/services/salt';
+export * from './src/services/event.service';

packages/db/package.json

@@ -12,7 +12,10 @@
     "with-env": "dotenv -e ../../.env -c --"
   },
   "dependencies": {
-    "@prisma/client": "^5.1.1"
+    "@mixan/common": "workspace:*",
+    "@clickhouse/client": "^0.2.9",
+    "@prisma/client": "^5.1.1",
+    "ramda": "^0.29.1"
   },
   "devDependencies": {
     "@mixan/eslint-config": "workspace:*",
@@ -20,6 +23,7 @@
     "@mixan/tsconfig": "workspace:*",
     "@mixan/types": "workspace:*",
     "@types/node": "^18.16.0",
+    "@types/ramda": "^0.29.6",
     "eslint": "^8.48.0",
     "prettier": "^3.0.3",
     "prisma": "^5.1.1",

packages/db/prisma/migrations/20240122191559_remove_foreigen_key_profile_events/migration.sql

@@ -0,0 +1,17 @@
+/*
+  Warnings:
+
+  - The primary key for the `profiles` table will be changed. If it partially fails, the table could be left without primary key constraint.
+
+*/
+-- DropForeignKey
+ALTER TABLE "events" DROP CONSTRAINT "events_profile_id_fkey";
+
+-- AlterTable
+ALTER TABLE "events" ALTER COLUMN "profile_id" SET DATA TYPE TEXT;
+
+-- AlterTable
+ALTER TABLE "profiles" DROP CONSTRAINT "profiles_pkey",
+ALTER COLUMN "id" DROP DEFAULT,
+ALTER COLUMN "id" SET DATA TYPE TEXT,
+ADD CONSTRAINT "profiles_pkey" PRIMARY KEY ("id");

packages/db/prisma/migrations/20240129163925_add_salts/migration.sql

@@ -0,0 +1,8 @@
+-- CreateTable
+CREATE TABLE "salts" (
+    "salt" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "salts_pkey" PRIMARY KEY ("salt")
+);

packages/db/prisma/migrations/20240131204540_add_formula/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "reports" ADD COLUMN     "formula" TEXT;

packages/db/prisma/migrations/20240131212106_add_metric/migration.sql

@@ -0,0 +1,6 @@
+-- CreateEnum
+CREATE TYPE "Metric" AS ENUM ('sum', 'average', 'min', 'max');
+
+-- AlterTable
+ALTER TABLE "reports" ADD COLUMN     "metric" "Metric" NOT NULL DEFAULT 'sum',
+ADD COLUMN     "unit" TEXT;

packages/db/prisma/migrations/20240202175049_add_events_count_on_project/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN     "eventsCount" INTEGER NOT NULL DEFAULT 0;

packages/db/prisma/schema.prisma

@@ -30,6 +30,7 @@ model Project {
   organization_id String
   organization    Organization @relation(fields: [organization_id], references: [id])
   events          Event[]
+  eventsCount     Int          @default(0)
   profiles        Profile[]
   clients         Client[]
 
@@ -64,8 +65,7 @@ model Event {
   project_id String
   project    Project @relation(fields: [project_id], references: [id])
 
-  profile_id String?  @db.Uuid
-  profile    Profile? @relation(fields: [profile_id], references: [id])
+  profile_id String?
 
   createdAt DateTime @default(now())
   updatedAt DateTime @default(now()) @updatedAt
@@ -73,8 +73,16 @@ model Event {
   @@map("events")
 }
 
+model Salt {
+  salt      String   @id
+  createdAt DateTime @default(now())
+  updatedAt DateTime @default(now()) @updatedAt
+
+  @@map("salts")
+}
+
 model Profile {
-  id          String  @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+  id          String   @id
   external_id String?
   first_name  String?
   last_name   String?
@@ -82,11 +90,9 @@ model Profile {
   avatar      String?
   properties  Json
   project_id  String
-  project     Project @relation(fields: [project_id], references: [id])
-  events      Event[]
-
-  createdAt DateTime @default(now())
-  updatedAt DateTime @default(now()) @updatedAt
+  project     Project  @relation(fields: [project_id], references: [id])
+  createdAt   DateTime @default(now())
+  updatedAt   DateTime @default(now()) @updatedAt
 
   @@map("profiles")
 }
@@ -160,6 +166,13 @@ model Dashboard {
   @@map("dashboards")
 }
 
+enum Metric {
+  sum
+  average
+  min
+  max
+}
+
 model Report {
   id         String    @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
   name       String
@@ -169,6 +182,9 @@ model Report {
   line_type  String    @default("monotone")
   breakdowns Json
   events     Json
+  formula    String?
+  unit       String?
+  metric     Metric    @default(sum)
   project_id String
   project    Project   @relation(fields: [project_id], references: [id])
   previous   Boolean   @default(false)

packages/db/scripts/insert.ts

@@ -0,0 +1,163 @@
+// @ts-nocheck
+
+import { createEvent } from '@/services/event.service';
+import { last, omit } from 'ramda';
+
+import type { Event } from '../src/prisma-client';
+import { db } from '../src/prisma-client';
+
+async function push(event: Event) {
+  if (event.properties.ip && Number.isNaN(parseInt(event.properties.ip[0]))) {
+    return console.log('IGNORE', event.id);
+  }
+  await fetch('http://localhost:3030/api/event', {
+    method: 'POST',
+    body: JSON.stringify({
+      name: event.name,
+      timestamp: event.createdAt.toISOString(),
+      path: event.properties.path,
+      properties: omit(
+        [
+          'ip',
+          'os',
+          'ua',
+          'url',
+          'hash',
+          'host',
+          'path',
+          'device',
+          'screen',
+          'hostname',
+          'language',
+          'referrer',
+          'timezone',
+        ],
+        event.properties
+      ),
+      referrer: event.properties?.referrer?.host ?? undefined,
+    }),
+    headers: {
+      'Content-Type': 'application/json',
+      'User-Agent': event.properties.ua,
+      'X-Forwarded-For': event.properties.ip,
+      'mixan-client-id': 'c8b4962e-bc3d-4b23-8ea4-505c8fbdf09e',
+      origin: 'https://mixan.kiddo.se',
+    },
+  }).catch(() => {});
+}
+
+async function main() {
+  const events = await db.event.findMany({
+    where: {
+      project_id: '4e2798cb-e255-4e9d-960d-c9ad095aabd7',
+      name: 'screen_view',
+      createdAt: {
+        gte: new Date('2024-01-14'),
+        lt: new Date('2024-01-18'),
+      },
+    },
+    orderBy: {
+      createdAt: 'asc',
+    },
+  });
+
+  const grouped: Record<string, Event[]> = {};
+  let index = 0;
+  for (const event of events.slice()) {
+    console.log(index, event.name, event.createdAt.toISOString());
+    index++;
+
+    if (event.properties.ua?.includes('bot')) {
+      console.log('IGNORE', event.id);
+      continue;
+    }
+
+    if (grouped[event.profile_id]) {
+      grouped[event.profile_id].push(event);
+    } else {
+      grouped[event.profile_id] = [event];
+    }
+  }
+
+  for (const profile_id of Object.keys(grouped).slice(0, 10)) {
+    const events = grouped[profile_id];
+
+    if (events) {
+      console.log('new user...');
+      let eidx = -1;
+      for (const event of events) {
+        eidx++;
+        const lastEventAt = events[eidx - 1]?.createdAt;
+
+        const profileId: string | null = null;
+        const projectId = event.project_id;
+        const path = event.properties.path as string;
+        const ip = event.properties.ip as string;
+        const origin = 'https://mixan.kiddo.se';
+        const ua = event.properties.ua as string;
+        const uaInfo = parseUserAgent(ua);
+        const salts = await getSalts();
+
+        const [currentProfileId, previousProfileId, geo, eventsJobs] =
+          await Promise.all([
+            generateProfileId({
+              salt: salts.current,
+              origin,
+              ip,
+              ua,
+            }),
+            generateProfileId({
+              salt: salts.previous,
+              origin,
+              ip,
+              ua,
+            }),
+            parseIp(ip),
+            eventsQueue.getJobs(['delayed']),
+          ]);
+        const payload: IServiceCreateEventPayload = {
+          name: body.name,
+          profileId,
+          projectId,
+          properties: body.properties,
+          createdAt: body.timestamp,
+          country: geo.country,
+          city: geo.city,
+          region: geo.region,
+          continent: geo.continent,
+          os: uaInfo.os,
+          osVersion: uaInfo.osVersion,
+          browser: uaInfo.browser,
+          browserVersion: uaInfo.browserVersion,
+          device: uaInfo.device,
+          brand: uaInfo.brand,
+          model: uaInfo.model,
+          duration: 0,
+          path,
+          referrer: body.referrer, // TODO
+          referrerName: body.referrer, // TODO
+        };
+
+        if (!lastEventAt) {
+          createEvent({});
+          continue;
+        }
+
+        if (
+          event.createdAt.getTime() - lastEventAt.getTime() >
+          1000 * 60 * 30
+        ) {
+          console.log(
+            'new Session?',
+            event.createdAt.toISOString(),
+            event.properties.path
+          );
+        } else {
+          console.log('Same session?');
+        }
+      }
+    }
+  }
+}
+
+main();

packages/db/src/clickhouse-client.ts

@@ -0,0 +1,51 @@
+import { createClient } from '@clickhouse/client';
+
+export const ch = createClient({
+  host: process.env.CLICKHOUSE_URL,
+  username: process.env.CLICKHOUSE_USER,
+  password: process.env.CLICKHOUSE_PASSWORD,
+  database: process.env.CLICKHOUSE_DB,
+});
+
+interface ClickhouseJsonResponse<T> {
+  data: T[];
+  rows: number;
+  statistics: { elapsed: number; rows_read: number; bytes_read: number };
+  meta: { name: string; type: string }[];
+}
+
+export async function chQueryAll<T extends Record<string, any>>(
+  query: string
+): Promise<ClickhouseJsonResponse<T>> {
+  const res = await ch.query({
+    query,
+  });
+  const json = await res.json<ClickhouseJsonResponse<T>>();
+  return {
+    ...json,
+    data: json.data.map((item) => {
+      const keys = Object.keys(item);
+      return keys.reduce((acc, key) => {
+        const meta = json.meta.find((m) => m.name === key);
+        return {
+          ...acc,
+          [key]:
+            item[key] && meta?.type.includes('Int')
+              ? parseFloat(item[key] as string)
+              : item[key],
+        };
+      }, {} as T);
+    }),
+  };
+}
+
+export async function chQuery<T extends Record<string, any>>(
+  query: string
+): Promise<T[]> {
+  return (await chQueryAll<T>(query)).data;
+}
+
+export function formatClickhouseDate(_date: Date | string) {
+  const date = typeof _date === 'string' ? new Date(_date) : _date;
+  return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
+}

packages/db/src/prisma-client.ts

@@ -0,0 +1,15 @@
+import { PrismaClient } from '@prisma/client';
+
+export * from '@prisma/client';
+
+const globalForPrisma = globalThis as unknown as {
+  prisma: PrismaClient | undefined;
+};
+
+export const db =
+  globalForPrisma.prisma ??
+  new PrismaClient({
+    log: ['error'],
+  });
+
+if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = db;

packages/db/src/prisma-types.ts

@@ -0,0 +1,22 @@
+/* eslint-disable */
+
+export type IDBEvent = {
+  id: string;
+  name: string;
+  profile_id?: string;
+  project_id: string;
+  properties: Record<string, string>;
+  created_at: string;
+};
+
+export type IDBProfile = {
+  id: string;
+  external_id?: string;
+  first_name?: string;
+  last_name?: string;
+  email?: string;
+  avatar?: string;
+  properties: Record<string, string>;
+  project_id: String;
+  created_at: string;
+};

packages/db/src/services/event.service.ts

@@ -0,0 +1,113 @@
+import { omit } from 'ramda';
+
+import { toDots } from '@mixan/common';
+
+import { ch, chQuery, formatClickhouseDate } from '../clickhouse-client';
+
+export interface IClickhouseEvent {
+  name: string;
+  profile_id: string;
+  project_id: string;
+  path: string;
+  referrer: string;
+  referrer_name: string;
+  duration: number;
+  properties: Record<string, string>;
+  created_at: string;
+  country: string;
+  city: string;
+  region: string;
+  os: string;
+  os_version: string;
+  browser: string;
+  browser_version: string;
+  device: string;
+  brand: string;
+  model: string;
+}
+
+export function transformEvent(
+  event: IClickhouseEvent
+): IServiceCreateEventPayload {
+  return {
+    name: event.name,
+    profileId: event.profile_id,
+    projectId: event.project_id,
+    properties: event.properties,
+    createdAt: event.created_at,
+    country: event.country,
+    city: event.city,
+    region: event.region,
+    os: event.os,
+    osVersion: event.os_version,
+    browser: event.browser,
+    browserVersion: event.browser_version,
+    device: event.device,
+    brand: event.brand,
+    model: event.model,
+    duration: event.duration,
+    path: event.path,
+    referrer: event.referrer,
+    referrerName: event.referrer_name,
+  };
+}
+
+export interface IServiceCreateEventPayload {
+  name: string;
+  profileId: string;
+  projectId: string;
+  properties: Record<string, unknown>;
+  createdAt: string;
+  country?: string | undefined;
+  city?: string | undefined;
+  region?: string | undefined;
+  continent?: string | undefined;
+  os?: string | undefined;
+  osVersion?: string | undefined;
+  browser?: string | undefined;
+  browserVersion?: string | undefined;
+  device?: string | undefined;
+  brand?: string | undefined;
+  model?: string | undefined;
+  duration: number;
+  path: string;
+  referrer: string | undefined;
+  referrerName: string | undefined;
+}
+
+export function getEvents(sql: string) {
+  return chQuery<IClickhouseEvent>(sql).then((events) =>
+    events.map(transformEvent)
+  );
+}
+
+export async function createEvent(payload: IServiceCreateEventPayload) {
+  console.log(`create event ${payload.name} for ${payload.profileId}`);
+
+  return ch.insert({
+    table: 'events',
+    values: [
+      {
+        name: payload.name,
+        profile_id: payload.profileId,
+        project_id: payload.projectId,
+        properties: toDots(omit(['_path'], payload.properties)),
+        path: payload.path ?? '',
+        created_at: formatClickhouseDate(payload.createdAt),
+        country: payload.country ?? '',
+        city: payload.city ?? '',
+        region: payload.region ?? '',
+        os: payload.os ?? '',
+        os_version: payload.osVersion ?? '',
+        browser: payload.browser ?? '',
+        browser_version: payload.browserVersion ?? '',
+        device: payload.device ?? '',
+        brand: payload.brand ?? '',
+        model: payload.model ?? '',
+        duration: payload.duration,
+        referrer: payload.referrer ?? '',
+      },
+    ],
+    format: 'JSONEachRow',
+  });
+}

packages/db/src/services/salt.ts

@@ -0,0 +1,37 @@
+import { db } from '../prisma-client';
+
+export async function getCurrentSalt() {
+  const salt = await db.salt.findFirst({
+    orderBy: {
+      createdAt: 'desc',
+    },
+  });
+
+  if (!salt) {
+    throw new Error('No salt found');
+  }
+
+  return salt.salt;
+}
+
+export async function getSalts() {
+  const [curr, prev] = await db.salt.findMany({
+    orderBy: {
+      createdAt: 'desc',
+    },
+    take: 2,
+  });
+
+  if (!curr) {
+    throw new Error('No salt found');
+  }
+
+  if (!prev) {
+    throw new Error('No previous salt found');
+  }
+
+  return {
+    current: curr.salt,
+    previous: prev.salt,
+  };
+}

packages/db/src/sql-builder.ts

@@ -0,0 +1,56 @@
+export function createSqlBuilder() {
+  const join = (obj: Record<string, string> | string[], joiner: string) =>
+    Object.values(obj).filter(Boolean).join(joiner);
+
+  const sb: {
+    where: Record<string, string>;
+    select: Record<string, string>;
+    groupBy: Record<string, string>;
+    orderBy: Record<string, string>;
+    from: string;
+    limit: number | undefined;
+    offset: number | undefined;
+  } = {
+    where: {},
+    from: 'events',
+    select: {},
+    groupBy: {},
+    orderBy: {},
+    limit: undefined,
+    offset: undefined,
+  };
+
+  const getWhere = () =>
+    Object.keys(sb.where).length ? 'WHERE ' + join(sb.where, ' AND ') : '';
+  const getFrom = () => `FROM ${sb.from}`;
+  const getSelect = () =>
+    'SELECT ' + (Object.keys(sb.select).length ? join(sb.select, ', ') : '*');
+  const getGroupBy = () =>
+    Object.keys(sb.groupBy).length ? 'GROUP BY ' + join(sb.groupBy, ', ') : '';
+  const getOrderBy = () =>
+    Object.keys(sb.orderBy).length ? 'ORDER BY ' + join(sb.orderBy, ', ') : '';
+  const getLimit = () => (sb.limit ? `LIMIT ${sb.limit}` : '');
+  const getOffset = () => (sb.offset ? `OFFSET ${sb.offset}` : '');
+
+  return {
+    sb,
+    join,
+    getWhere,
+    getFrom,
+    getSelect,
+    getGroupBy,
+    getOrderBy,
+    getSql: () =>
+      [
+        getSelect(),
+        getFrom(),
+        getWhere(),
+        getGroupBy(),
+        getOrderBy(),
+        getLimit(),
+        getOffset(),
+      ]
+        .filter(Boolean)
+        .join(' '),
+  };
+}

packages/queue/index.ts

@@ -1,2 +1,4 @@
 export { eventsQueue } from './src/queues';
 export { connection } from './src/connection';
+export { findJobByPrefix } from './src/utils';
+export type { JobsOptions } from 'bullmq';

packages/queue/package.json

@@ -8,6 +8,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@mixan/db": "workspace:*",
     "bullmq": "^5.1.1"
   },
   "devDependencies": {

packages/queue/src/queues.ts

@@ -1,12 +1,24 @@
 import { Queue } from 'bullmq';
 
-import type { BatchPayload } from '@mixan/types';
+import type { IServiceCreateEventPayload } from '@mixan/db';
 
 import { connection } from './connection';
 
-export interface EventsQueuePayload {
-  projectId: string;
-  payload: BatchPayload[];
+export interface EventsQueuePayloadCreateEvent {
+  type: 'createEvent';
+  payload: IServiceCreateEventPayload;
+}
+export interface EventsQueuePayloadCreateSessionEnd {
+  type: 'createSessionEnd';
+  payload: Pick<IServiceCreateEventPayload, 'profileId'>;
+}
+export type EventsQueuePayload =
+  | EventsQueuePayloadCreateEvent
+  | EventsQueuePayloadCreateSessionEnd;
+
+export interface CronQueuePayload {
+  type: 'salt';
+  payload: undefined;
 }
 
 export const eventsQueue = new Queue<EventsQueuePayload>('events', {
@@ -15,3 +27,10 @@ export const eventsQueue = new Queue<EventsQueuePayload>('events', {
     removeOnComplete: 10,
   },
 });
+
+export const cronQueue = new Queue<CronQueuePayload>('cron', {
+  connection,
+  defaultJobOptions: {
+    removeOnComplete: 10,
+  },
+});

packages/queue/src/utils.ts

@@ -0,0 +1,8 @@
+import type { Job } from 'bullmq';
+
+export function findJobByPrefix<T>(
+  jobs: Job<T, any, string>[],
+  prefix: string
+) {
+  return jobs.find((job) => job.opts.jobId?.startsWith(prefix));
+}

packages/redis/index.ts

@@ -0,0 +1,3 @@
+import Redis from 'ioredis';
+
+export const redis = new Redis(process.env.REDIS_URL);

packages/redis/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@mixan/redis",
+  "version": "0.0.1",
+  "main": "index.ts",
+  "scripts": {
+    "lint": "eslint .",
+    "format": "prettier --check \"**/*.{mjs,ts,md,json}\"",
+    "typecheck": "tsc --noEmit",
+    "with-env": "dotenv -e ../../.env -c --"
+  },
+  "dependencies": {
+    "ioredis": "^5.3.2"
+  },
+  "devDependencies": {
+    "@mixan/eslint-config": "workspace:*",
+    "@mixan/prettier-config": "workspace:*",
+    "@mixan/tsconfig": "workspace:*",
+    "@mixan/types": "workspace:*",
+    "@types/node": "^18.16.0",
+    "eslint": "^8.48.0",
+    "prettier": "^3.0.3",
+    "prisma": "^5.1.1",
+    "typescript": "^5.2.2"
+  },
+  "eslintConfig": {
+    "root": true,
+    "extends": [
+      "@mixan/eslint-config/base"
+    ]
+  },
+  "prettier": "@mixan/prettier-config"
+}

packages/redis/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "@mixan/tsconfig/base.json",
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./src/*"]
+    },
+    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+  },
+  "include": ["."],
+  "exclude": ["node_modules"]
+}

packages/sdk/index.ts

@@ -7,27 +7,33 @@ import type {
 
 type MixanLogger = (...args: unknown[]) => void;
 
+// -- 1. Besök
+// -- 2. Finns profile id?
+// --   NEJ
+// --   a. skicka events som vanligt (retunera genererat ID)
+// --   b. ge möjlighet att spara
+// --   JA
+// --   a. skicka event med profile_id
+// -- Payload
+// -- - user_agent?
+// -- - ip?
+// -- - profile_id?
+// -- - referrer
+
 export interface NewMixanOptions {
   url: string;
   clientId: string;
   clientSecret?: string;
-  batchInterval?: number;
-  maxBatchSize?: number;
-  sessionTimeout?: number;
-  session?: boolean;
   verbose?: boolean;
-  trackIp?: boolean;
-  ipUrl?: string;
-  setItem: (key: string, profileId: string) => void;
-  getItem: (key: string) => string | null;
-  removeItem: (key: string) => void;
+  setItem?: (key: string, profileId: string) => void;
+  getItem?: (key: string) => string | null;
+  removeItem?: (key: string) => void;
 }
 
 export type MixanOptions = Required<NewMixanOptions>;
 
 export interface MixanState {
-  profileId: string;
-  lastEventAt: number;
+  profileId: null | string;
   properties: Record<string, unknown>;
 }
 
@@ -35,14 +41,6 @@ function createLogger(verbose: boolean): MixanLogger {
   return verbose ? (...args) => console.log('[Mixan]', ...args) : () => {};
 }
 
-function uuid() {
-  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
-    const r = (Math.random() * 16) | 0;
-    const v = c === 'x' ? r : (r & 0x3) | 0x8;
-    return v.toString(16);
-  });
-}
-
 class Fetcher {
   private url: string;
   private clientId: string;
@@ -126,72 +124,23 @@ class Fetcher {
   }
 }
 
-class Batcher {
-  queue: BatchPayload[] = [];
-  timer?: ReturnType<typeof setTimeout>;
-
-  constructor(
-    private options: MixanOptions,
-    private callback: (payload: BatchPayload[]) => void,
-    private logger: MixanLogger
-  ) {}
-
-  add(action: BatchPayload) {
-    if (this.timer) {
-      clearTimeout(this.timer);
-    }
-
-    this.logger(`Add to queue ${action.type}`);
-    this.queue.push(action);
-
-    if (this.queue.length >= this.options.maxBatchSize) {
-      this.send();
-    } else {
-      this.timer = setTimeout(this.send.bind(this), this.options.batchInterval);
-    }
-  }
-
-  send() {
-    this.logger('Send queue', this.queue.length > 0);
-    if (this.queue.length > 0) {
-      this.callback(this.queue);
-      this.queue = [];
-    }
-  }
-}
-
 export class Mixan {
   private options: MixanOptions;
   private fetch: Fetcher;
-  private batcher: Batcher;
   private logger: (...args: any[]) => void;
   private state: MixanState = {
-    profileId: '',
-    lastEventAt: 0,
+    profileId: null,
     properties: {},
   };
 
   constructor(options: NewMixanOptions) {
     this.logger = createLogger(options.verbose ?? false);
     this.options = {
-      sessionTimeout: 1000 * 60 * 30,
-      session: true,
       verbose: false,
-      batchInterval: 10000,
-      maxBatchSize: 10,
-      trackIp: false,
       clientSecret: '',
-      ipUrl: 'https://api.ipify.org',
       ...options,
     };
     this.fetch = new Fetcher(this.options, this.logger);
-    this.batcher = new Batcher(
-      this.options,
-      (queue) => {
-        this.fetch.post('/batch', queue);
-      },
-      this.logger
-    );
   }
 
   // Public
@@ -199,14 +148,9 @@ export class Mixan {
   public init(properties?: Record<string, unknown>) {
     this.logger('Init');
     this.state.properties = properties ?? {};
-    this.createProfile();
-    this.createSession();
-    this.ipLookup();
   }
 
   public setUser(payload: Omit<BatchUpdateProfilePayload, 'profileId'>) {
-    this.createSession();
-
     this.batcher.add({
       type: 'update_profile',
       payload: {
@@ -217,21 +161,7 @@ export class Mixan {
     });
   }
 
-  public setSession(properties: BatchUpdateSessionPayload['properties']) {
-    this.createSession();
-
-    this.batcher.add({
-      type: 'update_session',
-      payload: {
-        properties,
-        profileId: this.state.profileId,
-      },
-    });
-  }
-
   public increment(name: string, value: number) {
-    this.createSession();
-
     this.batcher.add({
       type: 'increment',
       payload: {
@@ -243,8 +173,6 @@ export class Mixan {
   }
 
   public decrement(name: string, value: number) {
-    this.createSession();
-
     this.batcher.add({
       type: 'decrement',
       payload: {
@@ -256,11 +184,8 @@ export class Mixan {
   }
 
   public event(name: string, properties?: Record<string, unknown>) {
-    this.createSession();
-
-    this.batcher.add({
-      type: 'event',
-      payload: {
+    this.fetch
+      .post('/event', {
         name,
         properties: {
           ...this.state.properties,
@@ -268,17 +193,15 @@ export class Mixan {
         },
         time: this.timestamp(),
         profileId: this.state.profileId,
-      },
-    });
+      })
+      .then((response) => {
+        if ('profileId' in response) {
+          this.options.setItem('@mixan:profileId', response.profileId);
+        }
+      });
   }
 
   public setGlobalProperties(properties: Record<string, unknown>) {
-    if (typeof properties !== 'object') {
-      return this.logger(
-        'Set global properties failed, properties must be an object'
-      );
-    }
-
     this.logger('Set global properties', properties);
     this.state.properties = {
       ...this.state.properties,
@@ -286,128 +209,27 @@ export class Mixan {
     };
   }
 
-  public flush() {
-    this.batcher.send();
-  }
-
   public clear() {
     this.logger('Clear / Logout');
-    this.flush();
-    this.options.removeItem('@mixan:ip');
     this.options.removeItem('@mixan:profileId');
-    this.options.removeItem('@mixan:lastEventAt');
-    this.state.profileId = '';
-    this.state.lastEventAt = 0;
-    this.createProfile();
+    this.state.profileId = null;
   }
 
   public setUserProperty(name: string, value: unknown, update = true) {
-    this.batcher.add({
-      type: 'set_profile_property',
-      payload: {
-        name,
-        value,
-        update,
-        profileId: this.state.profileId,
-      },
-    });
+    // this.batcher.add({
+    //   type: 'set_profile_property',
+    //   payload: {
+    //     name,
+    //     value,
+    //     update,
+    //     profileId: this.state.profileId,
+    //   },
+    // });
   }
 
   // Private
 
-  private timestamp(modify = 0) {
-    this.setLastEventAt();
-    return new Date(Date.now() + modify).toISOString();
-  }
-
-  private createProfile() {
-    const profileId = this.options.getItem('@mixan:profileId');
-
-    if (profileId) {
-      this.logger('Reusing existing profile');
-      this.state.profileId = profileId;
-    } else {
-      this.logger('Creating profile');
-      this.state.profileId = uuid();
-      this.options.setItem('@mixan:profileId', this.state.profileId);
-      this.batcher.add({
-        type: 'create_profile',
-        payload: {
-          profileId: this.state.profileId,
-          properties: this.state.properties,
-        },
-      });
-    }
-  }
-
-  private checkSession() {
-    if (!this.options.session) {
-      return false;
-    }
-
-    if (this.state.lastEventAt === 0) {
-      const str = this.options.getItem('@mixan:lastEventAt') ?? '0';
-      const value = parseInt(str, 10);
-      this.state.lastEventAt = isNaN(value) ? 0 : value;
-    }
-
-    return Date.now() - this.state.lastEventAt > this.options.sessionTimeout;
-  }
-
-  private createSession() {
-    if (!this.checkSession()) {
-      return;
-    }
-
-    const time = this.timestamp(-10);
-
-    this.batcher.add({
-      type: 'event',
-      payload: {
-        name: 'session_start',
-        properties: this.state.properties,
-        profileId: this.state.profileId,
-        time,
-      },
-    });
-  }
-
-  private setLastEventAt() {
-    this.state.lastEventAt = Date.now();
-    this.options.setItem(
-      '@mixan:lastEventAt',
-      this.state.lastEventAt.toString()
-    );
-  }
-
-  private async ipLookup() {
-    if (!this.options.trackIp) {
-      return null;
-    }
-
-    let ip: string | null;
-
-    const cachedIp = this.options.getItem('@mixan:ip');
-    if (cachedIp) {
-      ip = cachedIp;
-    } else {
-      const controller = new AbortController();
-      const timeout = setTimeout(() => controller.abort(), 1000);
-      ip = await fetch(this.options.ipUrl, {
-        signal: controller.signal,
-      })
-        .then((res) => res.text())
-        .catch(() => null)
-        .finally(() => clearTimeout(timeout));
-    }
-
-    if (ip) {
-      this.options.setItem('@mixan:ip', ip);
-      this.setGlobalProperties({ ip });
-      if (!cachedIp) {
-        this.setUserProperty('ip', ip, false);
-        this.setSession({ ip });
-      }
-    }
+  private timestamp() {
+    return new Date().toISOString();
   }
 }