a lot

apps/sdk-api/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@mixan/sdk-api",
+  "version": "0.0.1",
+  "scripts": {
+    "dev": "dotenv -e ../../.env -c -v WATCH=1 tsup",
+    "start": "node dist/index.js",
+    "build": "rm -rf dist && tsup",
+    "lint": "eslint .",
+    "format": "prettier --check \"**/*.{mjs,ts,md,json}\"",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@fastify/cors": "^9.0.0",
+    "@mixan/common": "workspace:*",
+    "@mixan/db": "workspace:*",
+    "@mixan/queue": "workspace:*",
+    "fastify": "^4.25.2",
+    "pino": "^8.17.2",
+    "ramda": "^0.29.1",
+    "request-ip": "^3.3.0",
+    "ua-parser-js": "^1.0.37"
+  },
+  "devDependencies": {
+    "@mixan/eslint-config": "workspace:*",
+    "@mixan/prettier-config": "workspace:*",
+    "@mixan/tsconfig": "workspace:*",
+    "@mixan/types": "workspace:*",
+    "@types/ramda": "^0.29.6",
+    "@types/request-ip": "^0.0.41",
+    "@types/ua-parser-js": "^0.7.39",
+    "eslint": "^8.48.0",
+    "prettier": "^3.0.3",
+    "tsup": "^7.2.0",
+    "typescript": "^5.2.2"
+  },
+  "eslintConfig": {
+    "root": true,
+    "extends": [
+      "@mixan/eslint-config/base"
+    ]
+  },
+  "prettier": "@mixan/prettier-config"
+}

apps/sdk-api/scripts/test-events.ts

@@ -0,0 +1,226 @@
+import { omit, prop, uniqBy } from 'ramda';
+
+import { generateProfileId, getTime, toISOString } from '@mixan/common';
+import type { Event, IServiceCreateEventPayload } from '@mixan/db';
+import {
+  createEvent as createClickhouseEvent,
+  db,
+  formatClickhouseDate,
+  getSalts,
+} from '@mixan/db';
+
+import { parseIp } from '../src/utils/parseIp';
+import { parseUserAgent } from '../src/utils/parseUserAgent';
+
+const clean = omit([
+  'ip',
+  'os',
+  'ua',
+  'url',
+  'hash',
+  'host',
+  'path',
+  'device',
+  'screen',
+  'hostname',
+  'language',
+  'referrer',
+  'timezone',
+]);
+async function main() {
+  const events = await db.event.findMany({
+    where: {
+      project_id: '4e2798cb-e255-4e9d-960d-c9ad095aabd7',
+      name: 'screen_view',
+      createdAt: {
+        gte: new Date('2024-01-01'),
+        lt: new Date('2024-02-04'),
+      },
+    },
+    orderBy: {
+      createdAt: 'asc',
+    },
+  });
+
+  const grouped: Record<string, Event[]> = {};
+  let index = 0;
+  for (const event of events.slice()) {
+    // console.log(index, event.name, event.createdAt.toISOString());
+    index++;
+
+    const properties = event.properties as Record<string, any>;
+
+    if (properties.ua?.includes('bot')) {
+      // console.log('IGNORE', event.id);
+      continue;
+    }
+
+    if (!event.profile_id) {
+      // console.log('IGNORE', event.id);
+      continue;
+    }
+    const hej = grouped[event.profile_id];
+    if (hej) {
+      hej.push(event);
+    } else {
+      grouped[event.profile_id] = [event];
+    }
+  }
+
+  console.log('Total users', Object.keys(grouped).length);
+
+  let uidx = -1;
+  for (const profile_id of Object.keys(grouped)) {
+    uidx++;
+    console.log(`User index ${uidx}`);
+
+    const events = uniqBy(prop('createdAt'), grouped[profile_id] || []);
+
+    if (events) {
+      let lastSessionStart = null;
+      let screenViews = 0;
+      let totalDuration = 0;
+      console.log('new user...');
+      let eidx = -1;
+      for (const event of events) {
+        eidx++;
+        const prevEvent = events[eidx - 1];
+        const prevEventAt = prevEvent?.createdAt;
+
+        const nextEvent = events[eidx + 1];
+
+        const properties = event.properties as Record<string, any>;
+        const projectId = event.project_id;
+        const path = properties.path!;
+        const ip = properties.ip!;
+        const origin = 'https://mixan.kiddo.se';
+        const ua = properties.ua!;
+        const uaInfo = parseUserAgent(ua);
+        const salts = await getSalts();
+
+        const [profileId, geo] = await Promise.all([
+          generateProfileId({
+            salt: salts.current,
+            origin,
+            ip,
+            ua,
+          }),
+          parseIp(ip),
+        ]);
+
+        const isNextEventNewSession =
+          nextEvent &&
+          nextEvent.createdAt.getTime() - event.createdAt.getTime() >
+            1000 * 60 * 30;
+
+        const payload: IServiceCreateEventPayload = {
+          name: event.name,
+          profileId,
+          projectId,
+          properties: clean(properties),
+          createdAt: event.createdAt.toISOString(),
+          country: geo.country,
+          city: geo.city,
+          region: geo.region,
+          continent: geo.continent,
+          os: uaInfo.os,
+          osVersion: uaInfo.osVersion,
+          browser: uaInfo.browser,
+          browserVersion: uaInfo.browserVersion,
+          device: uaInfo.device,
+          brand: uaInfo.brand,
+          model: uaInfo.model,
+          duration:
+            nextEvent && !isNextEventNewSession
+              ? nextEvent.createdAt.getTime() - event.createdAt.getTime()
+              : 0,
+          path,
+          referrer: properties?.referrer?.host ?? '', // TODO
+          referrerName: properties?.referrer?.host ?? '', // TODO
+        };
+
+        if (!prevEventAt) {
+          lastSessionStart = await createSessionStart(payload);
+        } else if (
+          event.createdAt.getTime() - prevEventAt.getTime() >
+          1000 * 60 * 30
+        ) {
+          if (eidx > 0 && prevEventAt && lastSessionStart) {
+            await createSessionEnd(prevEventAt, lastSessionStart, {
+              screenViews,
+              totalDuration,
+            });
+            totalDuration = 0;
+            screenViews = 0;
+            lastSessionStart = await createSessionStart(payload);
+          }
+        }
+
+        screenViews++;
+        totalDuration += payload.duration;
+        await createEvent(payload);
+      } // for each user event
+
+      const prevEventAt = events[events.length - 1]?.createdAt;
+      if (prevEventAt && lastSessionStart) {
+        await createSessionEnd(prevEventAt, lastSessionStart, {
+          screenViews,
+          totalDuration,
+        });
+      }
+    }
+  }
+}
+
+async function createEvent(event: IServiceCreateEventPayload) {
+  console.log(
+    `Create ${event.name} - ${event.path} - ${formatClickhouseDate(
+      event.createdAt
+    )} - ${event.duration / 1000} sec`
+  );
+  await createClickhouseEvent(event);
+}
+
+async function createSessionStart(event: IServiceCreateEventPayload) {
+  const session: IServiceCreateEventPayload = {
+    ...event,
+    duration: 0,
+    name: 'session_start',
+    createdAt: toISOString(getTime(event.createdAt) - 10),
+  };
+
+  await createEvent(session);
+  return session;
+}
+
+async function createSessionEnd(
+  prevEventAt: Date,
+  sessionStart: IServiceCreateEventPayload,
+  options: {
+    screenViews: number;
+    totalDuration: number;
+  }
+) {
+  const properties: Record<string, unknown> = {};
+  if (options.screenViews === 1) {
+    properties._bounce = true;
+  } else {
+    properties._bounce = false;
+  }
+
+  const session: IServiceCreateEventPayload = {
+    ...sessionStart,
+    properties: {
+      ...properties,
+      ...sessionStart.properties,
+    },
+    duration: options.totalDuration,
+    name: 'session_end',
+    createdAt: toISOString(prevEventAt.getTime() + 10),
+  };
+
+  await createEvent(session);
+  return session;
+}
+
+main();

apps/sdk-api/src/controllers/event.controller.ts

@@ -0,0 +1,176 @@
+import { parseIp } from '@/utils/parseIp';
+import { parseUserAgent } from '@/utils/parseUserAgent';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { getClientIp } from 'request-ip';
+
+import { generateProfileId, getTime, toISOString } from '@mixan/common';
+import type { IServiceCreateEventPayload } from '@mixan/db';
+import { getSalts } from '@mixan/db';
+import type { JobsOptions } from '@mixan/queue';
+import { eventsQueue, findJobByPrefix } from '@mixan/queue';
+
+export interface PostEventPayload {
+  profileId?: string;
+  name: string;
+  timestamp: string;
+  properties: Record<string, unknown>;
+  referrer: string | undefined;
+  path: string;
+}
+
+const SESSION_TIMEOUT = 1000 * 30 * 1;
+const SESSION_END_TIMEOUT = SESSION_TIMEOUT + 1000;
+
+export async function postEvent(
+  request: FastifyRequest<{
+    Body: PostEventPayload;
+  }>,
+  reply: FastifyReply
+) {
+  let profileId: string | null = null;
+  const projectId = request.projectId;
+  const body = request.body;
+  const path = body.path;
+  const ip = getClientIp(request)!;
+  const origin = request.headers.origin!;
+  const ua = request.headers['user-agent']!;
+  const uaInfo = parseUserAgent(ua);
+  const salts = await getSalts();
+
+  const [currentProfileId, previousProfileId, geo, eventsJobs] =
+    await Promise.all([
+      generateProfileId({
+        salt: salts.current,
+        origin,
+        ip,
+        ua,
+      }),
+      generateProfileId({
+        salt: salts.previous,
+        origin,
+        ip,
+        ua,
+      }),
+      parseIp(ip),
+      eventsQueue.getJobs(['delayed']),
+    ]);
+
+  // find session_end job
+  const sessionEndJobCurrentProfileId = findJobByPrefix(
+    eventsJobs,
+    `sessionEnd:${projectId}:${currentProfileId}:`
+  );
+  const sessionEndJobPreviousProfileId = findJobByPrefix(
+    eventsJobs,
+    `sessionEnd:${projectId}:${previousProfileId}:`
+  );
+
+  const createSessionStart =
+    !sessionEndJobCurrentProfileId && !sessionEndJobPreviousProfileId;
+
+  if (sessionEndJobCurrentProfileId && !sessionEndJobPreviousProfileId) {
+    console.log('found session current');
+    profileId = currentProfileId;
+    const diff = Date.now() - sessionEndJobCurrentProfileId.timestamp;
+    sessionEndJobCurrentProfileId.changeDelay(diff + SESSION_END_TIMEOUT);
+  } else if (!sessionEndJobCurrentProfileId && sessionEndJobPreviousProfileId) {
+    console.log('found session previous');
+    profileId = previousProfileId;
+    const diff = Date.now() - sessionEndJobPreviousProfileId.timestamp;
+    sessionEndJobPreviousProfileId.changeDelay(diff + SESSION_END_TIMEOUT);
+  } else {
+    console.log('new session with current');
+    profileId = currentProfileId;
+    // Queue session end
+    eventsQueue.add(
+      'event',
+      {
+        type: 'createSessionEnd',
+        payload: {
+          profileId,
+        },
+      },
+      {
+        delay: SESSION_END_TIMEOUT,
+        jobId: `sessionEnd:${projectId}:${profileId}:${Date.now()}`,
+      }
+    );
+  }
+
+  const payload: IServiceCreateEventPayload = {
+    name: body.name,
+    profileId,
+    projectId,
+    properties: body.properties,
+    createdAt: body.timestamp,
+    country: geo.country,
+    city: geo.city,
+    region: geo.region,
+    continent: geo.continent,
+    os: uaInfo.os,
+    osVersion: uaInfo.osVersion,
+    browser: uaInfo.browser,
+    browserVersion: uaInfo.browserVersion,
+    device: uaInfo.device,
+    brand: uaInfo.brand,
+    model: uaInfo.model,
+    duration: 0,
+    path,
+    referrer: body.referrer, // TODO
+    referrerName: body.referrer, // TODO
+  };
+
+  console.log(payload);
+
+  const job = findJobByPrefix(eventsJobs, `event:${projectId}:${profileId}:`);
+
+  if (job?.isDelayed && job.data.type === 'createEvent') {
+    const prevEvent = job.data.payload;
+    const duration = getTime(payload.createdAt) - getTime(prevEvent.createdAt);
+
+    // Set path from prev screen_view event if current event is not a screen_view
+    if (payload.name != 'screen_view') {
+      payload.path = prevEvent.path;
+    }
+
+    if (payload.name === 'screen_view') {
+      await job.updateData({
+        type: 'createEvent',
+        payload: {
+          ...prevEvent,
+          duration,
+        },
+      });
+      job.promote();
+    }
+  }
+
+  if (createSessionStart) {
+    eventsQueue.add('event', {
+      type: 'createEvent',
+      payload: {
+        ...payload,
+        name: 'session_start',
+        createdAt: toISOString(getTime(payload.createdAt) - 10),
+      },
+    });
+  }
+
+  const options: JobsOptions = {};
+  if (payload.name === 'screen_view') {
+    options.delay = SESSION_TIMEOUT;
+    options.jobId = `event:${projectId}:${profileId}:${Date.now()}`;
+  }
+
+  // Queue current event
+  eventsQueue.add(
+    'event',
+    {
+      type: 'createEvent',
+      payload,
+    },
+    options
+  );
+
+  reply.status(202).send(profileId);
+}

apps/sdk-api/src/index.ts

@@ -0,0 +1,78 @@
+import cors from '@fastify/cors';
+import Fastify from 'fastify';
+import pino from 'pino';
+
+import eventRouter from './routes/event.router';
+import { validateSdkRequest } from './utils/auth';
+
+declare module 'fastify' {
+  interface FastifyRequest {
+    projectId: string;
+  }
+}
+
+const port = parseInt(process.env.API_PORT || '3030', 10);
+
+const startServer = async () => {
+  try {
+    const fastify = Fastify({
+      logger: pino({ level: 'info' }),
+    });
+
+    fastify.register(cors, {
+      origin: '*',
+    });
+
+    fastify.decorateRequest('projectId', '');
+
+    fastify.addHook('preHandler', (req, reply, done) => {
+      validateSdkRequest(req.headers)
+        .then((projectId) => {
+          req.projectId = projectId;
+          done();
+        })
+        .catch((e) => {
+          console.log(e);
+
+          reply.status(401).send();
+        });
+    });
+
+    fastify.register(eventRouter, { prefix: '/api/event' });
+    fastify.setErrorHandler((error, request, reply) => {
+      fastify.log.error(error);
+    });
+    fastify.get('/', (request, reply) => {
+      reply.send({ name: 'fastify-typescript' });
+    });
+    // fastify.get('/health-check', async (request, reply) => {
+    //   try {
+    //     await utils.healthCheck()
+    //     reply.status(200).send()
+    //   } catch (e) {
+    //     reply.status(500).send()
+    //   }
+    // })
+    if (process.env.NODE_ENV === 'production') {
+      for (const signal of ['SIGINT', 'SIGTERM']) {
+        process.on(signal, () =>
+          fastify.close().then((err) => {
+            console.log(`close application on ${signal}`);
+            process.exit(err ? 1 : 0);
+          })
+        );
+      }
+    }
+
+    await fastify.listen({ port });
+  } catch (e) {
+    console.error(e);
+  }
+};
+
+process.on('unhandledRejection', (e) => {
+  console.error(e);
+  process.exit(1);
+});
+
+startServer();

apps/sdk-api/src/routes/event.router.ts

@@ -0,0 +1,18 @@
+import * as controller from '@/controllers/event.controller';
+import type { FastifyPluginCallback } from 'fastify';
+
+const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
+  fastify.route({
+    method: 'POST',
+    url: '/',
+    handler: controller.postEvent,
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/',
+    handler: controller.postEvent,
+  });
+  done();
+};
+
+export default eventRouter;

apps/sdk-api/src/utils/auth.ts

@@ -0,0 +1,43 @@
+import type { RawRequestDefaultExpression } from 'fastify';
+
+import { verifyPassword } from '@mixan/common';
+import { db } from '@mixan/db';
+
+export async function validateSdkRequest(
+  headers: RawRequestDefaultExpression['headers']
+): Promise<string> {
+  const clientId = headers['mixan-client-id'] as string;
+  const clientSecret = headers['mixan-client-secret'] as string;
+  const origin = headers.origin;
+  if (!clientId) {
+    throw new Error('Misisng client id');
+  }
+
+  const client = await db.client.findUnique({
+    where: {
+      id: clientId,
+    },
+  });
+
+  if (!client) {
+    throw new Error('Invalid client id');
+  }
+
+  if (client.secret) {
+    if (!(await verifyPassword(clientSecret || '', client.secret))) {
+      throw new Error('Invalid client secret');
+    }
+  } else if (client.cors !== '*') {
+    const domainAllowed = client.cors.split(',').find((domain) => {
+      if (domain === origin) {
+        return true;
+      }
+    });
+
+    if (!domainAllowed) {
+      throw new Error('Invalid cors settings');
+    }
+  }
+
+  return client.project_id;
+}

apps/sdk-api/src/utils/parseIp.ts

@@ -0,0 +1,25 @@
+export async function parseIp(ip: string) {
+  try {
+    const geo = await fetch(`http://localhost:8080/${ip}`);
+    const res = (await geo.json()) as {
+      country: string | undefined;
+      city: string | undefined;
+      stateprov: string | undefined;
+      continent: string | undefined;
+    };
+
+    return {
+      country: res.country,
+      city: res.city,
+      region: res.stateprov,
+      continent: res.continent,
+    };
+  } catch (e) {
+    return {
+      country: undefined,
+      city: undefined,
+      region: undefined,
+      continent: undefined,
+    };
+  }
+}

apps/sdk-api/src/utils/parseUserAgent.ts

@@ -0,0 +1,29 @@
+import { UAParser } from 'ua-parser-js';
+
+export function parseUserAgent(ua: string) {
+  const res = new UAParser(ua).getResult();
+  return {
+    os: res.os.name,
+    osVersion: res.os.version,
+    browser: res.browser.name,
+    browserVersion: res.browser.version,
+    device: res.device.type ?? getDevice(ua),
+    brand: res.device.vendor,
+    model: res.device.model,
+  };
+}
+
+export function getDevice(ua: string) {
+  const t1 =
+    /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(
+      ua
+    );
+  const t2 =
+    /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test(
+      ua.slice(0, 4)
+    );
+  if (t1 || t2) {
+    return 'mobile';
+  }
+  return 'desktop';
+}

apps/sdk-api/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "@mixan/tsconfig/base.json",
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./src/*"]
+    },
+    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+  },
+  "include": ["."],
+  "exclude": ["node_modules"]
+}

apps/sdk-api/tsup.config.ts

@@ -0,0 +1,18 @@
+import { defineConfig } from 'tsup';
+import type { Options } from 'tsup';
+
+const options: Options = {
+  clean: true,
+  entry: ['src/index.ts'],
+  noExternal: [/^@mixan\/.*$/u, /^@\/.*$/u],
+  sourcemap: true,
+  splitting: false,
+};
+
+if (process.env.WATCH) {
+  options.watch = ['src/**/*', '../../packages/**/*'];
+  options.onSuccess = 'node dist/index.js';
+  options.minify = false;
+}
+
+export default defineConfig(options);