move sdk packages to its own folder and rename api & dashboard

apps/api/scripts/get-referrers.ts

@@ -0,0 +1,48 @@
+import fs from 'fs';
+import path from 'path';
+
+function transform(data: any) {
+  const obj: Record<string, unknown> = {};
+  for (const type in data) {
+    for (const name in data[type]) {
+      const domains = data[type][name].domains ?? [];
+      for (const domain of domains) {
+        obj[domain] = {
+          type,
+          name,
+        };
+      }
+    }
+  }
+
+  return obj;
+}
+
+async function main() {
+  // Get document, or throw exception on error
+  try {
+    const data = await fetch(
+      'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json'
+    ).then((res) => res.json());
+
+    fs.writeFileSync(
+      path.resolve(__dirname, '../src/referrers/index.ts'),
+      [
+        '// This file is generated by the script get-referrers.ts',
+        '',
+        '// The data is fetch from snowplow-referer-parser https://github.com/snowplow-referer-parser/referer-parser',
+        `// The orginal referers.yml is based on Piwik's SearchEngines.php and Socials.php, copyright 2012 Matthieu Aubry and available under the GNU General Public License v3.`,
+        '',
+        `const referrers: Record<string, { type: string, name: string }> = ${JSON.stringify(
+          transform(data)
+        )} as const;`,
+        'export default referrers;',
+      ].join('\n'),
+      'utf-8'
+    );
+  } catch (e) {
+    console.log(e);
+  }
+}
+
+main();

apps/api/scripts/test-events.ts

@@ -0,0 +1,224 @@
+import { omit, prop, uniqBy } from 'ramda';
+
+import { generateProfileId, getTime, toISOString } from '@mixan/common';
+import type { Event, IServiceCreateEventPayload } from '@mixan/db';
+import {
+  createEvent as createClickhouseEvent,
+  db,
+  formatClickhouseDate,
+  getSalts,
+} from '@mixan/db';
+
+import { parseIp } from '../src/utils/parseIp';
+import { parseUserAgent } from '../src/utils/parseUserAgent';
+
+const clean = omit([
+  'ip',
+  'os',
+  'ua',
+  'url',
+  'hash',
+  'host',
+  'path',
+  'device',
+  'screen',
+  'hostname',
+  'language',
+  'referrer',
+  'timezone',
+]);
+async function main() {
+  const events = await db.event.findMany({
+    where: {
+      project_id: '4e2798cb-e255-4e9d-960d-c9ad095aabd7',
+      name: 'screen_view',
+      createdAt: {
+        gte: new Date('2024-01-01'),
+        lt: new Date('2024-02-04'),
+      },
+    },
+    orderBy: {
+      createdAt: 'asc',
+    },
+  });
+
+  const grouped: Record<string, Event[]> = {};
+  let index = 0;
+  for (const event of events.slice()) {
+    // console.log(index, event.name, event.createdAt.toISOString());
+    index++;
+
+    const properties = event.properties as Record<string, any>;
+
+    if (properties.ua?.includes('bot')) {
+      // console.log('IGNORE', event.id);
+      continue;
+    }
+
+    if (!event.profile_id) {
+      // console.log('IGNORE', event.id);
+      continue;
+    }
+    const hej = grouped[event.profile_id];
+    if (hej) {
+      hej.push(event);
+    } else {
+      grouped[event.profile_id] = [event];
+    }
+  }
+
+  console.log('Total users', Object.keys(grouped).length);
+
+  let uidx = -1;
+  for (const profile_id of Object.keys(grouped)) {
+    uidx++;
+    console.log(`User index ${uidx}`);
+
+    const events = uniqBy(prop('createdAt'), grouped[profile_id] || []);
+
+    if (events) {
+      let lastSessionStart = null;
+      let screenViews = 0;
+      let totalDuration = 0;
+      console.log('new user...');
+      let eidx = -1;
+      for (const event of events) {
+        eidx++;
+        const prevEvent = events[eidx - 1];
+        const prevEventAt = prevEvent?.createdAt;
+
+        const nextEvent = events[eidx + 1];
+
+        const properties = event.properties as Record<string, any>;
+        const projectId = event.project_id;
+        const path = properties.path!;
+        const ip = properties.ip!;
+        const origin = 'https://mixan.kiddo.se';
+        const ua = properties.ua!;
+        const uaInfo = parseUserAgent(ua);
+        const salts = await getSalts();
+        const profileId = generateProfileId({
+          salt: salts.current,
+          origin,
+          ip,
+          ua,
+        });
+
+        const geo = parseIp(ip);
+
+        const isNextEventNewSession =
+          nextEvent &&
+          nextEvent.createdAt.getTime() - event.createdAt.getTime() >
+            1000 * 60 * 30;
+
+        const payload: IServiceCreateEventPayload = {
+          name: event.name,
+          profileId,
+          projectId,
+          properties: clean(properties),
+          createdAt: event.createdAt.toISOString(),
+          country: geo.country,
+          city: geo.city,
+          region: geo.region,
+          continent: geo.continent,
+          os: uaInfo.os,
+          osVersion: uaInfo.osVersion,
+          browser: uaInfo.browser,
+          browserVersion: uaInfo.browserVersion,
+          device: uaInfo.device,
+          brand: uaInfo.brand,
+          model: uaInfo.model,
+          duration:
+            nextEvent && !isNextEventNewSession
+              ? nextEvent.createdAt.getTime() - event.createdAt.getTime()
+              : 0,
+          path,
+          referrer: properties?.referrer?.host ?? '', // TODO
+          referrerName: properties?.referrer?.host ?? '', // TODO
+        };
+
+        if (!prevEventAt) {
+          lastSessionStart = await createSessionStart(payload);
+        } else if (
+          event.createdAt.getTime() - prevEventAt.getTime() >
+          1000 * 60 * 30
+        ) {
+          if (eidx > 0 && prevEventAt && lastSessionStart) {
+            await createSessionEnd(prevEventAt, lastSessionStart, {
+              screenViews,
+              totalDuration,
+            });
+            totalDuration = 0;
+            screenViews = 0;
+            lastSessionStart = await createSessionStart(payload);
+          }
+        }
+
+        screenViews++;
+        totalDuration += payload.duration;
+        await createEvent(payload);
+      } // for each user event
+
+      const prevEventAt = events[events.length - 1]?.createdAt;
+      if (prevEventAt && lastSessionStart) {
+        await createSessionEnd(prevEventAt, lastSessionStart, {
+          screenViews,
+          totalDuration,
+        });
+      }
+    }
+  }
+}
+
+async function createEvent(event: IServiceCreateEventPayload) {
+  console.log(
+    `Create ${event.name} - ${event.path} - ${formatClickhouseDate(
+      event.createdAt
+    )} - ${event.duration / 1000} sec`
+  );
+  await createClickhouseEvent(event);
+}
+
+async function createSessionStart(event: IServiceCreateEventPayload) {
+  const session: IServiceCreateEventPayload = {
+    ...event,
+    duration: 0,
+    name: 'session_start',
+    createdAt: toISOString(getTime(event.createdAt) - 100),
+  };
+
+  await createEvent(session);
+  return session;
+}
+
+async function createSessionEnd(
+  prevEventAt: Date,
+  sessionStart: IServiceCreateEventPayload,
+  options: {
+    screenViews: number;
+    totalDuration: number;
+  }
+) {
+  const properties: Record<string, unknown> = {};
+  if (options.screenViews === 1) {
+    properties.__bounce = true;
+  } else {
+    properties.__bounce = false;
+  }
+
+  const session: IServiceCreateEventPayload = {
+    ...sessionStart,
+    properties: {
+      ...properties,
+      ...sessionStart.properties,
+    },
+    duration: options.totalDuration,
+    name: 'session_end',
+    createdAt: toISOString(prevEventAt.getTime() + 10),
+  };
+
+  await createEvent(session);
+  return session;
+}
+
+main();

apps/api/scripts/test-ua.ts

@@ -0,0 +1,18 @@
+//@ts-nocheck
+
+async function main() {
+  const crypto = require('crypto');
+
+  function createHash(data, len) {
+    return crypto
+      .createHash('shake256', { outputLength: len })
+      .update(data)
+      .digest('hex');
+  }
+
+  console.log(createHash('foo', 2));
+  // 1af9
+  console.log(createHash('foo', 32));
+  // 1af97f7818a28edf}
+}
+main();