move sdk packages to its own folder and rename api & dashboard

apps/api/src/bots/bots.readme.md

@@ -0,0 +1,6 @@
+# Device Detector - The Universal Device Detection library for parsing User Agents
+
+> @link https://matomo.org
+> @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or lat
+
+[bots.ts](./bots.ts) is based on matomo bots.yml file. You can see the original version here [here](https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml).

apps/api/src/bots/index.ts

@@ -0,0 +1,19 @@
+import bots from './bots';
+
+export function isBot(ua: string) {
+  const res = bots.find((bot) => {
+    if (new RegExp(bot.regex).test(ua)) {
+      return true;
+    }
+    return false;
+  });
+
+  if (!res) {
+    return null;
+  }
+
+  return {
+    name: res.name,
+    type: res.category || 'Unknown',
+  };
+}

apps/api/src/controllers/event.controller.ts

@@ -0,0 +1,325 @@
+import { logger, logInfo, noop } from '@/utils/logger';
+import { getClientIp, parseIp } from '@/utils/parseIp';
+import { getReferrerWithQuery, parseReferrer } from '@/utils/parseReferrer';
+import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
+import { isSameDomain, parsePath } from '@/utils/url';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { omit } from 'ramda';
+import { v4 as uuid } from 'uuid';
+
+import { generateDeviceId, getTime, toISOString } from '@mixan/common';
+import type { IServiceCreateEventPayload } from '@mixan/db';
+import { createEvent, getEvents, getSalts } from '@mixan/db';
+import type { JobsOptions } from '@mixan/queue';
+import { eventsQueue } from '@mixan/queue';
+import { findJobByPrefix } from '@mixan/queue/src/utils';
+import type { PostEventPayload } from '@mixan/sdk';
+
+const SESSION_TIMEOUT = 1000 * 60 * 30;
+const SESSION_END_TIMEOUT = SESSION_TIMEOUT + 1000;
+
+async function withTiming<T>(name: string, promise: T) {
+  try {
+    const start = Date.now();
+    const res = await promise;
+    const end = Date.now();
+    if (end - start > 1000) {
+      logInfo(`${name} took too long: ${end - start}ms`);
+    }
+    return res;
+  } catch (error) {
+    logger.error(error, `Failed to execute ${name}`);
+    throw error;
+  }
+}
+
+function createContextLogger(request: FastifyRequest) {
+  const _log = request.log.child({
+    requestId: request.id,
+    requestUrl: request.url,
+    headers: request.headers,
+    projectId: request.projectId,
+  });
+  let obj: Record<string, unknown> = {};
+  return {
+    add: (key: string, value: unknown) => (obj[key] = value),
+    addObject: (key: string, value: Record<string, unknown>) => {
+      obj = { ...obj, ...value };
+    },
+    send: (message: string, value: Record<string, unknown>) =>
+      _log.info({ ...obj, ...value }, message),
+  };
+}
+
+export async function postEvent(
+  request: FastifyRequest<{
+    Body: PostEventPayload;
+  }>,
+  reply: FastifyReply
+) {
+  const contextLogger = createContextLogger(request);
+  let deviceId: string | null = null;
+  const { projectId, body } = request;
+  const properties = body.properties ?? {};
+  const getProperty = (name: string): string | undefined => {
+    // replace thing is just for older sdks when we didn't have `__`
+    // remove when kiddokitchen app (24.09.02) is not used anymore
+    return (
+      ((properties[name] || properties[name.replace('__', '')]) as
+        | string
+        | null
+        | undefined) ?? undefined
+    );
+  };
+  const profileId = body.profileId ?? '';
+  const createdAt = new Date(body.timestamp);
+  const url = getProperty('__path');
+  const { path, hash, query } = parsePath(url);
+  const referrer = isSameDomain(getProperty('__referrer'), url)
+    ? null
+    : parseReferrer(getProperty('__referrer'));
+  const utmReferrer = getReferrerWithQuery(query);
+  const ip = getClientIp(request)!;
+  const origin = request.headers.origin!;
+  const ua = request.headers['user-agent']!;
+  const uaInfo = parseUserAgent(ua);
+  const salts = await getSalts();
+  const currentDeviceId = generateDeviceId({
+    salt: salts.current,
+    origin,
+    ip,
+    ua,
+  });
+  const previousDeviceId = generateDeviceId({
+    salt: salts.previous,
+    origin,
+    ip,
+    ua,
+  });
+
+  const isServerEvent = !ip && !origin && !isUserAgentSet(ua);
+
+  if (isServerEvent) {
+    const [event] = await withTiming(
+      'Get last event (server-event)',
+      getEvents(
+        `SELECT * FROM events WHERE name = 'screen_view' AND profile_id = '${profileId}' AND project_id = '${projectId}' ORDER BY created_at DESC LIMIT 1`
+      )
+    );
+
+    eventsQueue.add('event', {
+      type: 'createEvent',
+      payload: {
+        name: body.name,
+        deviceId: event?.deviceId || '',
+        sessionId: event?.sessionId || '',
+        profileId,
+        projectId,
+        properties: Object.assign(
+          {},
+          omit(['__path', '__referrer'], properties),
+          {
+            hash,
+            query,
+          }
+        ),
+        createdAt,
+        country: event?.country ?? '',
+        city: event?.city ?? '',
+        region: event?.region ?? '',
+        continent: event?.continent ?? '',
+        os: event?.os ?? '',
+        osVersion: event?.osVersion ?? '',
+        browser: event?.browser ?? '',
+        browserVersion: event?.browserVersion ?? '',
+        device: event?.device ?? '',
+        brand: event?.brand ?? '',
+        model: event?.model ?? '',
+        duration: 0,
+        path: event?.path ?? '',
+        referrer: event?.referrer ?? '',
+        referrerName: event?.referrerName ?? '',
+        referrerType: event?.referrerType ?? '',
+        profile: undefined,
+        meta: undefined,
+      },
+    });
+    return reply.status(200).send('');
+  }
+
+  const [geo, sessionEndJobCurrentDeviceId, sessionEndJobPreviousDeviceId] =
+    await withTiming(
+      'Get geo and jobs from queue',
+      Promise.all([
+        parseIp(ip),
+        findJobByPrefix(
+          eventsQueue,
+          `sessionEnd:${projectId}:${currentDeviceId}:`
+        ),
+        findJobByPrefix(
+          eventsQueue,
+          `sessionEnd:${projectId}:${previousDeviceId}:`
+        ),
+      ])
+    );
+
+  const createSessionStart =
+    !sessionEndJobCurrentDeviceId && !sessionEndJobPreviousDeviceId;
+
+  if (sessionEndJobCurrentDeviceId && !sessionEndJobPreviousDeviceId) {
+    deviceId = currentDeviceId;
+    const diff = Date.now() - sessionEndJobCurrentDeviceId.timestamp;
+    sessionEndJobCurrentDeviceId.changeDelay(diff + SESSION_END_TIMEOUT);
+  } else if (!sessionEndJobCurrentDeviceId && sessionEndJobPreviousDeviceId) {
+    deviceId = previousDeviceId;
+    const diff = Date.now() - sessionEndJobPreviousDeviceId.timestamp;
+    sessionEndJobPreviousDeviceId.changeDelay(diff + SESSION_END_TIMEOUT);
+  } else {
+    deviceId = currentDeviceId;
+    // Queue session end
+    eventsQueue.add(
+      'event',
+      {
+        type: 'createSessionEnd',
+        payload: {
+          deviceId,
+        },
+      },
+      {
+        delay: SESSION_END_TIMEOUT,
+        jobId: `sessionEnd:${projectId}:${deviceId}:${Date.now()}`,
+      }
+    );
+  }
+
+  const [[sessionStartEvent], prevEventJob] = await withTiming(
+    'Get session start event',
+    Promise.all([
+      getEvents(
+        `SELECT * FROM events WHERE name = 'session_start' AND device_id = '${deviceId}' AND project_id = '${projectId}' ORDER BY created_at DESC LIMIT 1`
+      ),
+      findJobByPrefix(eventsQueue, `event:${projectId}:${deviceId}:`),
+    ])
+  );
+
+  const payload: Omit<IServiceCreateEventPayload, 'id'> = {
+    name: body.name,
+    deviceId,
+    profileId,
+    projectId,
+    sessionId: createSessionStart ? uuid() : sessionStartEvent?.sessionId ?? '',
+    properties: Object.assign({}, omit(['__path', '__referrer'], properties), {
+      hash,
+      query,
+    }),
+    createdAt,
+    country: geo.country,
+    city: geo.city,
+    region: geo.region,
+    continent: geo.continent,
+    os: uaInfo.os,
+    osVersion: uaInfo.osVersion,
+    browser: uaInfo.browser,
+    browserVersion: uaInfo.browserVersion,
+    device: uaInfo.device,
+    brand: uaInfo.brand,
+    model: uaInfo.model,
+    duration: 0,
+    path: path,
+    referrer: referrer?.url,
+    referrerName: referrer?.name ?? utmReferrer?.name ?? '',
+    referrerType: referrer?.type ?? utmReferrer?.type ?? '',
+    profile: undefined,
+    meta: undefined,
+  };
+
+  const isDelayed = prevEventJob ? await prevEventJob?.isDelayed() : false;
+
+  if (isDelayed && prevEventJob && prevEventJob.data.type === 'createEvent') {
+    const prevEvent = prevEventJob.data.payload;
+    const duration = getTime(payload.createdAt) - getTime(prevEvent.createdAt);
+    contextLogger.add('prevEvent', prevEvent);
+
+    // Set path from prev screen_view event if current event is not a screen_view
+    if (payload.name != 'screen_view') {
+      payload.path = prevEvent.path;
+    }
+
+    if (payload.name === 'screen_view') {
+      if (duration < 0) {
+        contextLogger.send('duration is wrong', {
+          payload,
+          duration,
+        });
+      } else {
+        // Skip update duration if it's wrong
+        // Seems like request is not in right order
+        await withTiming(
+          'Update previous job with duration',
+          prevEventJob.updateData({
+            type: 'createEvent',
+            payload: {
+              ...prevEvent,
+              duration,
+            },
+          })
+        );
+      }
+
+      await withTiming('Promote previous job', prevEventJob.promote());
+    }
+  } else if (payload.name !== 'screen_view') {
+    contextLogger.send('no previous job', {
+      prevEventJob,
+      payload,
+    });
+  }
+
+  if (createSessionStart) {
+    // We do not need to queue session_start
+    await withTiming(
+      'Create session start event',
+      createEvent({
+        ...payload,
+        name: 'session_start',
+        // @ts-expect-error
+        createdAt: toISOString(getTime(payload.createdAt) - 100),
+      })
+    );
+  }
+
+  const options: JobsOptions = {};
+  if (payload.name === 'screen_view') {
+    options.delay = SESSION_TIMEOUT;
+    options.jobId = `event:${projectId}:${deviceId}:${Date.now()}`;
+  }
+
+  contextLogger.send('event is queued', {
+    ip,
+    origin,
+    ua,
+    uaInfo,
+    referrer,
+    profileId,
+    projectId,
+    deviceId,
+    geo,
+    sessionStartEvent,
+    path,
+    payload,
+  });
+
+  // Queue current event
+  eventsQueue
+    .add(
+      'event',
+      {
+        type: 'createEvent',
+        payload,
+      },
+      options
+    )
+    .catch(noop('Failed to queue event'));
+
+  reply.status(202).send(deviceId);
+}

apps/api/src/controllers/live.controller.ts

@@ -0,0 +1,109 @@
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import type * as WebSocket from 'ws';
+
+import { getSafeJson } from '@mixan/common';
+import type { IServiceCreateEventPayload } from '@mixan/db';
+import { getEvents, getLiveVisitors } from '@mixan/db';
+import { redis, redisPub, redisSub } from '@mixan/redis';
+
+export function getLiveEventInfo(key: string) {
+  return key.split(':').slice(2) as [string, string];
+}
+
+export async function test(
+  req: FastifyRequest<{
+    Params: {
+      projectId: string;
+    };
+  }>,
+  reply: FastifyReply
+) {
+  const [event] = await getEvents(
+    `SELECT * FROM events WHERE project_id = '${req.params.projectId}' AND name = 'screen_view' LIMIT 1`
+  );
+  if (!event) {
+    return reply.status(404).send('No event found');
+  }
+  redisPub.publish('event', JSON.stringify(event));
+  redis.set(
+    `live:event:${event.projectId}:${Math.random() * 1000}`,
+    '',
+    'EX',
+    10
+  );
+  reply.status(202).send(event);
+}
+
+export function wsVisitors(
+  connection: {
+    socket: WebSocket;
+  },
+  req: FastifyRequest<{
+    Params: {
+      projectId: string;
+    };
+  }>
+) {
+  const { params } = req;
+
+  redisSub.subscribe('event');
+  redisSub.psubscribe('__key*:expired');
+
+  const message = (channel: string, message: string) => {
+    if (channel === 'event') {
+      const event = getSafeJson<IServiceCreateEventPayload>(message);
+      if (event?.projectId === params.projectId) {
+        getLiveVisitors(params.projectId).then((count) => {
+          connection.socket.send(String(count));
+        });
+      }
+    }
+  };
+  const pmessage = (pattern: string, channel: string, message: string) => {
+    const [projectId] = getLiveEventInfo(message);
+    if (projectId && projectId === params.projectId) {
+      getLiveVisitors(params.projectId).then((count) => {
+        connection.socket.send(String(count));
+      });
+    }
+  };
+
+  redisSub.on('message', message);
+  redisSub.on('pmessage', pmessage);
+
+  connection.socket.on('close', () => {
+    redisSub.unsubscribe('event');
+    redisSub.punsubscribe('__key*:expired');
+    redisSub.off('message', message);
+    redisSub.off('pmessage', pmessage);
+  });
+}
+
+export function wsEvents(
+  connection: {
+    socket: WebSocket;
+  },
+  req: FastifyRequest<{
+    Params: {
+      projectId: string;
+    };
+  }>
+) {
+  const { params } = req;
+
+  redisSub.subscribe('event');
+
+  const message = (channel: string, message: string) => {
+    const event = getSafeJson<IServiceCreateEventPayload>(message);
+    if (event?.projectId === params.projectId) {
+      connection.socket.send(JSON.stringify(event));
+    }
+  };
+
+  redisSub.on('message', message);
+
+  connection.socket.on('close', () => {
+    redisSub.unsubscribe('event');
+    redisSub.off('message', message);
+  });
+}

apps/api/src/controllers/misc.controller.ts

@@ -0,0 +1,127 @@
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import icoToPng from 'ico-to-png';
+import sharp from 'sharp';
+
+import { createHash } from '@mixan/common';
+import { redis } from '@mixan/redis';
+
+interface GetFaviconParams {
+  url: string;
+}
+
+async function getImageBuffer(url: string) {
+  try {
+    const res = await fetch(url);
+    const contentType = res.headers.get('content-type');
+
+    if (!contentType?.includes('image')) {
+      return null;
+    }
+
+    if (!res.ok) {
+      return null;
+    }
+
+    if (contentType === 'image/x-icon' || url.endsWith('.ico')) {
+      const arrayBuffer = await res.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      return await icoToPng(buffer, 30);
+    }
+
+    return await sharp(await res.arrayBuffer())
+      .resize(30, 30, {
+        fit: 'cover',
+      })
+      .png()
+      .toBuffer();
+  } catch (e) {
+    console.log('Failed to get image from url', url);
+    console.log(e);
+  }
+}
+
+const imageExtensions = ['svg', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'ico'];
+
+export async function getFavicon(
+  request: FastifyRequest<{
+    Querystring: GetFaviconParams;
+  }>,
+  reply: FastifyReply
+) {
+  function sendBuffer(buffer: Buffer, cacheKey?: string) {
+    if (cacheKey) {
+      redis.set(`favicon:${cacheKey}`, buffer.toString('base64'));
+    }
+    reply.type('image/png');
+    console.log('buffer', buffer.byteLength);
+
+    return reply.send(buffer);
+  }
+
+  if (!request.query.url) {
+    return reply.status(404).send('Not found');
+  }
+
+  const url = decodeURIComponent(request.query.url);
+
+  // DIRECT IMAGE
+  if (imageExtensions.find((ext) => url.endsWith(ext))) {
+    const cacheKey = createHash(url, 32);
+    const cache = await redis.get(`favicon:${cacheKey}`);
+    if (cache) {
+      return sendBuffer(Buffer.from(cache, 'base64'));
+    }
+    const buffer = await getImageBuffer(url);
+    if (buffer && buffer.byteLength > 0) {
+      return sendBuffer(buffer, cacheKey);
+    }
+  }
+
+  const { hostname, origin } = new URL(url);
+  const cache = await redis.get(`favicon:${hostname}`);
+  if (cache) {
+    return sendBuffer(Buffer.from(cache, 'base64'));
+  }
+
+  // TRY FAVICON.ICO
+  const buffer = await getImageBuffer(`${origin}/favicon.ico`);
+  if (buffer && buffer.byteLength > 0) {
+    return sendBuffer(buffer, hostname);
+  }
+
+  // PARSE HTML
+  const res = await fetch(url).then((res) => res.text());
+
+  function findFavicon(res: string) {
+    const match = res.match(
+      /(\<link(.+?)image\/x-icon(.+?)\>|\<link(.+?)shortcut\sicon(.+?)\>)/
+    );
+    if (!match) {
+      return null;
+    }
+
+    return match[0].match(/href="(.+?)"/)?.[1] ?? null;
+  }
+
+  const favicon = findFavicon(res);
+  if (favicon) {
+    const buffer = await getImageBuffer(favicon);
+
+    if (buffer && buffer.byteLength > 0) {
+      return sendBuffer(buffer, hostname);
+    }
+  }
+
+  return reply.status(404).send('Not found');
+}
+
+export async function clearFavicons(
+  request: FastifyRequest,
+  reply: FastifyReply
+) {
+  const keys = await redis.keys('favicon:*');
+  for (const key of keys) {
+    await redis.del(key);
+  }
+  return reply.status(404).send('OK');
+}

apps/api/src/controllers/profile.controller.ts

@@ -0,0 +1,110 @@
+import { getClientIp, parseIp } from '@/utils/parseIp';
+import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { assocPath, pathOr } from 'ramda';
+
+import { getProfileById, upsertProfile } from '@mixan/db';
+import type { IncrementProfilePayload, UpdateProfilePayload } from '@mixan/sdk';
+
+export async function updateProfile(
+  request: FastifyRequest<{
+    Body: UpdateProfilePayload;
+  }>,
+  reply: FastifyReply
+) {
+  const { profileId, properties, ...rest } = request.body;
+  const projectId = request.projectId;
+  const ip = getClientIp(request)!;
+  const ua = request.headers['user-agent']!;
+  const uaInfo = parseUserAgent(ua);
+  const geo = await parseIp(ip);
+
+  await upsertProfile({
+    id: profileId,
+    projectId,
+    properties: {
+      ...(properties ?? {}),
+      ...(ip ? geo : {}),
+      ...(isUserAgentSet(ua) ? uaInfo : {}),
+    },
+    ...rest,
+  });
+
+  reply.status(202).send(profileId);
+}
+
+export async function incrementProfileProperty(
+  request: FastifyRequest<{
+    Body: IncrementProfilePayload;
+  }>,
+  reply: FastifyReply
+) {
+  const { profileId, property, value } = request.body;
+  const projectId = request.projectId;
+
+  const profile = await getProfileById(profileId);
+  if (!profile) {
+    return reply.status(404).send('Not found');
+  }
+
+  const parsed = parseInt(
+    pathOr<string>('0', property.split('.'), profile.properties),
+    10
+  );
+
+  if (isNaN(parsed)) {
+    return reply.status(400).send('Not number');
+  }
+
+  profile.properties = assocPath(
+    property.split('.'),
+    parsed + value,
+    profile.properties
+  );
+
+  await upsertProfile({
+    id: profile.id,
+    projectId,
+    properties: profile.properties,
+  });
+
+  reply.status(202).send(profile.id);
+}
+
+export async function decrementProfileProperty(
+  request: FastifyRequest<{
+    Body: IncrementProfilePayload;
+  }>,
+  reply: FastifyReply
+) {
+  const { profileId, property, value } = request.body;
+  const projectId = request.projectId;
+
+  const profile = await getProfileById(profileId);
+  if (!profile) {
+    return reply.status(404).send('Not found');
+  }
+
+  const parsed = parseInt(
+    pathOr<string>('0', property.split('.'), profile.properties),
+    10
+  );
+
+  if (isNaN(parsed)) {
+    return reply.status(400).send('Not number');
+  }
+
+  profile.properties = assocPath(
+    property.split('.'),
+    parsed - value,
+    profile.properties
+  );
+
+  await upsertProfile({
+    id: profile.id,
+    projectId,
+    properties: profile.properties,
+  });
+
+  reply.status(202).send(profile.id);
+}

apps/api/src/index.ts

@@ -0,0 +1,73 @@
+import cors from '@fastify/cors';
+import Fastify from 'fastify';
+
+import { redisPub } from '@mixan/redis';
+
+import eventRouter from './routes/event.router';
+import liveRouter from './routes/live.router';
+import miscRouter from './routes/misc.router';
+import profileRouter from './routes/profile.router';
+import { logger, logInfo } from './utils/logger';
+
+declare module 'fastify' {
+  interface FastifyRequest {
+    projectId: string;
+  }
+}
+
+const port = parseInt(process.env.API_PORT || '3000', 10);
+
+const startServer = async () => {
+  logInfo('Starting server');
+  try {
+    const fastify = Fastify({
+      logger: logger,
+    });
+
+    fastify.register(cors, {
+      origin: '*',
+    });
+
+    fastify.decorateRequest('projectId', '');
+    fastify.register(eventRouter, { prefix: '/event' });
+    fastify.register(profileRouter, { prefix: '/profile' });
+    fastify.register(liveRouter, { prefix: '/live' });
+    fastify.register(miscRouter, { prefix: '/misc' });
+    fastify.setErrorHandler((error, request, reply) => {
+      fastify.log.error(error);
+    });
+    fastify.get('/', (request, reply) => {
+      reply.send({ name: 'openpanel sdk api' });
+    });
+    // fastify.get('/health-check', async (request, reply) => {
+    //   try {
+    //     await utils.healthCheck()
+    //     reply.status(200).send()
+    //   } catch (e) {
+    //     reply.status(500).send()
+    //   }
+    // })
+    if (process.env.NODE_ENV === 'production') {
+      for (const signal of ['SIGINT', 'SIGTERM']) {
+        process.on(signal, (err) => {
+          logger.fatal(err, `uncaught exception detected ${signal}`);
+          fastify.close().then((err) => {
+            process.exit(err ? 1 : 0);
+          });
+        });
+      }
+    }
+
+    await fastify.listen({
+      host: process.env.NODE_ENV === 'production' ? '0.0.0.0' : 'localhost',
+      port,
+    });
+
+    // Notify when keys expires
+    redisPub.config('SET', 'notify-keyspace-events', 'Ex');
+  } catch (e) {
+    console.error(e);
+  }
+};
+
+startServer();

apps/api/src/referrers/referrers.readme.md

@@ -0,0 +1,5 @@
+# Snowplow Referer Parser
+
+The file index.ts in this dir is generated from snowplows referer database [Snowplow Referer Parser](https://github.com/snowplow-referer-parser/referer-parser).
+
+The orginal [referers.yml](https://github.com/snowplow-referer-parser/referer-parser/blob/master/resources/referers.yml) is based on Piwik's SearchEngines.php and Socials.php, copyright 2012 Matthieu Aubry and available under the GNU General Public License v3.

apps/api/src/routes/event.router.ts

@@ -0,0 +1,59 @@
+import { isBot } from '@/bots';
+import * as controller from '@/controllers/event.controller';
+import { validateSdkRequest } from '@/utils/auth';
+import type { FastifyPluginCallback, FastifyRequest } from 'fastify';
+
+import { createBotEvent } from '@mixan/db';
+import type { PostEventPayload } from '@mixan/sdk';
+
+const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
+  fastify.addHook(
+    'preHandler',
+    async (
+      req: FastifyRequest<{
+        Body: PostEventPayload;
+      }>,
+      reply
+    ) => {
+      try {
+        const projectId = await validateSdkRequest(req.headers);
+        req.projectId = projectId;
+
+        const bot = req.headers['user-agent']
+          ? isBot(req.headers['user-agent'])
+          : null;
+
+        if (bot) {
+          const path = (req.body?.properties?.__path ||
+            req.body?.properties?.path) as string | undefined;
+          req.log.warn({ ...req.headers, bot }, 'Bot detected (event)');
+          await createBotEvent({
+            ...bot,
+            projectId,
+            path: path ?? '',
+            createdAt: new Date(req.body?.timestamp),
+          });
+          reply.status(202).send('OK');
+        }
+      } catch (e) {
+        req.log.error(e, 'Failed to create bot event');
+        reply.status(401).send();
+        return;
+      }
+    }
+  );
+
+  fastify.route({
+    method: 'POST',
+    url: '/',
+    handler: controller.postEvent,
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/',
+    handler: controller.postEvent,
+  });
+  done();
+};
+
+export default eventRouter;

apps/api/src/routes/live.router.ts

@@ -0,0 +1,27 @@
+import * as controller from '@/controllers/live.controller';
+import fastifyWS from '@fastify/websocket';
+import type { FastifyPluginCallback } from 'fastify';
+
+const liveRouter: FastifyPluginCallback = (fastify, opts, done) => {
+  fastify.route({
+    method: 'GET',
+    url: '/events/test/:projectId',
+    handler: controller.test,
+  });
+
+  fastify.register(fastifyWS);
+
+  fastify.register((fastify, _, done) => {
+    fastify.get(
+      '/visitors/:projectId',
+      { websocket: true },
+      controller.wsVisitors
+    );
+    fastify.get('/events/:projectId', { websocket: true }, controller.wsEvents);
+    done();
+  });
+
+  done();
+};
+
+export default liveRouter;

apps/api/src/routes/misc.router.ts

@@ -0,0 +1,20 @@
+import * as controller from '@/controllers/misc.controller';
+import type { FastifyPluginCallback } from 'fastify';
+
+const miscRouter: FastifyPluginCallback = (fastify, opts, done) => {
+  fastify.route({
+    method: 'GET',
+    url: '/favicon',
+    handler: controller.getFavicon,
+  });
+
+  fastify.route({
+    method: 'GET',
+    url: '/favicon/clear',
+    handler: controller.clearFavicons,
+  });
+
+  done();
+};
+
+export default miscRouter;

apps/api/src/routes/profile.router.ts

@@ -0,0 +1,46 @@
+import { isBot } from '@/bots';
+import * as controller from '@/controllers/profile.controller';
+import { validateSdkRequest } from '@/utils/auth';
+import type { FastifyPluginCallback } from 'fastify';
+
+const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
+  fastify.addHook('preHandler', async (req, reply) => {
+    try {
+      const projectId = await validateSdkRequest(req.headers);
+      req.projectId = projectId;
+
+      const bot = req.headers['user-agent']
+        ? isBot(req.headers['user-agent'])
+        : null;
+
+      if (bot) {
+        reply.log.warn({ ...req.headers, bot }, 'Bot detected (profile)');
+        reply.status(202).send('OK');
+      }
+    } catch (e) {
+      reply.status(401).send();
+      return;
+    }
+  });
+
+  fastify.route({
+    method: 'POST',
+    url: '/',
+    handler: controller.updateProfile,
+  });
+
+  fastify.route({
+    method: 'POST',
+    url: '/increment',
+    handler: controller.incrementProfileProperty,
+  });
+
+  fastify.route({
+    method: 'POST',
+    url: '/decrement',
+    handler: controller.decrementProfileProperty,
+  });
+  done();
+};
+
+export default eventRouter;

apps/api/src/utils/auth.ts

@@ -0,0 +1,43 @@
+import type { RawRequestDefaultExpression } from 'fastify';
+
+import { verifyPassword } from '@mixan/common';
+import { db } from '@mixan/db';
+
+export async function validateSdkRequest(
+  headers: RawRequestDefaultExpression['headers']
+): Promise<string> {
+  const clientId = headers['mixan-client-id'] as string;
+  const clientSecret = headers['mixan-client-secret'] as string;
+  const origin = headers.origin;
+  if (!clientId) {
+    throw new Error('Misisng client id');
+  }
+
+  const client = await db.client.findUnique({
+    where: {
+      id: clientId,
+    },
+  });
+
+  if (!client) {
+    throw new Error('Invalid client id');
+  }
+
+  if (client.secret) {
+    if (!(await verifyPassword(clientSecret || '', client.secret))) {
+      throw new Error('Invalid client secret');
+    }
+  } else if (client.cors !== '*') {
+    const domainAllowed = client.cors.split(',').find((domain) => {
+      if (domain === origin) {
+        return true;
+      }
+    });
+
+    if (!domainAllowed) {
+      throw new Error('Invalid cors settings');
+    }
+  }
+
+  return client.project_id;
+}

apps/api/src/utils/logger.ts

@@ -0,0 +1,29 @@
+import type { TransportTargetOptions } from 'pino';
+import pino from 'pino';
+
+const targets: TransportTargetOptions[] =
+  process.env.NODE_ENV === 'production'
+    ? [
+        {
+          target: '@logtail/pino',
+          options: { sourceToken: process.env.BETTERSTACK_TOKEN },
+        },
+      ]
+    : [
+        {
+          target: 'pino-pretty',
+        },
+      ];
+
+const transport = pino.transport({
+  targets,
+});
+
+export const logger = pino(transport);
+
+export function logInfo(msg: string, obj?: unknown) {
+  logger.info(obj, msg);
+}
+
+export const noop = (message: string) => (error: unknown) =>
+  logger.error(error, message);

apps/api/src/utils/parseIp.ts

@@ -0,0 +1,61 @@
+import type { FastifyRequest } from 'fastify';
+
+import { logger } from './logger';
+
+interface RemoteIpLookupResponse {
+  country: string | undefined;
+  city: string | undefined;
+  stateprov: string | undefined;
+  continent: string | undefined;
+}
+
+interface GeoLocation {
+  country: string | undefined;
+  city: string | undefined;
+  region: string | undefined;
+  continent: string | undefined;
+}
+
+const geo: GeoLocation = {
+  country: undefined,
+  city: undefined,
+  region: undefined,
+  continent: undefined,
+};
+
+const ignore = ['127.0.0.1', '::1'];
+
+export function getClientIp(req: FastifyRequest) {
+  if (req.headers['cf-connecting-ip']) {
+    return String(req.headers['cf-connecting-ip']);
+  }
+
+  if (req.headers['x-forwarded-for']) {
+    return String(req.headers['x-forwarded-for']);
+  }
+
+  return null;
+}
+
+export async function parseIp(ip?: string): Promise<GeoLocation> {
+  if (!ip || ignore.includes(ip)) {
+    return geo;
+  }
+
+  try {
+    const geo = await fetch(`${process.env.GEO_IP_HOST}/${ip}`, {
+      signal: AbortSignal.timeout(2000),
+    });
+    const res = (await geo.json()) as RemoteIpLookupResponse;
+
+    return {
+      country: res.country,
+      city: res.city,
+      region: res.stateprov,
+      continent: res.continent,
+    };
+  } catch (e) {
+    logger.error('Failed to fetch geo location for ip', e);
+    return geo;
+  }
+}

apps/api/src/utils/parseReferrer.ts

@@ -0,0 +1,50 @@
+import { stripTrailingSlash } from '@mixan/common';
+
+import referrers from '../referrers';
+
+function getHostname(url: string | undefined) {
+  if (!url) {
+    return '';
+  }
+
+  try {
+    return new URL(url).hostname;
+  } catch (e) {
+    return '';
+  }
+}
+
+export function parseReferrer(url: string | undefined) {
+  const hostname = getHostname(url);
+  const match = referrers[hostname] ?? referrers[hostname.replace('www.', '')];
+
+  return {
+    name: match?.name ?? '',
+    type: match?.type ?? 'unknown',
+    url: stripTrailingSlash(url ?? ''),
+  };
+}
+
+export function getReferrerWithQuery(
+  query: Record<string, string> | undefined
+) {
+  if (!query) {
+    return null;
+  }
+
+  const source = query.utm_source ?? query.ref ?? query.utm_referrer ?? '';
+
+  const match = Object.values(referrers).find(
+    (referrer) => referrer.name.toLowerCase() === source?.toLowerCase()
+  );
+
+  if (!match) {
+    return null;
+  }
+
+  return {
+    name: match.name,
+    type: match.type,
+    url: '',
+  };
+}

apps/api/src/utils/parseUserAgent.ts

@@ -0,0 +1,33 @@
+import { UAParser } from 'ua-parser-js';
+
+export function isUserAgentSet(ua: string) {
+  return ua !== 'node' && ua !== 'undici' && !!ua;
+}
+
+export function parseUserAgent(ua: string) {
+  const res = new UAParser(ua).getResult();
+  return {
+    os: res.os.name,
+    osVersion: res.os.version,
+    browser: res.browser.name,
+    browserVersion: res.browser.version,
+    device: res.device.type ?? getDevice(ua),
+    brand: res.device.vendor,
+    model: res.device.model,
+  };
+}
+
+export function getDevice(ua: string) {
+  const t1 =
+    /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(
+      ua
+    );
+  const t2 =
+    /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test(
+      ua.slice(0, 4)
+    );
+  if (t1 || t2) {
+    return 'mobile';
+  }
+  return 'desktop';
+}

apps/api/src/utils/url.ts

@@ -0,0 +1,48 @@
+export function parseSearchParams(
+  params: URLSearchParams
+): Record<string, string> | undefined {
+  const result: Record<string, string> = {};
+  for (const [key, value] of params.entries()) {
+    result[key] = value;
+  }
+  return Object.keys(result).length ? result : undefined;
+}
+
+export function parsePath(path?: string): {
+  query?: Record<string, string>;
+  path: string;
+  hash?: string;
+} {
+  if (!path) {
+    return {
+      path: '',
+    };
+  }
+
+  try {
+    const url = new URL(path);
+    return {
+      query: parseSearchParams(url.searchParams),
+      path: url.pathname,
+      hash: url.hash || undefined,
+    };
+  } catch (error) {
+    return {
+      path,
+    };
+  }
+}
+
+export function isSameDomain(
+  url1: string | undefined,
+  url2: string | undefined
+) {
+  if (!url1 || !url2) {
+    return false;
+  }
+  try {
+    return new URL(url1).hostname === new URL(url2).hostname;
+  } catch (e) {
+    return false;
+  }
+}