move sdk packages to its own folder and rename api & dashboard

apps/api/src/controllers/event.controller.ts

@@ -0,0 +1,325 @@
+import { logger, logInfo, noop } from '@/utils/logger';
+import { getClientIp, parseIp } from '@/utils/parseIp';
+import { getReferrerWithQuery, parseReferrer } from '@/utils/parseReferrer';
+import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
+import { isSameDomain, parsePath } from '@/utils/url';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { omit } from 'ramda';
+import { v4 as uuid } from 'uuid';
+
+import { generateDeviceId, getTime, toISOString } from '@mixan/common';
+import type { IServiceCreateEventPayload } from '@mixan/db';
+import { createEvent, getEvents, getSalts } from '@mixan/db';
+import type { JobsOptions } from '@mixan/queue';
+import { eventsQueue } from '@mixan/queue';
+import { findJobByPrefix } from '@mixan/queue/src/utils';
+import type { PostEventPayload } from '@mixan/sdk';
+
+const SESSION_TIMEOUT = 1000 * 60 * 30;
+const SESSION_END_TIMEOUT = SESSION_TIMEOUT + 1000;
+
+async function withTiming<T>(name: string, promise: T) {
+  try {
+    const start = Date.now();
+    const res = await promise;
+    const end = Date.now();
+    if (end - start > 1000) {
+      logInfo(`${name} took too long: ${end - start}ms`);
+    }
+    return res;
+  } catch (error) {
+    logger.error(error, `Failed to execute ${name}`);
+    throw error;
+  }
+}
+
+function createContextLogger(request: FastifyRequest) {
+  const _log = request.log.child({
+    requestId: request.id,
+    requestUrl: request.url,
+    headers: request.headers,
+    projectId: request.projectId,
+  });
+  let obj: Record<string, unknown> = {};
+  return {
+    add: (key: string, value: unknown) => (obj[key] = value),
+    addObject: (key: string, value: Record<string, unknown>) => {
+      obj = { ...obj, ...value };
+    },
+    send: (message: string, value: Record<string, unknown>) =>
+      _log.info({ ...obj, ...value }, message),
+  };
+}
+
+export async function postEvent(
+  request: FastifyRequest<{
+    Body: PostEventPayload;
+  }>,
+  reply: FastifyReply
+) {
+  const contextLogger = createContextLogger(request);
+  let deviceId: string | null = null;
+  const { projectId, body } = request;
+  const properties = body.properties ?? {};
+  const getProperty = (name: string): string | undefined => {
+    // replace thing is just for older sdks when we didn't have `__`
+    // remove when kiddokitchen app (24.09.02) is not used anymore
+    return (
+      ((properties[name] || properties[name.replace('__', '')]) as
+        | string
+        | null
+        | undefined) ?? undefined
+    );
+  };
+  const profileId = body.profileId ?? '';
+  const createdAt = new Date(body.timestamp);
+  const url = getProperty('__path');
+  const { path, hash, query } = parsePath(url);
+  const referrer = isSameDomain(getProperty('__referrer'), url)
+    ? null
+    : parseReferrer(getProperty('__referrer'));
+  const utmReferrer = getReferrerWithQuery(query);
+  const ip = getClientIp(request)!;
+  const origin = request.headers.origin!;
+  const ua = request.headers['user-agent']!;
+  const uaInfo = parseUserAgent(ua);
+  const salts = await getSalts();
+  const currentDeviceId = generateDeviceId({
+    salt: salts.current,
+    origin,
+    ip,
+    ua,
+  });
+  const previousDeviceId = generateDeviceId({
+    salt: salts.previous,
+    origin,
+    ip,
+    ua,
+  });
+
+  const isServerEvent = !ip && !origin && !isUserAgentSet(ua);
+
+  if (isServerEvent) {
+    const [event] = await withTiming(
+      'Get last event (server-event)',
+      getEvents(
+        `SELECT * FROM events WHERE name = 'screen_view' AND profile_id = '${profileId}' AND project_id = '${projectId}' ORDER BY created_at DESC LIMIT 1`
+      )
+    );
+
+    eventsQueue.add('event', {
+      type: 'createEvent',
+      payload: {
+        name: body.name,
+        deviceId: event?.deviceId || '',
+        sessionId: event?.sessionId || '',
+        profileId,
+        projectId,
+        properties: Object.assign(
+          {},
+          omit(['__path', '__referrer'], properties),
+          {
+            hash,
+            query,
+          }
+        ),
+        createdAt,
+        country: event?.country ?? '',
+        city: event?.city ?? '',
+        region: event?.region ?? '',
+        continent: event?.continent ?? '',
+        os: event?.os ?? '',
+        osVersion: event?.osVersion ?? '',
+        browser: event?.browser ?? '',
+        browserVersion: event?.browserVersion ?? '',
+        device: event?.device ?? '',
+        brand: event?.brand ?? '',
+        model: event?.model ?? '',
+        duration: 0,
+        path: event?.path ?? '',
+        referrer: event?.referrer ?? '',
+        referrerName: event?.referrerName ?? '',
+        referrerType: event?.referrerType ?? '',
+        profile: undefined,
+        meta: undefined,
+      },
+    });
+    return reply.status(200).send('');
+  }
+
+  const [geo, sessionEndJobCurrentDeviceId, sessionEndJobPreviousDeviceId] =
+    await withTiming(
+      'Get geo and jobs from queue',
+      Promise.all([
+        parseIp(ip),
+        findJobByPrefix(
+          eventsQueue,
+          `sessionEnd:${projectId}:${currentDeviceId}:`
+        ),
+        findJobByPrefix(
+          eventsQueue,
+          `sessionEnd:${projectId}:${previousDeviceId}:`
+        ),
+      ])
+    );
+
+  const createSessionStart =
+    !sessionEndJobCurrentDeviceId && !sessionEndJobPreviousDeviceId;
+
+  if (sessionEndJobCurrentDeviceId && !sessionEndJobPreviousDeviceId) {
+    deviceId = currentDeviceId;
+    const diff = Date.now() - sessionEndJobCurrentDeviceId.timestamp;
+    sessionEndJobCurrentDeviceId.changeDelay(diff + SESSION_END_TIMEOUT);
+  } else if (!sessionEndJobCurrentDeviceId && sessionEndJobPreviousDeviceId) {
+    deviceId = previousDeviceId;
+    const diff = Date.now() - sessionEndJobPreviousDeviceId.timestamp;
+    sessionEndJobPreviousDeviceId.changeDelay(diff + SESSION_END_TIMEOUT);
+  } else {
+    deviceId = currentDeviceId;
+    // Queue session end
+    eventsQueue.add(
+      'event',
+      {
+        type: 'createSessionEnd',
+        payload: {
+          deviceId,
+        },
+      },
+      {
+        delay: SESSION_END_TIMEOUT,
+        jobId: `sessionEnd:${projectId}:${deviceId}:${Date.now()}`,
+      }
+    );
+  }
+
+  const [[sessionStartEvent], prevEventJob] = await withTiming(
+    'Get session start event',
+    Promise.all([
+      getEvents(
+        `SELECT * FROM events WHERE name = 'session_start' AND device_id = '${deviceId}' AND project_id = '${projectId}' ORDER BY created_at DESC LIMIT 1`
+      ),
+      findJobByPrefix(eventsQueue, `event:${projectId}:${deviceId}:`),
+    ])
+  );
+
+  const payload: Omit<IServiceCreateEventPayload, 'id'> = {
+    name: body.name,
+    deviceId,
+    profileId,
+    projectId,
+    sessionId: createSessionStart ? uuid() : sessionStartEvent?.sessionId ?? '',
+    properties: Object.assign({}, omit(['__path', '__referrer'], properties), {
+      hash,
+      query,
+    }),
+    createdAt,
+    country: geo.country,
+    city: geo.city,
+    region: geo.region,
+    continent: geo.continent,
+    os: uaInfo.os,
+    osVersion: uaInfo.osVersion,
+    browser: uaInfo.browser,
+    browserVersion: uaInfo.browserVersion,
+    device: uaInfo.device,
+    brand: uaInfo.brand,
+    model: uaInfo.model,
+    duration: 0,
+    path: path,
+    referrer: referrer?.url,
+    referrerName: referrer?.name ?? utmReferrer?.name ?? '',
+    referrerType: referrer?.type ?? utmReferrer?.type ?? '',
+    profile: undefined,
+    meta: undefined,
+  };
+
+  const isDelayed = prevEventJob ? await prevEventJob?.isDelayed() : false;
+
+  if (isDelayed && prevEventJob && prevEventJob.data.type === 'createEvent') {
+    const prevEvent = prevEventJob.data.payload;
+    const duration = getTime(payload.createdAt) - getTime(prevEvent.createdAt);
+    contextLogger.add('prevEvent', prevEvent);
+
+    // Set path from prev screen_view event if current event is not a screen_view
+    if (payload.name != 'screen_view') {
+      payload.path = prevEvent.path;
+    }
+
+    if (payload.name === 'screen_view') {
+      if (duration < 0) {
+        contextLogger.send('duration is wrong', {
+          payload,
+          duration,
+        });
+      } else {
+        // Skip update duration if it's wrong
+        // Seems like request is not in right order
+        await withTiming(
+          'Update previous job with duration',
+          prevEventJob.updateData({
+            type: 'createEvent',
+            payload: {
+              ...prevEvent,
+              duration,
+            },
+          })
+        );
+      }
+
+      await withTiming('Promote previous job', prevEventJob.promote());
+    }
+  } else if (payload.name !== 'screen_view') {
+    contextLogger.send('no previous job', {
+      prevEventJob,
+      payload,
+    });
+  }
+
+  if (createSessionStart) {
+    // We do not need to queue session_start
+    await withTiming(
+      'Create session start event',
+      createEvent({
+        ...payload,
+        name: 'session_start',
+        // @ts-expect-error
+        createdAt: toISOString(getTime(payload.createdAt) - 100),
+      })
+    );
+  }
+
+  const options: JobsOptions = {};
+  if (payload.name === 'screen_view') {
+    options.delay = SESSION_TIMEOUT;
+    options.jobId = `event:${projectId}:${deviceId}:${Date.now()}`;
+  }
+
+  contextLogger.send('event is queued', {
+    ip,
+    origin,
+    ua,
+    uaInfo,
+    referrer,
+    profileId,
+    projectId,
+    deviceId,
+    geo,
+    sessionStartEvent,
+    path,
+    payload,
+  });
+
+  // Queue current event
+  eventsQueue
+    .add(
+      'event',
+      {
+        type: 'createEvent',
+        payload,
+      },
+      options
+    )
+    .catch(noop('Failed to queue event'));
+
+  reply.status(202).send(deviceId);
+}

apps/api/src/controllers/live.controller.ts

@@ -0,0 +1,109 @@
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import type * as WebSocket from 'ws';
+
+import { getSafeJson } from '@mixan/common';
+import type { IServiceCreateEventPayload } from '@mixan/db';
+import { getEvents, getLiveVisitors } from '@mixan/db';
+import { redis, redisPub, redisSub } from '@mixan/redis';
+
+export function getLiveEventInfo(key: string) {
+  return key.split(':').slice(2) as [string, string];
+}
+
+export async function test(
+  req: FastifyRequest<{
+    Params: {
+      projectId: string;
+    };
+  }>,
+  reply: FastifyReply
+) {
+  const [event] = await getEvents(
+    `SELECT * FROM events WHERE project_id = '${req.params.projectId}' AND name = 'screen_view' LIMIT 1`
+  );
+  if (!event) {
+    return reply.status(404).send('No event found');
+  }
+  redisPub.publish('event', JSON.stringify(event));
+  redis.set(
+    `live:event:${event.projectId}:${Math.random() * 1000}`,
+    '',
+    'EX',
+    10
+  );
+  reply.status(202).send(event);
+}
+
+export function wsVisitors(
+  connection: {
+    socket: WebSocket;
+  },
+  req: FastifyRequest<{
+    Params: {
+      projectId: string;
+    };
+  }>
+) {
+  const { params } = req;
+
+  redisSub.subscribe('event');
+  redisSub.psubscribe('__key*:expired');
+
+  const message = (channel: string, message: string) => {
+    if (channel === 'event') {
+      const event = getSafeJson<IServiceCreateEventPayload>(message);
+      if (event?.projectId === params.projectId) {
+        getLiveVisitors(params.projectId).then((count) => {
+          connection.socket.send(String(count));
+        });
+      }
+    }
+  };
+  const pmessage = (pattern: string, channel: string, message: string) => {
+    const [projectId] = getLiveEventInfo(message);
+    if (projectId && projectId === params.projectId) {
+      getLiveVisitors(params.projectId).then((count) => {
+        connection.socket.send(String(count));
+      });
+    }
+  };
+
+  redisSub.on('message', message);
+  redisSub.on('pmessage', pmessage);
+
+  connection.socket.on('close', () => {
+    redisSub.unsubscribe('event');
+    redisSub.punsubscribe('__key*:expired');
+    redisSub.off('message', message);
+    redisSub.off('pmessage', pmessage);
+  });
+}
+
+export function wsEvents(
+  connection: {
+    socket: WebSocket;
+  },
+  req: FastifyRequest<{
+    Params: {
+      projectId: string;
+    };
+  }>
+) {
+  const { params } = req;
+
+  redisSub.subscribe('event');
+
+  const message = (channel: string, message: string) => {
+    const event = getSafeJson<IServiceCreateEventPayload>(message);
+    if (event?.projectId === params.projectId) {
+      connection.socket.send(JSON.stringify(event));
+    }
+  };
+
+  redisSub.on('message', message);
+
+  connection.socket.on('close', () => {
+    redisSub.unsubscribe('event');
+    redisSub.off('message', message);
+  });
+}

apps/api/src/controllers/misc.controller.ts

@@ -0,0 +1,127 @@
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import icoToPng from 'ico-to-png';
+import sharp from 'sharp';
+
+import { createHash } from '@mixan/common';
+import { redis } from '@mixan/redis';
+
+interface GetFaviconParams {
+  url: string;
+}
+
+async function getImageBuffer(url: string) {
+  try {
+    const res = await fetch(url);
+    const contentType = res.headers.get('content-type');
+
+    if (!contentType?.includes('image')) {
+      return null;
+    }
+
+    if (!res.ok) {
+      return null;
+    }
+
+    if (contentType === 'image/x-icon' || url.endsWith('.ico')) {
+      const arrayBuffer = await res.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      return await icoToPng(buffer, 30);
+    }
+
+    return await sharp(await res.arrayBuffer())
+      .resize(30, 30, {
+        fit: 'cover',
+      })
+      .png()
+      .toBuffer();
+  } catch (e) {
+    console.log('Failed to get image from url', url);
+    console.log(e);
+  }
+}
+
+const imageExtensions = ['svg', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'ico'];
+
+export async function getFavicon(
+  request: FastifyRequest<{
+    Querystring: GetFaviconParams;
+  }>,
+  reply: FastifyReply
+) {
+  function sendBuffer(buffer: Buffer, cacheKey?: string) {
+    if (cacheKey) {
+      redis.set(`favicon:${cacheKey}`, buffer.toString('base64'));
+    }
+    reply.type('image/png');
+    console.log('buffer', buffer.byteLength);
+
+    return reply.send(buffer);
+  }
+
+  if (!request.query.url) {
+    return reply.status(404).send('Not found');
+  }
+
+  const url = decodeURIComponent(request.query.url);
+
+  // DIRECT IMAGE
+  if (imageExtensions.find((ext) => url.endsWith(ext))) {
+    const cacheKey = createHash(url, 32);
+    const cache = await redis.get(`favicon:${cacheKey}`);
+    if (cache) {
+      return sendBuffer(Buffer.from(cache, 'base64'));
+    }
+    const buffer = await getImageBuffer(url);
+    if (buffer && buffer.byteLength > 0) {
+      return sendBuffer(buffer, cacheKey);
+    }
+  }
+
+  const { hostname, origin } = new URL(url);
+  const cache = await redis.get(`favicon:${hostname}`);
+  if (cache) {
+    return sendBuffer(Buffer.from(cache, 'base64'));
+  }
+
+  // TRY FAVICON.ICO
+  const buffer = await getImageBuffer(`${origin}/favicon.ico`);
+  if (buffer && buffer.byteLength > 0) {
+    return sendBuffer(buffer, hostname);
+  }
+
+  // PARSE HTML
+  const res = await fetch(url).then((res) => res.text());
+
+  function findFavicon(res: string) {
+    const match = res.match(
+      /(\<link(.+?)image\/x-icon(.+?)\>|\<link(.+?)shortcut\sicon(.+?)\>)/
+    );
+    if (!match) {
+      return null;
+    }
+
+    return match[0].match(/href="(.+?)"/)?.[1] ?? null;
+  }
+
+  const favicon = findFavicon(res);
+  if (favicon) {
+    const buffer = await getImageBuffer(favicon);
+
+    if (buffer && buffer.byteLength > 0) {
+      return sendBuffer(buffer, hostname);
+    }
+  }
+
+  return reply.status(404).send('Not found');
+}
+
+export async function clearFavicons(
+  request: FastifyRequest,
+  reply: FastifyReply
+) {
+  const keys = await redis.keys('favicon:*');
+  for (const key of keys) {
+    await redis.del(key);
+  }
+  return reply.status(404).send('OK');
+}

apps/api/src/controllers/profile.controller.ts

@@ -0,0 +1,110 @@
+import { getClientIp, parseIp } from '@/utils/parseIp';
+import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { assocPath, pathOr } from 'ramda';
+
+import { getProfileById, upsertProfile } from '@mixan/db';
+import type { IncrementProfilePayload, UpdateProfilePayload } from '@mixan/sdk';
+
+export async function updateProfile(
+  request: FastifyRequest<{
+    Body: UpdateProfilePayload;
+  }>,
+  reply: FastifyReply
+) {
+  const { profileId, properties, ...rest } = request.body;
+  const projectId = request.projectId;
+  const ip = getClientIp(request)!;
+  const ua = request.headers['user-agent']!;
+  const uaInfo = parseUserAgent(ua);
+  const geo = await parseIp(ip);
+
+  await upsertProfile({
+    id: profileId,
+    projectId,
+    properties: {
+      ...(properties ?? {}),
+      ...(ip ? geo : {}),
+      ...(isUserAgentSet(ua) ? uaInfo : {}),
+    },
+    ...rest,
+  });
+
+  reply.status(202).send(profileId);
+}
+
+export async function incrementProfileProperty(
+  request: FastifyRequest<{
+    Body: IncrementProfilePayload;
+  }>,
+  reply: FastifyReply
+) {
+  const { profileId, property, value } = request.body;
+  const projectId = request.projectId;
+
+  const profile = await getProfileById(profileId);
+  if (!profile) {
+    return reply.status(404).send('Not found');
+  }
+
+  const parsed = parseInt(
+    pathOr<string>('0', property.split('.'), profile.properties),
+    10
+  );
+
+  if (isNaN(parsed)) {
+    return reply.status(400).send('Not number');
+  }
+
+  profile.properties = assocPath(
+    property.split('.'),
+    parsed + value,
+    profile.properties
+  );
+
+  await upsertProfile({
+    id: profile.id,
+    projectId,
+    properties: profile.properties,
+  });
+
+  reply.status(202).send(profile.id);
+}
+
+export async function decrementProfileProperty(
+  request: FastifyRequest<{
+    Body: IncrementProfilePayload;
+  }>,
+  reply: FastifyReply
+) {
+  const { profileId, property, value } = request.body;
+  const projectId = request.projectId;
+
+  const profile = await getProfileById(profileId);
+  if (!profile) {
+    return reply.status(404).send('Not found');
+  }
+
+  const parsed = parseInt(
+    pathOr<string>('0', property.split('.'), profile.properties),
+    10
+  );
+
+  if (isNaN(parsed)) {
+    return reply.status(400).send('Not number');
+  }
+
+  profile.properties = assocPath(
+    property.split('.'),
+    parsed - value,
+    profile.properties
+  );
+
+  await upsertProfile({
+    id: profile.id,
+    projectId,
+    properties: profile.properties,
+  });
+
+  reply.status(202).send(profile.id);
+}