feat: dashboard v2, esm, upgrades (#211)

* esm * wip * wip * wip * wip * wip * wip * subscription notice * wip * wip * wip * fix envs * fix: update docker build * fix * esm/types * delete dashboard :D * add patches to dockerfiles * update packages + catalogs + ts * wip * remove native libs * ts * improvements * fix redirects and fetching session * try fix favicon * fixes * fix * order and resize reportds within a dashboard * improvements * wip * added userjot to dashboard * fix * add op * wip * different cache key * improve date picker * fix table * event details loading * redo onboarding completely * fix login * fix * fix * extend session, billing and improve bars * fix * reduce price on 10M
2025-10-16 12:27:44 +02:00
parent 436e81ecc9
commit 81a7e5d62e
741 changed files with 32695 additions and 16996 deletions
--- a/apps/api/src/controllers/ai.controller.ts
+++ b/apps/api/src/controllers/ai.controller.ts
@@ -112,7 +112,7 @@ export async function chat(

        await db.chat.create({
          data: {
-            messages: messagesToSave.slice(-10),
+            messages: messagesToSave.slice(-10) as any,
            projectId,
          },
        });
--- a/apps/api/src/controllers/misc.controller.ts
+++ b/apps/api/src/controllers/misc.controller.ts
@@ -1,53 +1,231 @@
+import crypto from 'node:crypto';
 import { logger } from '@/utils/logger';
 import { parseUrlMeta } from '@/utils/parseUrlMeta';
 import type { FastifyReply, FastifyRequest } from 'fastify';
-import icoToPng from 'ico-to-png';
 import sharp from 'sharp';

 import { getClientIp } from '@/utils/get-client-ip';
-import { createHash } from '@openpanel/common/server';
 import { TABLE_NAMES, ch, chQuery, formatClickhouseDate } from '@openpanel/db';
 import { getGeoLocation } from '@openpanel/geo';
-import { cacheable, getCache, getRedisCache } from '@openpanel/redis';
+import { getCache, getRedisCache } from '@openpanel/redis';

 interface GetFaviconParams {
  url: string;
 }

-async function getImageBuffer(url: string) {
+// Configuration
+const TTL_SECONDS = 60 * 60 * 24; // 24h
+const MAX_BYTES = 1_000_000; // 1MB cap
+const USER_AGENT = 'OpenPanel-FaviconProxy/1.0 (+https://openpanel.dev)';
+
+// Helper functions
+function createCacheKey(url: string, prefix = 'favicon'): string {
+  const hash = crypto.createHash('sha256').update(url).digest('hex');
+  return `${prefix}:v2:${hash}`;
+}
+
+function validateUrl(raw?: string): URL | null {
  try {
-    const res = await fetch(url);
-    const contentType = res.headers.get('content-type');
+    if (!raw) throw new Error('Missing ?url');
+    const url = new URL(raw);
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+      throw new Error('Only http/https URLs are allowed');
+    }
+    return url;
+  } catch (error) {
+    return null;
+  }
+}

-    if (!contentType?.includes('image')) {
-      return null;
+// Binary cache functions (more efficient than base64)
+async function getFromCacheBinary(
+  key: string,
+): Promise<{ buffer: Buffer; contentType: string } | null> {
+  const redis = getRedisCache();
+  const [bufferBase64, contentType] = await Promise.all([
+    redis.get(key),
+    redis.get(`${key}:ctype`),
+  ]);
+
+  if (!bufferBase64 || !contentType) return null;
+  return { buffer: Buffer.from(bufferBase64, 'base64'), contentType };
+}
+
+async function setToCacheBinary(
+  key: string,
+  buffer: Buffer,
+  contentType: string,
+): Promise<void> {
+  const redis = getRedisCache();
+  await Promise.all([
+    redis.set(key, buffer.toString('base64'), 'EX', TTL_SECONDS),
+    redis.set(`${key}:ctype`, contentType, 'EX', TTL_SECONDS),
+  ]);
+}
+
+// Fetch image with timeout and size limits
+async function fetchImage(
+  url: URL,
+): Promise<{ buffer: Buffer; contentType: string; status: number }> {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 10000); // 10s timeout
+
+  try {
+    const response = await fetch(url.toString(), {
+      redirect: 'follow',
+      signal: controller.signal,
+      headers: {
+        'user-agent': USER_AGENT,
+        accept: 'image/*,*/*;q=0.8',
+      },
+    });
+
+    clearTimeout(timeout);
+
+    if (!response.ok) {
+      return {
+        buffer: Buffer.alloc(0),
+        contentType: 'text/plain',
+        status: response.status,
+      };
    }

-    if (!res.ok) {
-      return null;
+    // Size guard
+    const contentLength = Number(response.headers.get('content-length') ?? '0');
+    if (contentLength > MAX_BYTES) {
+      throw new Error(`Remote file too large: ${contentLength} bytes`);
    }

-    if (contentType === 'image/x-icon' || url.endsWith('.ico')) {
-      const arrayBuffer = await res.arrayBuffer();
-      const buffer = Buffer.from(arrayBuffer);
-      return await icoToPng(buffer, 30);
+    const arrayBuffer = await response.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+
+    // Additional size check for actual content
+    if (buffer.length > MAX_BYTES) {
+      throw new Error('Remote file exceeded size limit');
    }

-    return await sharp(await res.arrayBuffer())
+    const contentType =
+      response.headers.get('content-type') || 'application/octet-stream';
+    return { buffer, contentType, status: 200 };
+  } catch (error) {
+    clearTimeout(timeout);
+    return { buffer: Buffer.alloc(0), contentType: 'text/plain', status: 500 };
+  }
+}
+
+// Check if URL is an ICO file
+function isIcoFile(url: string, contentType?: string): boolean {
+  return url.toLowerCase().endsWith('.ico') || contentType === 'image/x-icon';
+}
+function isSvgFile(url: string, contentType?: string): boolean {
+  return url.toLowerCase().endsWith('.svg') || contentType === 'image/svg+xml';
+}
+
+// Process image with Sharp (resize to 30x30 PNG)
+async function processImage(
+  buffer: Buffer,
+  originalUrl?: string,
+  contentType?: string,
+): Promise<Buffer> {
+  // If it's an ICO file, just return it as-is (no conversion needed)
+  if (originalUrl && isIcoFile(originalUrl, contentType)) {
+    logger.info('Serving ICO file directly', {
+      originalUrl,
+      bufferSize: buffer.length,
+    });
+    return buffer;
+  }
+
+  if (originalUrl && isSvgFile(originalUrl, contentType)) {
+    logger.info('Serving SVG file directly', {
+      originalUrl,
+      bufferSize: buffer.length,
+    });
+    return buffer;
+  }
+
+  // If buffer isnt to big just return it as well
+  if (buffer.length < 5000) {
+    logger.info('Serving image directly without processing', {
+      originalUrl,
+      bufferSize: buffer.length,
+    });
+    return buffer;
+  }
+
+  try {
+    // For other formats, process with Sharp
+    return await sharp(buffer)
      .resize(30, 30, {
        fit: 'cover',
      })
      .png()
      .toBuffer();
  } catch (error) {
-    logger.error('Failed to get image from url', {
-      error,
-      url,
+    logger.warn('Sharp failed to process image, trying fallback', {
+      error: error instanceof Error ? error.message : 'Unknown error',
+      originalUrl,
+      bufferSize: buffer.length,
    });
+
+    // If Sharp fails, try to create a simple fallback image
+    return createFallbackImage();
  }
 }

-const imageExtensions = ['svg', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'ico'];
+// Create a simple transparent fallback image when Sharp can't process the original
+function createFallbackImage(): Buffer {
+  // 1x1 transparent PNG
+  return Buffer.from(
+    'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=',
+    'base64',
+  );
+}
+
+// Process OG image with Sharp (resize to 300px width)
+async function processOgImage(
+  buffer: Buffer,
+  originalUrl?: string,
+  contentType?: string,
+): Promise<Buffer> {
+  // If buffer is small enough, return it as-is
+  if (buffer.length < 10000) {
+    logger.info('Serving OG image directly without processing', {
+      originalUrl,
+      bufferSize: buffer.length,
+    });
+    return buffer;
+  }
+
+  try {
+    // For OG images, process with Sharp to 300px width, maintaining aspect ratio
+    return await sharp(buffer)
+      .resize(300, null, {
+        fit: 'inside',
+        withoutEnlargement: true,
+      })
+      .png()
+      .toBuffer();
+  } catch (error) {
+    logger.warn('Sharp failed to process OG image, trying fallback', {
+      error: error instanceof Error ? error.message : 'Unknown error',
+      originalUrl,
+      bufferSize: buffer.length,
+    });
+
+    // If Sharp fails, try to create a simple fallback image
+    return createFallbackImage();
+  }
+}
+
+// Check if URL is a direct image
+function isDirectImage(url: URL): boolean {
+  const imageExtensions = ['svg', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'ico'];
+  return (
+    imageExtensions.some((ext) => url.pathname.endsWith(`.${ext}`)) ||
+    url.toString().includes('googleusercontent.com')
+  );
+}

 export async function getFavicon(
  request: FastifyRequest<{
@@ -55,68 +233,110 @@ export async function getFavicon(
  }>,
  reply: FastifyReply,
 ) {
-  function sendBuffer(buffer: Buffer, cacheKey?: string) {
-    if (cacheKey) {
-      getRedisCache().set(`favicon:${cacheKey}`, buffer.toString('base64'));
+  try {
+    const url = validateUrl(request.query.url);
+    if (!url) {
+      return createFallbackImage();
    }
-    reply.header('Cache-Control', 'public, max-age=604800');
-    reply.header('Expires', new Date(Date.now() + 604800000).toUTCString());
-    reply.type('image/png');
-    return reply.send(buffer);
-  }

-  if (!request.query.url) {
-    return reply.status(404).send('Not found');
-  }
+    const cacheKey = createCacheKey(url.toString());

-  const url = decodeURIComponent(request.query.url);
-
-  if (imageExtensions.find((ext) => url.endsWith(ext))) {
-    const cacheKey = createHash(url, 32);
-    const cache = await getRedisCache().get(`favicon:${cacheKey}`);
-    if (cache) {
-      return sendBuffer(Buffer.from(cache, 'base64'));
+    // Check cache first
+    const cached = await getFromCacheBinary(cacheKey);
+    if (cached) {
+      reply.header('Content-Type', cached.contentType);
+      reply.header('Cache-Control', 'public, max-age=604800, immutable');
+      return reply.send(cached.buffer);
    }
-    const buffer = await getImageBuffer(url);
-    if (buffer && buffer.byteLength > 0) {
-      return sendBuffer(buffer, cacheKey);
+
+    let imageUrl: URL;
+
+    // If it's a direct image URL, use it directly
+    if (isDirectImage(url)) {
+      imageUrl = url;
+    } else {
+      // For website URLs, extract favicon from HTML
+      const meta = await parseUrlMeta(url.toString());
+      if (meta?.favicon) {
+        imageUrl = new URL(meta.favicon);
+      } else {
+        // Fallback to Google's favicon service
+        const { hostname } = url;
+        imageUrl = new URL(
+          `https://www.google.com/s2/favicons?domain=${hostname}&sz=256`,
+        );
+      }
    }
-  }

-  const { hostname } = new URL(url);
-  const cache = await getRedisCache().get(`favicon:${hostname}`);
+    // Fetch the image
+    const { buffer, contentType, status } = await fetchImage(imageUrl);

-  if (cache) {
-    return sendBuffer(Buffer.from(cache, 'base64'));
-  }
-
-  const meta = await parseUrlMeta(url);
-  if (meta?.favicon) {
-    const buffer = await getImageBuffer(meta.favicon);
-    if (buffer && buffer.byteLength > 0) {
-      return sendBuffer(buffer, hostname);
+    if (status !== 200 || buffer.length === 0) {
+      return reply.send(createFallbackImage());
    }
-  }

-  const buffer = await getImageBuffer(
-    'https://www.iconsdb.com/icons/download/orange/warning-128.png',
-  );
-  if (buffer && buffer.byteLength > 0) {
-    return sendBuffer(buffer, hostname);
-  }
+    // Process the image (resize to 30x30 PNG, or serve ICO as-is)
+    const processedBuffer = await processImage(
+      buffer,
+      imageUrl.toString(),
+      contentType,
+    );

-  return reply.status(404).send('Not found');
+    // Determine the correct content type for caching and response
+    const isIco = isIcoFile(imageUrl.toString(), contentType);
+    const responseContentType = isIco ? 'image/x-icon' : contentType;
+
+    // Cache the result with correct content type
+    await setToCacheBinary(cacheKey, processedBuffer, responseContentType);
+
+    reply.header('Content-Type', responseContentType);
+    reply.header('Cache-Control', 'public, max-age=3600, immutable');
+    return reply.send(processedBuffer);
+  } catch (error: any) {
+    logger.error('Favicon fetch error', {
+      error: error.message,
+      url: request.query.url,
+    });
+
+    const message =
+      process.env.NODE_ENV === 'production'
+        ? 'Bad request'
+        : (error?.message ?? 'Error');
+    reply.header('Cache-Control', 'no-store');
+    return reply.status(400).send(message);
+  }
 }

 export async function clearFavicons(
  request: FastifyRequest,
  reply: FastifyReply,
 ) {
-  const keys = await getRedisCache().keys('favicon:*');
+  const redis = getRedisCache();
+  const keys = await redis.keys('favicon:*');
+
+  // Delete both the binary data and content-type keys
  for (const key of keys) {
-    await getRedisCache().del(key);
+    await redis.del(key);
+    await redis.del(`${key}:ctype`);
  }
-  return reply.status(404).send('OK');
+
+  return reply.status(200).send('OK');
+}
+
+export async function clearOgImages(
+  request: FastifyRequest,
+  reply: FastifyReply,
+) {
+  const redis = getRedisCache();
+  const keys = await redis.keys('og:*');
+
+  // Delete both the binary data and content-type keys
+  for (const key of keys) {
+    await redis.del(key);
+    await redis.del(`${key}:ctype`);
+  }
+
+  return reply.status(200).send('OK');
 }

 export async function ping(
@@ -181,3 +401,77 @@ export async function getGeo(request: FastifyRequest, reply: FastifyReply) {
  const geo = await getGeoLocation(ip);
  return reply.status(200).send(geo);
 }
+
+export async function getOgImage(
+  request: FastifyRequest<{
+    Querystring: {
+      url: string;
+    };
+  }>,
+  reply: FastifyReply,
+) {
+  try {
+    const url = validateUrl(request.query.url);
+    if (!url) {
+      return getFavicon(request, reply);
+    }
+    const cacheKey = createCacheKey(url.toString(), 'og');
+
+    // Check cache first
+    const cached = await getFromCacheBinary(cacheKey);
+    if (cached) {
+      reply.header('Content-Type', cached.contentType);
+      reply.header('Cache-Control', 'public, max-age=604800, immutable');
+      return reply.send(cached.buffer);
+    }
+
+    let imageUrl: URL;
+
+    // If it's a direct image URL, use it directly
+    if (isDirectImage(url)) {
+      imageUrl = url;
+    } else {
+      // For website URLs, extract OG image from HTML
+      const meta = await parseUrlMeta(url.toString());
+      if (meta?.ogImage) {
+        imageUrl = new URL(meta.ogImage);
+      } else {
+        // No OG image found, return a fallback
+        return getFavicon(request, reply);
+      }
+    }
+
+    // Fetch the image
+    const { buffer, contentType, status } = await fetchImage(imageUrl);
+
+    if (status !== 200 || buffer.length === 0) {
+      return getFavicon(request, reply);
+    }
+
+    // Process the image (resize to 1200x630 for OG standards, or serve as-is if reasonable size)
+    const processedBuffer = await processOgImage(
+      buffer,
+      imageUrl.toString(),
+      contentType,
+    );
+
+    // Cache the result
+    await setToCacheBinary(cacheKey, processedBuffer, 'image/png');
+
+    reply.header('Content-Type', 'image/png');
+    reply.header('Cache-Control', 'public, max-age=3600, immutable');
+    return reply.send(processedBuffer);
+  } catch (error: any) {
+    logger.error('OG image fetch error', {
+      error: error.message,
+      url: request.query.url,
+    });
+
+    const message =
+      process.env.NODE_ENV === 'production'
+        ? 'Bad request'
+        : (error?.message ?? 'Error');
+    reply.header('Cache-Control', 'no-store');
+    return reply.status(400).send(message);
+  }
+}
--- a/apps/api/src/controllers/oauth-callback.controller.tsx
+++ b/apps/api/src/controllers/oauth-callback.controller.tsx
@@ -76,7 +76,9 @@ async function handleExistingUser({
    sessionToken,
    session.expiresAt,
  );
-  return reply.redirect(process.env.NEXT_PUBLIC_DASHBOARD_URL!);
+  return reply.redirect(
+    process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!,
+  );
 }

 async function handleNewUser({
@@ -138,7 +140,9 @@ async function handleNewUser({
    sessionToken,
    session.expiresAt,
  );
-  return reply.redirect(process.env.NEXT_PUBLIC_DASHBOARD_URL!);
+  return reply.redirect(
+    process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!,
+  );
 }

 // Provider-specific user fetching
@@ -348,7 +352,9 @@ export async function googleCallback(req: FastifyRequest, reply: FastifyReply) {
 }

 function redirectWithError(reply: FastifyReply, error: LogError | unknown) {
-  const url = new URL(process.env.NEXT_PUBLIC_DASHBOARD_URL!);
+  const url = new URL(
+    process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!,
+  );
  url.pathname = '/login';
  if (error instanceof LogError) {
    url.searchParams.set('error', error.message);
--- a/apps/api/src/controllers/webhook.controller.ts
+++ b/apps/api/src/controllers/webhook.controller.ts
@@ -1,5 +1,10 @@
 import fs from 'node:fs';
 import path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 import { db, getOrganizationByProjectIdCached } from '@openpanel/db';
 import {
  sendSlackNotification,
@@ -100,7 +105,7 @@ export async function slackWebhook(
    });

    return reply.redirect(
-      `${process.env.NEXT_PUBLIC_DASHBOARD_URL}/${organizationId}/${projectId}/settings/integrations?tab=installed`,
+      `${process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL}/${organizationId}/${projectId}/settings/integrations?tab=installed`,
    );
  } catch (err) {
    request.log.error(err);
@@ -184,7 +189,7 @@ export async function polarWebhook(
          data: {
            subscriptionId: event.data.id,
            subscriptionCustomerId: event.data.customer.id,
-            subscriptionPriceId: event.data.priceId,
+            subscriptionPriceId: event.data.prices[0]?.id ?? null,
            subscriptionProductId: event.data.productId,
            subscriptionStatus: event.data.status,
            subscriptionStartsAt: event.data.currentPeriodStart,
--- a/apps/api/src/hooks/request-logging.hook.ts
+++ b/apps/api/src/hooks/request-logging.hook.ts
@@ -7,7 +7,7 @@ const ignoreMethods = ['OPTIONS'];
 const getTrpcInput = (
  request: FastifyRequest,
 ): Record<string, unknown> | undefined => {
-  const input = path(['query', 'input'], request);
+  const input = path<any>(['query', 'input'], request);
  try {
    return typeof input === 'string' ? JSON.parse(input).json : input;
  } catch (e) {
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -95,15 +95,13 @@ const startServer = async () => {
        if (isPrivatePath) {
          // Allow multiple dashboard domains
          const allowedOrigins = [
-            process.env.NEXT_PUBLIC_DASHBOARD_URL,
+            process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL,
            ...(process.env.API_CORS_ORIGINS?.split(',') ?? []),
          ].filter(Boolean);

          const origin = req.headers.origin;
          const isAllowed = origin && allowedOrigins.includes(origin);

-          logger.info('Allowed origins', { allowedOrigins, origin, isAllowed });
-
          return callback(null, {
            origin: isAllowed ? origin : false,
            credentials: true,
@@ -160,6 +158,12 @@ const startServer = async () => {
          router: appRouter,
          createContext: createContext,
          onError(ctx) {
+            if (
+              ctx.error.code === 'UNAUTHORIZED' &&
+              ctx.path === 'organization.list'
+            ) {
+              return;
+            }
            ctx.req.log.error('trpc error', {
              error: ctx.error,
              path: ctx.path,
@@ -191,7 +195,10 @@ const startServer = async () => {
      instance.get('/healthz/live', liveness);
      instance.get('/healthz/ready', readiness);
      instance.get('/', (_request, reply) =>
-        reply.send({ name: 'openpanel sdk api' }),
+        reply.send({
+          status: 'ok',
+          message: 'Successfully running OpenPanel.dev API',
+        }),
      );
    });

--- a/apps/api/src/routes/misc.router.ts
+++ b/apps/api/src/routes/misc.router.ts
@@ -20,6 +20,18 @@ const miscRouter: FastifyPluginCallback = async (fastify) => {
    handler: controller.getFavicon,
  });

+  fastify.route({
+    method: 'GET',
+    url: '/og',
+    handler: controller.getOgImage,
+  });
+
+  fastify.route({
+    method: 'GET',
+    url: '/og/clear',
+    handler: controller.clearOgImages,
+  });
+
  fastify.route({
    method: 'GET',
    url: '/favicon/clear',
--- a/apps/api/src/utils/parseUrlMeta.ts
+++ b/apps/api/src/utils/parseUrlMeta.ts
@@ -5,12 +5,16 @@ function fallbackFavicon(url: string) {
 }

 function findBestFavicon(favicons: UrlMetaData['favicons']) {
-  const match = favicons.find(
-    (favicon) =>
-      favicon.rel === 'shortcut icon' ||
-      favicon.rel === 'icon' ||
-      favicon.rel === 'apple-touch-icon',
-  );
+  const match = favicons
+    .sort((a, b) => {
+      return a.rel.length - b.rel.length;
+    })
+    .find(
+      (favicon) =>
+        favicon.rel === 'shortcut icon' ||
+        favicon.rel === 'icon' ||
+        favicon.rel === 'apple-touch-icon',
+    );

  if (match) {
    return match.href;
@@ -18,11 +22,32 @@ function findBestFavicon(favicons: UrlMetaData['favicons']) {
  return null;
 }

+function findBestOgImage(data: UrlMetaData): string | null {
+  // Priority order for OG images
+  const candidates = [
+    data['og:image:secure_url'],
+    data['og:image:url'],
+    data['og:image'],
+    data['twitter:image:src'],
+    data['twitter:image'],
+  ];
+
+  for (const candidate of candidates) {
+    if (candidate?.trim()) {
+      return candidate.trim();
+    }
+  }
+
+  return null;
+}
+
 function transform(data: UrlMetaData, url: string) {
  const favicon = findBestFavicon(data.favicons);
+  const ogImage = findBestOgImage(data);

  return {
    favicon: favicon ? new URL(favicon, url).toString() : fallbackFavicon(url),
+    ogImage: ogImage ? new URL(ogImage, url).toString() : null,
  };
 }

@@ -32,6 +57,11 @@ interface UrlMetaData {
    href: string;
    sizes: string;
  }[];
+  'og:image'?: string;
+  'og:image:url'?: string;
+  'og:image:secure_url'?: string;
+  'twitter:image'?: string;
+  'twitter:image:src'?: string;
 }

 export async function parseUrlMeta(url: string) {
@@ -42,6 +72,7 @@ export async function parseUrlMeta(url: string) {
  } catch (err) {
    return {
      favicon: fallbackFavicon(url),
+      ogImage: null,
    };
  }
 }