improve favicons

2024-02-14 23:12:10 +01:00
parent 176ddc410b
commit a1d5104166
6 changed files with 232 additions and 94 deletions
--- a/apps/sdk-api/package.json
+++ b/apps/sdk-api/package.json
@@ -17,8 +17,10 @@
    "@mixan/queue": "workspace:*",
    "@mixan/redis": "workspace:*",
    "fastify": "^4.25.2",
+    "ico-to-png": "^0.2.1",
    "pino": "^8.17.2",
    "ramda": "^0.29.1",
+    "sharp": "^0.33.2",
    "ua-parser-js": "^1.0.37"
  },
  "devDependencies": {
--- a/apps/sdk-api/src/controllers/misc.controller.ts
+++ b/apps/sdk-api/src/controllers/misc.controller.ts
@@ -1,83 +1,129 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
+import icoToPng from 'ico-to-png';
+import sharp from 'sharp';

+import { createHash } from '@mixan/common';
 import { redis } from '@mixan/redis';

 interface GetFaviconParams {
  url: string;
 }

-function toBuffer(arrayBuffer: ArrayBuffer) {
-  const buffer = Buffer.alloc(arrayBuffer.byteLength);
-  const view = new Uint8Array(arrayBuffer);
-  for (let i = 0; i < buffer.length; ++i) {
-    buffer[i] = view[i]!;
-  }
-  return buffer;
-}
+async function getImageBuffer(url: string) {
+  try {
+    const res = await fetch(url);
+    const contentType = res.headers.get('content-type');

-async function getUrlBuffer(url: string) {
-  const arrayBuffer = await fetch(url).then((res) => {
-    if (res.ok) {
-      return res.arrayBuffer();
+    if (!contentType?.includes('image')) {
+      return null;
    }
-  });

-  if (arrayBuffer) {
-    return toBuffer(arrayBuffer);
+    if (!res.ok) {
+      return null;
+    }
+
+    if (contentType === 'image/x-icon' || url.endsWith('.ico')) {
+      const arrayBuffer = await res.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      return await icoToPng(buffer, 30);
+    }
+
+    return await sharp(await res.arrayBuffer())
+      .resize(30, 30, {
+        fit: 'cover',
+      })
+      .png()
+      .toBuffer();
+  } catch (e) {
+    console.log('Failed to get image from url', url);
+    console.log(e);
  }
-
-  return null;
 }

+const imageExtensions = ['svg', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'ico'];
+
 export async function getFavicon(
  request: FastifyRequest<{
    Querystring: GetFaviconParams;
  }>,
  reply: FastifyReply
 ) {
+  function sendBuffer(buffer: Buffer, cacheKey?: string) {
+    if (cacheKey) {
+      redis.set(`favicon:${cacheKey}`, buffer.toString('base64'));
+    }
+    reply.type('image/png');
+    console.log('buffer', buffer.byteLength);
+
+    return reply.send(buffer);
+  }
+
  if (!request.query.url) {
    return reply.status(404).send('Not found');
  }

-  function sendBuffer(buffer: Buffer, hostname?: string) {
-    if (hostname) {
-      redis.set(`favicon:${hostname}`, buffer.toString('base64'));
+  const url = decodeURIComponent(request.query.url);
+
+  // DIRECT IMAGE
+  if (imageExtensions.find((ext) => url.endsWith(ext))) {
+    const cacheKey = createHash(url, 32);
+    const cache = await redis.get(`favicon:${cacheKey}`);
+    if (cache) {
+      return sendBuffer(Buffer.from(cache, 'base64'));
+    }
+    const buffer = await getImageBuffer(url);
+    if (buffer && buffer.byteLength > 0) {
+      return sendBuffer(buffer, cacheKey);
    }
-    reply.type('image/png');
-    return reply.send(buffer);
  }

-  const url = decodeURIComponent(request.query.url);
  const { hostname, origin } = new URL(url);
-
  const cache = await redis.get(`favicon:${hostname}`);
  if (cache) {
    return sendBuffer(Buffer.from(cache, 'base64'));
  }

-  // Try just get the favicon.ico
-  const buffer = await getUrlBuffer(`${origin}/favicon.ico`);
-  if (buffer) {
+  // TRY FAVICON.ICO
+  const buffer = await getImageBuffer(`${origin}/favicon.ico`);
+  console.log('buffer', buffer?.length);
+
+  if (buffer && buffer.byteLength > 0) {
    return sendBuffer(buffer, hostname);
  }

-  // If that didnt work try parse html
+  // PARSE HTML
  const res = await fetch(url).then((res) => res.text());
-  const favicon =
-    res.match(/<link.*?rel="icon".*?href="(.+?)".*?>/) ||
-    res.match(/<link.*?rel="shortcut icon".*?href="(.+?)".*?>/);

-  if (favicon?.[1]) {
-    const faviconUrl = favicon[1].startsWith('http')
-      ? favicon[1]
-      : `${origin}${favicon[1]}`;
+  function findFavicon(res: string) {
+    const match = res.match(
+      /(\<link(.+?)image\/x-icon(.+?)\>|\<link(.+?)shortcut\sicon(.+?)\>)/
+    );
+    if (!match) {
+      return null;
+    }

-    const buffer = await getUrlBuffer(faviconUrl);
+    return match[0].match(/href="(.+?)"/)?.[1] ?? null;
+  }

-    if (buffer) {
+  const favicon = findFavicon(res);
+  if (favicon) {
+    const buffer = await getImageBuffer(favicon);
+
+    if (buffer && buffer.byteLength > 0) {
      return sendBuffer(buffer, hostname);
    }
  }

  return reply.status(404).send('Not found');
 }
+
+export async function clearFavicons(
+  request: FastifyRequest,
+  reply: FastifyReply
+) {
+  const keys = await redis.keys('favicon:*');
+  for (const key of keys) {
+    await redis.del(key);
+  }
+  return reply.status(404).send('OK');
+}
--- a/apps/sdk-api/src/routes/misc.router.ts
+++ b/apps/sdk-api/src/routes/misc.router.ts
@@ -8,6 +8,12 @@ const miscRouter: FastifyPluginCallback = (fastify, opts, done) => {
    handler: controller.getFavicon,
  });

+  fastify.route({
+    method: 'GET',
+    url: '/favicon/clear',
+    handler: controller.clearFavicons,
+  });
+
  done();
 };

--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -66,7 +66,6 @@
    "react-in-viewport": "1.0.0-alpha.30",
    "react-redux": "^8.1.3",
    "react-responsive": "^9.0.2",
-    "react-social-icons": "^6.12.0",
    "react-svg-worldmap": "2.0.0-alpha.16",
    "react-syntax-highlighter": "^15.5.0",
    "react-use-websocket": "^4.7.0",
--- a/apps/web/src/components/report/chart/SerieIcon.tsx
+++ b/apps/web/src/components/report/chart/SerieIcon.tsx
@@ -1,64 +1,108 @@
+import { useMemo } from 'react';
 import { NOT_SET_VALUE } from '@/utils/constants';
 import type { LucideIcon, LucideProps } from 'lucide-react';
 import {
  ActivityIcon,
  ExternalLinkIcon,
  HelpCircleIcon,
+  MailIcon,
  MonitorIcon,
  MonitorPlayIcon,
-  PhoneIcon,
+  PodcastIcon,
+  ScanIcon,
+  SearchIcon,
  SmartphoneIcon,
-  SquareAsteriskIcon,
  TabletIcon,
-  TabletSmartphoneIcon,
-  TwitterIcon,
 } from 'lucide-react';
-import {
-  getKeys,
-  getNetworks,
-  networkFor,
-  register,
-  SocialIcon,
-} from 'react-social-icons';

 interface SerieIconProps extends LucideProps {
  name: string;
 }

+function getProxyImage(url: string) {
+  return `${String(process.env.NEXT_PUBLIC_API_URL)}/misc/favicon?url=${encodeURIComponent(url)}`;
+}
+
+const createImageIcon = (url: string) => {
+  return function (props: LucideProps) {
+    return <img className="w-4 h-4 object-cover rounded" src={url} />;
+  } as LucideIcon;
+};
+
 const mapper: Record<string, LucideIcon> = {
+  // Events
  screen_view: MonitorPlayIcon,
  session_start: ActivityIcon,
+  session_end: ActivityIcon,
  link_out: ExternalLinkIcon,
+
+  // Websites
+  google: createImageIcon(getProxyImage('https://google.com')),
+  facebook: createImageIcon(getProxyImage('https://facebook.com')),
+  bing: createImageIcon(getProxyImage('https://bing.com')),
+  twitter: createImageIcon(getProxyImage('https://x.com')),
+  duckduckgo: createImageIcon(getProxyImage('https://duckduckgo.com')),
+  'yahoo!': createImageIcon(getProxyImage('https://yahoo.com')),
+  instagram: createImageIcon(getProxyImage('https://instagram.com')),
+  gmail: createImageIcon(getProxyImage('https://mail.google.com/')),
+
+  'mobile safari': createImageIcon(
+    getProxyImage(
+      'https://upload.wikimedia.org/wikipedia/commons/5/52/Safari_browser_logo.svg'
+    )
+  ),
+  chrome: createImageIcon(
+    getProxyImage(
+      'https://upload.wikimedia.org/wikipedia/commons/e/e1/Google_Chrome_icon_%28February_2022%29.svg'
+    )
+  ),
+  'samsung internet': createImageIcon(
+    getProxyImage(
+      'https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Samsung_Internet_logo.svg/1024px-Samsung_Internet_logo.svg.png'
+    )
+  ),
+  safari: createImageIcon(
+    getProxyImage(
+      'https://upload.wikimedia.org/wikipedia/commons/5/52/Safari_browser_logo.svg'
+    )
+  ),
+  edge: createImageIcon(
+    getProxyImage(
+      'https://upload.wikimedia.org/wikipedia/commons/7/7e/Microsoft_Edge_logo_%282019%29.png'
+    )
+  ),
+  firefox: createImageIcon(
+    getProxyImage(
+      'https://upload.wikimedia.org/wikipedia/commons/thumb/a/a0/Firefox_logo%2C_2019.svg/1920px-Firefox_logo%2C_2019.svg.png'
+    )
+  ),
+  snapchat: createImageIcon(getProxyImage('https://snapchat.com')),
+
+  // Misc
  mobile: SmartphoneIcon,
  desktop: MonitorIcon,
  tablet: TabletIcon,
-  [NOT_SET_VALUE]: HelpCircleIcon,
+  search: SearchIcon,
+  social: PodcastIcon,
+  email: MailIcon,
+  unknown: HelpCircleIcon,
+  [NOT_SET_VALUE]: ScanIcon,
 };

-const networks = getNetworks();
-
-register('duckduckgo', {
-  color: 'red',
-  path: 'https://duckduckgo.com/favicon.ico',
-});
-
 export function SerieIcon({ name, ...props }: SerieIconProps) {
-  let Icon = mapper[name] ?? null;
+  const Icon = useMemo(() => {
+    const mapped = mapper[name.toLowerCase()] ?? null;

-  if (name.includes('http')) {
-    Icon = ((_props) => (
-      <img
-        className="w-4 h-4 object-cover"
-        src={`${String(process.env.NEXT_PUBLIC_API_URL)}/misc/favicon?url=${encodeURIComponent(name)}`}
-      />
-    )) as LucideIcon;
-  }
+    if (mapped) {
+      return mapped;
+    }

-  if (Icon === null && networks.includes(name.toLowerCase())) {
-    Icon = ((_props) => (
-      <SocialIcon network={name.toLowerCase()} />
-    )) as LucideIcon;
-  }
+    if (name.includes('http')) {
+      return createImageIcon(getProxyImage(name));
+    }
+
+    return null;
+  }, [name]);

  return Icon ? (
    <div className="w-4 h-4 flex-shrink-0 relative [&_a]:!w-4 [&_a]:!h-4 [&_svg]:!rounded">
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -210,12 +210,18 @@ importers:
      fastify:
        specifier: ^4.25.2
        version: 4.25.2
+      ico-to-png:
+        specifier: ^0.2.1
+        version: 0.2.1
      pino:
        specifier: ^8.17.2
        version: 8.17.2
      ramda:
        specifier: ^0.29.1
        version: 0.29.1
+      sharp:
+        specifier: ^0.33.2
+        version: 0.33.2
      ua-parser-js:
        specifier: ^1.0.37
        version: 1.0.37
@@ -482,9 +488,6 @@ importers:
      react-responsive:
        specifier: ^9.0.2
        version: 9.0.2(react@18.2.0)
-      react-social-icons:
-        specifier: ^6.12.0
-        version: 6.12.0(react-dom@18.2.0)(react@18.2.0)
      react-svg-worldmap:
        specifier: 2.0.0-alpha.16
        version: 2.0.0-alpha.16(react-dom@18.2.0)(react@18.2.0)
@@ -3342,6 +3345,10 @@ packages:
      '@bull-board/api': 5.13.0(@bull-board/ui@5.13.0)
    dev: false

+  /@canvas/image-data@1.0.0:
+    resolution: {integrity: sha512-BxOqI5LgsIQP1odU5KMwV9yoijleOPzHL18/YvNqF9KFSGF2K/DLlYAbDQsWqd/1nbaFuSkYD/191dpMtNh4vw==}
+    dev: false
+
  /@clerk/backend@0.38.1(react@18.2.0):
    resolution: {integrity: sha512-Nnr+j2V0RwFp/CFjlp7VenGPACilhAVD2j1c49fxjQUuAWeLd/z/5efb9mp7kgZup8oxpOHoMDjO2ndWY4rPqA==}
    engines: {node: '>=14'}
@@ -3846,7 +3853,7 @@ packages:
      getenv: 1.0.0
      glob: 7.1.6
      resolve-from: 5.0.0
-      semver: 7.5.4
+      semver: 7.6.0
      slash: 3.0.0
      xcode: 3.0.1
      xml2js: 0.6.0
@@ -4899,7 +4906,7 @@ packages:
    resolution: {integrity: sha512-coUfuoMeIB7B8/NMekxaDzLhaYmp0HZNPEjYRm9goRou8UZIC3z21s0sL9AWoCw4EG876QyO3kYrc61WNF9B/w==}
    engines: {node: '>=8.0.0'}
    dependencies:
-      tslib: 2.4.1
+      tslib: 2.6.2
    dev: false

  /@peculiar/webcrypto@1.4.1:
@@ -6758,7 +6765,7 @@ packages:
      node-stream-zip: 1.15.0
      ora: 5.4.1
      prompts: 2.4.2
-      semver: 7.5.4
+      semver: 7.6.0
      strip-ansi: 5.2.0
      sudo-prompt: 9.2.1
      wcwidth: 1.0.1
@@ -6855,7 +6862,7 @@ packages:
      node-fetch: 2.7.0
      open: 6.4.0
      ora: 5.4.1
-      semver: 7.5.4
+      semver: 7.6.0
      shell-quote: 1.8.1
    transitivePeerDependencies:
      - encoding
@@ -6888,7 +6895,7 @@ packages:
      fs-extra: 8.1.0
      graceful-fs: 4.2.11
      prompts: 2.4.2
-      semver: 7.5.4
+      semver: 7.6.0
    transitivePeerDependencies:
      - '@babel/core'
      - bufferutil
@@ -9429,6 +9436,23 @@ packages:
    resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==}
    dev: false

+  /decode-bmp@0.2.1:
+    resolution: {integrity: sha512-NiOaGe+GN0KJqi2STf24hfMkFitDUaIoUU3eKvP/wAbLe8o6FuW5n/x7MHPR0HKvBokp6MQY/j7w8lewEeVCIA==}
+    engines: {node: '>=8.6.0'}
+    dependencies:
+      '@canvas/image-data': 1.0.0
+      to-data-view: 1.1.0
+    dev: false
+
+  /decode-ico@0.4.1:
+    resolution: {integrity: sha512-69NZfbKIzux1vBOd31al3XnMnH+2mqDhEgLdpygErm4d60N+UwA5Sq5WFjmEDQzumgB9fElojGwWG0vybVfFmA==}
+    engines: {node: '>=8.6'}
+    dependencies:
+      '@canvas/image-data': 1.0.0
+      decode-bmp: 0.2.1
+      to-data-view: 1.1.0
+    dev: false
+
  /deep-extend@0.6.0:
    resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
    engines: {node: '>=4.0.0'}
@@ -9590,7 +9614,7 @@ packages:
    resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==}
    dependencies:
      no-case: 3.0.4
-      tslib: 2.4.1
+      tslib: 2.6.2
    dev: false

  /dotenv-cli@7.3.0:
@@ -11182,6 +11206,15 @@ packages:
    resolution: {integrity: sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==}
    dev: false

+  /ico-to-png@0.2.1:
+    resolution: {integrity: sha512-wP2Jmsj9ZMxi5fIv3VrcQ9w7vmUu4r6ocfMgeDwoHkzG50sY/LYsZcXEZypaD4FkMdjGQU9klNVzxQMMF6rYBw==}
+    engines: {node: '>=8.6.0'}
+    dependencies:
+      decode-ico: 0.4.1
+      lodepng: 2.2.0
+      resize-image-data: 0.3.1
+    dev: false
+
  /iconv-lite@0.4.24:
    resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==}
    engines: {node: '>=0.10.0'}
@@ -12122,6 +12155,14 @@ packages:
    resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
    dev: false

+  /lodepng@2.2.0:
+    resolution: {integrity: sha512-5sq2pmnehly+wMOvWr9CMlsLI0L8ZHDde1a/Ueu/mOu5E3laD3txFQv6tNwH2BSSaYKtyWfWV1+k6+FN5gnoHw==}
+    engines: {node: '>=8.6.0'}
+    requiresBuild: true
+    dependencies:
+      '@canvas/image-data': 1.0.0
+    dev: false
+
  /log-symbols@2.2.0:
    resolution: {integrity: sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==}
    engines: {node: '>=4'}
@@ -12171,7 +12212,7 @@ packages:
  /lower-case@2.0.2:
    resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==}
    dependencies:
-      tslib: 2.4.1
+      tslib: 2.6.2
    dev: false

  /lowlight@1.20.0:
@@ -13014,7 +13055,7 @@ packages:
    resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==}
    dependencies:
      lower-case: 2.0.2
-      tslib: 2.4.1
+      tslib: 2.6.2
    dev: false

  /nocache@3.0.4:
@@ -14370,17 +14411,6 @@ packages:
      react-transition-group: 4.4.5(react-dom@18.2.0)(react@18.2.0)
    dev: false

-  /react-social-icons@6.12.0(react-dom@18.2.0)(react@18.2.0):
-    resolution: {integrity: sha512-XiiWlN4F7srBy0VDDwbYip5l2UA5ttQqQIsY3tZ2ouqdyN9NDZ5ZMKBLjNPM0pNF0qjosRaP/LqgHtmUK2TFRA==}
-    peerDependencies:
-      react: 15.x.x || 16.x.x || 17.x.x || 18.x.x
-      react-dom: 15.x.x || 16.x.x || 17.x.x || 18.x.x
-    dependencies:
-      '@babel/runtime': 7.23.9
-      react: 18.2.0
-      react-dom: 18.2.0(react@18.2.0)
-    dev: false
-
  /react-style-singleton@2.2.1(@types/react@18.2.34)(react@18.2.0):
    resolution: {integrity: sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==}
    engines: {node: '>=10'}
@@ -14710,6 +14740,13 @@ packages:
    resolution: {integrity: sha512-ab9EmR80F/zQTMNeneUr4cv+jSwPJgIlvEmVwLerwrWVbpLlBuls9XHzIeTFy4cegU2NHBp3va0LKOzU5qFEYQ==}
    dev: false

+  /resize-image-data@0.3.1:
+    resolution: {integrity: sha512-6hVRn2S6W1cdycreA6Vth5XRN2NnGs7/RnVpxNw/1OCK8aCoevRFH2WprmQRZDnnH3e6awLv2tTIPuv7/7xeGg==}
+    engines: {node: '>=8.6.0'}
+    dependencies:
+      '@canvas/image-data': 1.0.0
+    dev: false
+
  /resolve-from@3.0.0:
    resolution: {integrity: sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw==}
    engines: {node: '>=4'}
@@ -15135,7 +15172,7 @@ packages:
    resolution: {integrity: sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==}
    dependencies:
      dot-case: 3.0.4
-      tslib: 2.4.1
+      tslib: 2.6.2
    dev: false

  /snakecase-keys@3.2.1:
@@ -15699,6 +15736,10 @@ packages:
    resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==}
    dev: false

+  /to-data-view@1.1.0:
+    resolution: {integrity: sha512-1eAdufMg6mwgmlojAx3QeMnzB/BTVp7Tbndi3U7ftcT2zCZadjxkkmLmd97zmaxWi+sgGcgWrokmpEoy0Dn0vQ==}
+    dev: false
+
  /to-fast-properties@2.0.0:
    resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==}
    engines: {node: '>=4'}