api: clean up favicon route

2024-03-21 09:01:22 +01:00
parent 2469c6e1b8
commit 709449e55c
4 changed files with 196 additions and 30 deletions
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -25,6 +25,7 @@
    "ramda": "^0.29.1",
    "sharp": "^0.33.2",
    "ua-parser-js": "^1.0.37",
+    "url-metadata": "^4.1.0",
    "uuid": "^9.0.1"
  },
  "devDependencies": {
--- a/apps/api/src/controllers/misc.controller.ts
+++ b/apps/api/src/controllers/misc.controller.ts
@@ -1,3 +1,5 @@
+import { logger } from '@/utils/logger';
+import { parseUrlMeta } from '@/utils/parseUrlMeta';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import icoToPng from 'ico-to-png';
 import sharp from 'sharp';
@@ -35,8 +37,7 @@ async function getImageBuffer(url: string) {
      .png()
      .toBuffer();
  } catch (e) {
-    console.log('Failed to get image from url', url);
-    console.log(e);
+    logger.error(e, `Failed to get image from url ${url}`);
  }
 }

@@ -53,8 +54,6 @@ export async function getFavicon(
      redis.set(`favicon:${cacheKey}`, buffer.toString('base64'));
    }
    reply.type('image/png');
-    console.log('buffer', buffer.byteLength);
-
    return reply.send(buffer);
  }

@@ -64,7 +63,6 @@ export async function getFavicon(

  const url = decodeURIComponent(request.query.url);

-  // DIRECT IMAGE
  if (imageExtensions.find((ext) => url.endsWith(ext))) {
    const cacheKey = createHash(url, 32);
    const cache = await redis.get(`favicon:${cacheKey}`);
@@ -77,41 +75,28 @@ export async function getFavicon(
    }
  }

-  const { hostname, origin } = new URL(url);
+  const { hostname } = new URL(url);
  const cache = await redis.get(`favicon:${hostname}`);
+
  if (cache) {
    return sendBuffer(Buffer.from(cache, 'base64'));
  }

-  // TRY FAVICON.ICO
-  const buffer = await getImageBuffer(`${origin}/favicon.ico`);
-  if (buffer && buffer.byteLength > 0) {
-    return sendBuffer(buffer, hostname);
-  }
-
-  // PARSE HTML
-  const res = await fetch(url).then((res) => res.text());
-
-  function findFavicon(res: string) {
-    const match = res.match(
-      /(\<link(.+?)image\/x-icon(.+?)\>|\<link(.+?)shortcut\sicon(.+?)\>)/
-    );
-    if (!match) {
-      return null;
-    }
-
-    return match[0].match(/href="(.+?)"/)?.[1] ?? null;
-  }
-
-  const favicon = findFavicon(res);
-  if (favicon) {
-    const buffer = await getImageBuffer(favicon);
-
+  const meta = await parseUrlMeta(url);
+  if (meta?.favicon) {
+    const buffer = await getImageBuffer(meta.favicon);
    if (buffer && buffer.byteLength > 0) {
      return sendBuffer(buffer, hostname);
    }
  }

+  const buffer = await getImageBuffer(
+    'https://www.iconsdb.com/icons/download/orange/warning-128.png'
+  );
+  if (buffer && buffer.byteLength > 0) {
+    return sendBuffer(buffer, hostname);
+  }
+
  return reply.status(404).send('Not found');
 }

--- a/apps/api/src/utils/parseUrlMeta.ts
+++ b/apps/api/src/utils/parseUrlMeta.ts
@@ -0,0 +1,38 @@
+import urlMetadata from 'url-metadata';
+
+function findBestFavicon(favicons: UrlMetaData['favicons']) {
+  const match = favicons.find(
+    (favicon) =>
+      favicon.rel === 'shortcut icon' ||
+      favicon.rel === 'icon' ||
+      favicon.rel === 'apple-touch-icon'
+  );
+  if (match) {
+    return match.href;
+  }
+  return null;
+}
+
+function transform(data: UrlMetaData, url: string) {
+  const favicon = findBestFavicon(data.favicons);
+  return {
+    favicon: favicon ? new URL(favicon, url).toString() : null,
+  };
+}
+
+interface UrlMetaData {
+  favicons: {
+    rel: string;
+    href: string;
+    sizes: string;
+  }[];
+}
+
+export async function parseUrlMeta(url: string) {
+  try {
+    const metadata = (await urlMetadata(url)) as UrlMetaData;
+    return transform(metadata, url);
+  } catch (err) {
+    return null;
+  }
+}
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -68,6 +68,9 @@ importers:
      ua-parser-js:
        specifier: ^1.0.37
        version: 1.0.37
+      url-metadata:
+        specifier: ^4.1.0
+        version: 4.1.0
      uuid:
        specifier: ^9.0.1
        version: 9.0.1
@@ -7686,6 +7689,10 @@ packages:
      - supports-color
    dev: false

+  /boolbase@1.0.0:
+    resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
+    dev: false
+
  /bplist-creator@0.1.0:
    resolution: {integrity: sha512-sXaHZicyEEmY86WyueLTQesbeoH/mquvarJaQNbjuOQO+7gbFcDEWqKmcWA4cOTLzFlfgvkiVxolk1k5bBIpmg==}
    dependencies:
@@ -7974,6 +7981,30 @@ packages:
    resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==}
    dev: false

+  /cheerio-select@2.1.0:
+    resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==}
+    dependencies:
+      boolbase: 1.0.0
+      css-select: 5.1.0
+      css-what: 6.1.0
+      domelementtype: 2.3.0
+      domhandler: 5.0.3
+      domutils: 3.1.0
+    dev: false
+
+  /cheerio@1.0.0-rc.12:
+    resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}
+    engines: {node: '>= 6'}
+    dependencies:
+      cheerio-select: 2.1.0
+      dom-serializer: 2.0.0
+      domhandler: 5.0.3
+      domutils: 3.1.0
+      htmlparser2: 8.0.2
+      parse5: 7.1.2
+      parse5-htmlparser2-tree-adapter: 7.0.0
+    dev: false
+
  /chokidar@3.6.0:
    resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}
    engines: {node: '>= 8.10.0'}
@@ -8405,6 +8436,21 @@ packages:
    resolution: {integrity: sha512-COtn4EROW5dBGlE/4PiKnh6rZpAPxDeFLaEEwt4i10jpDMFt2EhQGS79QmmrO+iKCHv0PU/HrOWEhijFd1x99Q==}
    dev: false

+  /css-select@5.1.0:
+    resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==}
+    dependencies:
+      boolbase: 1.0.0
+      css-what: 6.1.0
+      domhandler: 5.0.3
+      domutils: 3.1.0
+      nth-check: 2.1.1
+    dev: false
+
+  /css-what@6.1.0:
+    resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}
+    engines: {node: '>= 6'}
+    dev: false
+
  /cssesc@3.0.0:
    resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}
    engines: {node: '>=4'}
@@ -8726,6 +8772,11 @@ packages:
    resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==}
    dev: false

+  /data-uri-to-buffer@4.0.1:
+    resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
+    engines: {node: '>= 12'}
+    dev: false
+
  /date-fns@3.3.1:
    resolution: {integrity: sha512-y8e109LYGgoQDveiEBD3DYXKba1jWf5BA8YU1FL5Tvm0BTdEfy54WLCwnuYWZNnzzvALy/QQ4Hov+Q9RVRv+Zw==}

@@ -8981,10 +9032,37 @@ packages:
      csstype: 3.1.3
    dev: false

+  /dom-serializer@2.0.0:
+    resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==}
+    dependencies:
+      domelementtype: 2.3.0
+      domhandler: 5.0.3
+      entities: 4.5.0
+    dev: false
+
+  /domelementtype@2.3.0:
+    resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==}
+    dev: false
+
+  /domhandler@5.0.3:
+    resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==}
+    engines: {node: '>= 4'}
+    dependencies:
+      domelementtype: 2.3.0
+    dev: false
+
  /dompurify@3.0.9:
    resolution: {integrity: sha512-uyb4NDIvQ3hRn6NiC+SIFaP4mJ/MdXlvtunaqK9Bn6dD3RuB/1S/gasEjDHD8eiaqdSael2vBv+hOs7Y+jhYOQ==}
    dev: false

+  /domutils@3.1.0:
+    resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==}
+    dependencies:
+      dom-serializer: 2.0.0
+      domelementtype: 2.3.0
+      domhandler: 5.0.3
+    dev: false
+
  /dot-case@3.0.4:
    resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==}
    dependencies:
@@ -10031,6 +10109,14 @@ packages:
      - encoding
    dev: false

+  /fetch-blob@3.2.0:
+    resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
+    engines: {node: ^12.20 || >= 14.13}
+    dependencies:
+      node-domexception: 1.0.0
+      web-streams-polyfill: 3.3.3
+    dev: false
+
  /fetch-retry@4.1.1:
    resolution: {integrity: sha512-e6eB7zN6UBSwGVwrbWVH+gdLnkW9WwHhmq2YDK1Sh30pzx1onRVGBvogTlUeWxwTa+L86NYdo4hFkh7O8ZjSnA==}
    dev: false
@@ -10192,6 +10278,13 @@ packages:
    engines: {node: '>=0.4.x'}
    dev: false

+  /formdata-polyfill@4.0.10:
+    resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
+    engines: {node: '>=12.20.0'}
+    dependencies:
+      fetch-blob: 3.2.0
+    dev: false
+
  /forwarded@0.2.0:
    resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
    engines: {node: '>= 0.6'}
@@ -10799,6 +10892,15 @@ packages:
    resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}
    dev: false

+  /htmlparser2@8.0.2:
+    resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}
+    dependencies:
+      domelementtype: 2.3.0
+      domhandler: 5.0.3
+      domutils: 3.1.0
+      entities: 4.5.0
+    dev: false
+
  /http-errors@2.0.0:
    resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
    engines: {node: '>= 0.8'}
@@ -13390,6 +13492,11 @@ packages:
      minimatch: 3.1.2
    dev: false

+  /node-domexception@1.0.0:
+    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
+    engines: {node: '>=10.5.0'}
+    dev: false
+
  /node-fetch-native@1.0.1:
    resolution: {integrity: sha512-VzW+TAk2wE4X9maiKMlT+GsPU4OMmR1U9CrHSmd3DFLn2IcZ9VJ6M6BBugGfYUnPCLSYxXdZy17M0BEJyhUTwg==}
    dev: false
@@ -13406,6 +13513,15 @@ packages:
      whatwg-url: 5.0.0
    dev: false

+  /node-fetch@3.3.2:
+    resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      data-uri-to-buffer: 4.0.1
+      fetch-blob: 3.2.0
+      formdata-polyfill: 4.0.10
+    dev: false
+
  /node-forge@1.3.1:
    resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==}
    engines: {node: '>= 6.13.0'}
@@ -13487,6 +13603,12 @@ packages:
      set-blocking: 2.0.0
    dev: false

+  /nth-check@2.1.1:
+    resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
+    dependencies:
+      boolbase: 1.0.0
+    dev: false
+
  /nullthrows@1.1.1:
    resolution: {integrity: sha512-2vPPEi+Z7WqML2jZYddDIfy5Dqb0r2fze2zTxNNknZaFpVHU3mFB3R+DWeJWGVx0ecvttSGlJTI+WG+8Z4cDWw==}
    dev: false
@@ -13835,6 +13957,13 @@ packages:
      parse-path: 7.0.0
    dev: false

+  /parse5-htmlparser2-tree-adapter@7.0.0:
+    resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==}
+    dependencies:
+      domhandler: 5.0.3
+      parse5: 7.1.2
+    dev: false
+
  /parse5@7.1.2:
    resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==}
    dependencies:
@@ -16632,6 +16761,14 @@ packages:
    resolution: {integrity: sha512-EGXjXJZhIHiQMK2pQukuFcL303nskqIRzWvPvV5O8miOfwoUb9G+a/Cld60kUyeaybEI94wvVClT10DtfeAExA==}
    dev: false

+  /url-metadata@4.1.0:
+    resolution: {integrity: sha512-Ce6V/XzPUf5/0isAkn8EDLueV7+fDLnVPOf/42gudkKI4GF9q3ddMrROK61bTU+neB+LP33drToW+Fo+rJleMg==}
+    engines: {node: '>=18.0.0'}
+    dependencies:
+      cheerio: 1.0.0-rc.12
+      node-fetch: 3.3.2
+    dev: false
+
  /use-callback-ref@1.3.1(@types/react@18.2.56)(react@18.2.0):
    resolution: {integrity: sha512-Lg4Vx1XZQauB42Hw3kK7JM6yjVjgFmFC5/Ab797s79aARomD2nEErc4mCgM8EZrARLmmbWpi5DGCadmK50DcAQ==}
    engines: {node: '>=10'}
@@ -16837,6 +16974,11 @@ packages:
    resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==}
    dev: false

+  /web-streams-polyfill@3.3.3:
+    resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
+    engines: {node: '>= 8'}
+    dev: false
+
  /web-worker@1.3.0:
    resolution: {integrity: sha512-BSR9wyRsy/KOValMgd5kMyr3JzpdeoR9KVId8u5GVlTTAtNChlsE4yTxeY7zMdNSyOmoKBv8NH2qeRY9Tg+IaA==}
    dev: false