diff --git a/apps/api/package.json b/apps/api/package.json index 348016a3..6bf07684 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -25,6 +25,7 @@ "ramda": "^0.29.1", "sharp": "^0.33.2", "ua-parser-js": "^1.0.37", + "url-metadata": "^4.1.0", "uuid": "^9.0.1" }, "devDependencies": { diff --git a/apps/api/src/controllers/misc.controller.ts b/apps/api/src/controllers/misc.controller.ts index 2932b5be..41b172ba 100644 --- a/apps/api/src/controllers/misc.controller.ts +++ b/apps/api/src/controllers/misc.controller.ts @@ -1,3 +1,5 @@ +import { logger } from '@/utils/logger'; +import { parseUrlMeta } from '@/utils/parseUrlMeta'; import type { FastifyReply, FastifyRequest } from 'fastify'; import icoToPng from 'ico-to-png'; import sharp from 'sharp'; @@ -35,8 +37,7 @@ async function getImageBuffer(url: string) { .png() .toBuffer(); } catch (e) { - console.log('Failed to get image from url', url); - console.log(e); + logger.error(e, `Failed to get image from url ${url}`); } } @@ -53,8 +54,6 @@ export async function getFavicon( redis.set(`favicon:${cacheKey}`, buffer.toString('base64')); } reply.type('image/png'); - console.log('buffer', buffer.byteLength); - return reply.send(buffer); } @@ -64,7 +63,6 @@ export async function getFavicon( const url = decodeURIComponent(request.query.url); - // DIRECT IMAGE if (imageExtensions.find((ext) => url.endsWith(ext))) { const cacheKey = createHash(url, 32); const cache = await redis.get(`favicon:${cacheKey}`); @@ -77,41 +75,28 @@ export async function getFavicon( } } - const { hostname, origin } = new URL(url); + const { hostname } = new URL(url); const cache = await redis.get(`favicon:${hostname}`); + if (cache) { return sendBuffer(Buffer.from(cache, 'base64')); } - // TRY FAVICON.ICO - const buffer = await getImageBuffer(`${origin}/favicon.ico`); - if (buffer && buffer.byteLength > 0) { - return sendBuffer(buffer, hostname); - } - - // PARSE HTML - const res = await fetch(url).then((res) => res.text()); - - function findFavicon(res: string) { - const match = res.match( - /(\|\)/ - ); - if (!match) { - return null; - } - - return match[0].match(/href="(.+?)"/)?.[1] ?? null; - } - - const favicon = findFavicon(res); - if (favicon) { - const buffer = await getImageBuffer(favicon); - + const meta = await parseUrlMeta(url); + if (meta?.favicon) { + const buffer = await getImageBuffer(meta.favicon); if (buffer && buffer.byteLength > 0) { return sendBuffer(buffer, hostname); } } + const buffer = await getImageBuffer( + 'https://www.iconsdb.com/icons/download/orange/warning-128.png' + ); + if (buffer && buffer.byteLength > 0) { + return sendBuffer(buffer, hostname); + } + return reply.status(404).send('Not found'); } diff --git a/apps/api/src/utils/parseUrlMeta.ts b/apps/api/src/utils/parseUrlMeta.ts new file mode 100644 index 00000000..a6c49735 --- /dev/null +++ b/apps/api/src/utils/parseUrlMeta.ts @@ -0,0 +1,38 @@ +import urlMetadata from 'url-metadata'; + +function findBestFavicon(favicons: UrlMetaData['favicons']) { + const match = favicons.find( + (favicon) => + favicon.rel === 'shortcut icon' || + favicon.rel === 'icon' || + favicon.rel === 'apple-touch-icon' + ); + if (match) { + return match.href; + } + return null; +} + +function transform(data: UrlMetaData, url: string) { + const favicon = findBestFavicon(data.favicons); + return { + favicon: favicon ? new URL(favicon, url).toString() : null, + }; +} + +interface UrlMetaData { + favicons: { + rel: string; + href: string; + sizes: string; + }[]; +} + +export async function parseUrlMeta(url: string) { + try { + const metadata = (await urlMetadata(url)) as UrlMetaData; + return transform(metadata, url); + } catch (err) { + return null; + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8bff5760..6c2fb50f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,6 +68,9 @@ importers: ua-parser-js: specifier: ^1.0.37 version: 1.0.37 + url-metadata: + specifier: ^4.1.0 + version: 4.1.0 uuid: specifier: ^9.0.1 version: 9.0.1 @@ -7686,6 +7689,10 @@ packages: - supports-color dev: false + /boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + dev: false + /bplist-creator@0.1.0: resolution: {integrity: sha512-sXaHZicyEEmY86WyueLTQesbeoH/mquvarJaQNbjuOQO+7gbFcDEWqKmcWA4cOTLzFlfgvkiVxolk1k5bBIpmg==} dependencies: @@ -7974,6 +7981,30 @@ packages: resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} dev: false + /cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + dev: false + + /cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.1.0 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + dev: false + /chokidar@3.6.0: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} @@ -8405,6 +8436,21 @@ packages: resolution: {integrity: sha512-COtn4EROW5dBGlE/4PiKnh6rZpAPxDeFLaEEwt4i10jpDMFt2EhQGS79QmmrO+iKCHv0PU/HrOWEhijFd1x99Q==} dev: false + /css-select@5.1.0: + resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 5.0.3 + domutils: 3.1.0 + nth-check: 2.1.1 + dev: false + + /css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + dev: false + /cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -8726,6 +8772,11 @@ packages: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} dev: false + /data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + dev: false + /date-fns@3.3.1: resolution: {integrity: sha512-y8e109LYGgoQDveiEBD3DYXKba1jWf5BA8YU1FL5Tvm0BTdEfy54WLCwnuYWZNnzzvALy/QQ4Hov+Q9RVRv+Zw==} @@ -8981,10 +9032,37 @@ packages: csstype: 3.1.3 dev: false + /dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + dev: false + + /domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + dev: false + + /domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + /dompurify@3.0.9: resolution: {integrity: sha512-uyb4NDIvQ3hRn6NiC+SIFaP4mJ/MdXlvtunaqK9Bn6dD3RuB/1S/gasEjDHD8eiaqdSael2vBv+hOs7Y+jhYOQ==} dev: false + /domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dev: false + /dot-case@3.0.4: resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==} dependencies: @@ -10031,6 +10109,14 @@ packages: - encoding dev: false + /fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + dev: false + /fetch-retry@4.1.1: resolution: {integrity: sha512-e6eB7zN6UBSwGVwrbWVH+gdLnkW9WwHhmq2YDK1Sh30pzx1onRVGBvogTlUeWxwTa+L86NYdo4hFkh7O8ZjSnA==} dev: false @@ -10192,6 +10278,13 @@ packages: engines: {node: '>=0.4.x'} dev: false + /formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.2.0 + dev: false + /forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -10799,6 +10892,15 @@ packages: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} dev: false + /htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + dev: false + /http-errors@2.0.0: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} engines: {node: '>= 0.8'} @@ -13390,6 +13492,11 @@ packages: minimatch: 3.1.2 dev: false + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + /node-fetch-native@1.0.1: resolution: {integrity: sha512-VzW+TAk2wE4X9maiKMlT+GsPU4OMmR1U9CrHSmd3DFLn2IcZ9VJ6M6BBugGfYUnPCLSYxXdZy17M0BEJyhUTwg==} dev: false @@ -13406,6 +13513,15 @@ packages: whatwg-url: 5.0.0 dev: false + /node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + dev: false + /node-forge@1.3.1: resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} engines: {node: '>= 6.13.0'} @@ -13487,6 +13603,12 @@ packages: set-blocking: 2.0.0 dev: false + /nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + dependencies: + boolbase: 1.0.0 + dev: false + /nullthrows@1.1.1: resolution: {integrity: sha512-2vPPEi+Z7WqML2jZYddDIfy5Dqb0r2fze2zTxNNknZaFpVHU3mFB3R+DWeJWGVx0ecvttSGlJTI+WG+8Z4cDWw==} dev: false @@ -13835,6 +13957,13 @@ packages: parse-path: 7.0.0 dev: false + /parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + dev: false + /parse5@7.1.2: resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} dependencies: @@ -16632,6 +16761,14 @@ packages: resolution: {integrity: sha512-EGXjXJZhIHiQMK2pQukuFcL303nskqIRzWvPvV5O8miOfwoUb9G+a/Cld60kUyeaybEI94wvVClT10DtfeAExA==} dev: false + /url-metadata@4.1.0: + resolution: {integrity: sha512-Ce6V/XzPUf5/0isAkn8EDLueV7+fDLnVPOf/42gudkKI4GF9q3ddMrROK61bTU+neB+LP33drToW+Fo+rJleMg==} + engines: {node: '>=18.0.0'} + dependencies: + cheerio: 1.0.0-rc.12 + node-fetch: 3.3.2 + dev: false + /use-callback-ref@1.3.1(@types/react@18.2.56)(react@18.2.0): resolution: {integrity: sha512-Lg4Vx1XZQauB42Hw3kK7JM6yjVjgFmFC5/Ab797s79aARomD2nEErc4mCgM8EZrARLmmbWpi5DGCadmK50DcAQ==} engines: {node: '>=10'} @@ -16837,6 +16974,11 @@ packages: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} dev: false + /web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + dev: false + /web-worker@1.3.0: resolution: {integrity: sha512-BSR9wyRsy/KOValMgd5kMyr3JzpdeoR9KVId8u5GVlTTAtNChlsE4yTxeY7zMdNSyOmoKBv8NH2qeRY9Tg+IaA==} dev: false