import fs from 'node:fs'; import path, { dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); // extras const extraReferrers = { 'zoom.us': { type: 'social', name: 'Zoom' }, 'apple.com': { type: 'tech', name: 'Apple' }, 'adobe.com': { type: 'tech', name: 'Adobe' }, 'figma.com': { type: 'tech', name: 'Figma' }, 'wix.com': { type: 'commerce', name: 'Wix' }, 'gmail.com': { type: 'email', name: 'Gmail' }, 'notion.so': { type: 'tech', name: 'Notion' }, 'ebay.com': { type: 'commerce', name: 'eBay' }, 'github.com': { type: 'tech', name: 'GitHub' }, 'gitlab.com': { type: 'tech', name: 'GitLab' }, 'slack.com': { type: 'social', name: 'Slack' }, 'etsy.com': { type: 'commerce', name: 'Etsy' }, 'bsky.app': { type: 'social', name: 'Bluesky' }, 'twitch.tv': { type: 'content', name: 'Twitch' }, 'dropbox.com': { type: 'tech', name: 'Dropbox' }, 'outlook.com': { type: 'email', name: 'Outlook' }, 'medium.com': { type: 'content', name: 'Medium' }, 'paypal.com': { type: 'commerce', name: 'PayPal' }, 'discord.com': { type: 'social', name: 'Discord' }, 'stripe.com': { type: 'commerce', name: 'Stripe' }, 'spotify.com': { type: 'content', name: 'Spotify' }, 'netflix.com': { type: 'content', name: 'Netflix' }, 'whatsapp.com': { type: 'social', name: 'WhatsApp' }, 'shopify.com': { type: 'commerce', name: 'Shopify' }, 'microsoft.com': { type: 'tech', name: 'Microsoft' }, 'alibaba.com': { type: 'commerce', name: 'Alibaba' }, 'telegram.org': { type: 'social', name: 'Telegram' }, 'substack.com': { type: 'content', name: 'Substack' }, 'salesforce.com': { type: 'tech', name: 'Salesforce' }, 'instagram.com': { type: 'social', name: 'Instagram' }, 'wikipedia.org': { type: 'content', name: 'Wikipedia' }, 'mastodon.social': { type: 'social', name: 'Mastodon' }, 'office.com': { type: 'tech', name: 'Microsoft Office' }, 'squarespace.com': { type: 'commerce', name: 'Squarespace' }, 'stackoverflow.com': { type: 'tech', name: 'Stack Overflow' }, 'teams.microsoft.com': { type: 'social', name: 'Microsoft Teams' }, 'chat.com': { type: 'ai', name: 'Chat.com' }, 'chatgpt.com': { type: 'ai', name: 'ChatGPT' }, 'openai.com': { type: 'ai', name: 'OpenAI' }, 'anthropic.com': { type: 'ai', name: 'Anthropic' }, 'claude.ai': { type: 'ai', name: 'Claude' }, 'gemini.google.com': { type: 'ai', name: 'Google Gemini' }, 'bard.google.com': { type: 'ai', name: 'Google Bard' }, 'copilot.microsoft.com': { type: 'ai', name: 'Microsoft Copilot' }, 'copilot.cloud.microsoft': { type: 'ai', name: 'Microsoft Copilot' }, 'perplexity.ai': { type: 'ai', name: 'Perplexity' }, 'you.com': { type: 'ai', name: 'You.com' }, 'poe.com': { type: 'ai', name: 'Poe' }, 'phind.com': { type: 'ai', name: 'Phind' }, 'huggingface.co': { type: 'ai', name: 'Hugging Face' }, 'hf.co': { type: 'ai', name: 'Hugging Face' }, 'character.ai': { type: 'ai', name: 'Character.AI' }, 'meta.ai': { type: 'ai', name: 'Meta AI' }, 'mistral.ai': { type: 'ai', name: 'Mistral' }, 'chat.mistral.ai': { type: 'ai', name: 'Mistral Le Chat' }, 'deepseek.com': { type: 'ai', name: 'DeepSeek' }, 'chat.deepseek.com': { type: 'ai', name: 'DeepSeek Chat' }, 'pi.ai': { type: 'ai', name: 'Pi' }, 'inflection.ai': { type: 'ai', name: 'Inflection' }, 'cohere.com': { type: 'ai', name: 'Cohere' }, 'coral.cohere.com': { type: 'ai', name: 'Cohere Coral' }, 'jasper.ai': { type: 'ai', name: 'Jasper' }, 'writesonic.com': { type: 'ai', name: 'Writesonic' }, 'copy.ai': { type: 'ai', name: 'Copy.ai' }, 'rytr.me': { type: 'ai', name: 'Rytr' }, 'notion.ai': { type: 'ai', name: 'Notion AI' }, 'grammarly.com': { type: 'ai', name: 'Grammarly' }, 'grok.com': { type: 'ai', name: 'Grok' }, 'x.ai': { type: 'ai', name: 'xAI' }, 'aistudio.google.com': { type: 'ai', name: 'Google AI Studio' }, 'labs.google.com': { type: 'ai', name: 'Google Labs' }, 'ai.google': { type: 'ai', name: 'Google AI' }, 'forefront.ai': { type: 'ai', name: 'Forefront' }, 'together.ai': { type: 'ai', name: 'Together AI' }, 'groq.com': { type: 'ai', name: 'Groq' }, 'replicate.com': { type: 'ai', name: 'Replicate' }, 'vercel.ai': { type: 'ai', name: 'Vercel AI' }, 'v0.dev': { type: 'ai', name: 'v0' }, 'bolt.new': { type: 'ai', name: 'Bolt' }, 'replit.com': { type: 'ai', name: 'Replit' }, 'cursor.com': { type: 'ai', name: 'Cursor' }, 'tabnine.com': { type: 'ai', name: 'Tabnine' }, 'codeium.com': { type: 'ai', name: 'Codeium' }, 'sourcegraph.com': { type: 'ai', name: 'Sourcegraph Cody' }, 'kimi.moonshot.cn': { type: 'ai', name: 'Kimi' }, 'moonshot.ai': { type: 'ai', name: 'Moonshot AI' }, 'doubao.com': { type: 'ai', name: 'Doubao' }, 'tongyi.aliyun.com': { type: 'ai', name: 'Tongyi Qianwen' }, 'yiyan.baidu.com': { type: 'ai', name: 'Ernie Bot' }, 'chatglm.cn': { type: 'ai', name: 'ChatGLM' }, 'zhipu.ai': { type: 'ai', name: 'Zhipu AI' }, 'minimax.chat': { type: 'ai', name: 'MiniMax' }, 'lmsys.org': { type: 'ai', name: 'LMSYS' }, 'chat.lmsys.org': { type: 'ai', name: 'LMSYS Chat' }, 'llama.meta.com': { type: 'ai', name: 'Meta Llama' }, }; function transform(data: any) { const obj: Record = {}; for (const type in data) { for (const name in data[type]) { const domains = data[type][name].domains ?? []; for (const domain of domains) { obj[domain] = { type, name, }; } } } return obj; } async function main() { // Get document, or throw exception on error try { const data = await fetch( 'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json' ).then((res) => res.json()); fs.writeFileSync( path.resolve(__dirname, '../../worker/src/referrers/index.ts'), [ '// This file is generated by the script get-referrers.ts', '', '// The data is fetch from snowplow-referer-parser https://github.com/snowplow-referer-parser/referer-parser', `// The orginal referers.yml is based on Piwik's SearchEngines.php and Socials.php, copyright 2012 Matthieu Aubry and available under the GNU General Public License v3.`, '', `const referrers: Record = ${JSON.stringify( { ...transform(data), ...extraReferrers, } )} as const;`, 'export default referrers;', ].join('\n'), 'utf-8' ); } catch (e) { console.log(e); } } main();