chore(api): update bots and referrers

apps/api/package.json

@@ -6,6 +6,8 @@
     "testing": "API_PORT=3333 pnpm dev",
     "start": "node dist/index.js",
     "build": "rm -rf dist && tsup",
+    "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write src/referrers/index.ts",
+    "gen:bots": "jiti scripts/get-bots.ts && biome format --write src/bots/bots.ts",
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
@@ -16,8 +18,8 @@
     "@fastify/websocket": "^8.3.1",
     "@openpanel/common": "workspace:*",
     "@openpanel/db": "workspace:*",
-    "@openpanel/logger": "workspace:*",
     "@openpanel/integrations": "workspace:^",
+    "@openpanel/logger": "workspace:*",
     "@openpanel/queue": "workspace:*",
     "@openpanel/redis": "workspace:*",
     "@openpanel/trpc": "workspace:*",
@@ -33,7 +35,6 @@
     "sqlstring": "^2.3.3",
     "superjson": "^1.13.3",
     "svix": "^1.24.0",
-    "ua-parser-js": "^1.0.37",
     "url-metadata": "^4.1.0",
     "uuid": "^9.0.1",
     "zod": "^3.22.4"
@@ -42,13 +43,14 @@
     "@faker-js/faker": "^9.0.1",
     "@openpanel/sdk": "workspace:*",
     "@openpanel/tsconfig": "workspace:*",
+    "@types/js-yaml": "^4.0.9",
     "@types/jsonwebtoken": "^9.0.6",
     "@types/ramda": "^0.29.6",
     "@types/request-ip": "^0.0.41",
     "@types/sqlstring": "^2.3.2",
-    "@types/ua-parser-js": "^0.7.39",
     "@types/uuid": "^9.0.8",
     "@types/ws": "^8.5.10",
+    "js-yaml": "^4.1.0",
     "tsup": "^7.2.0",
     "typescript": "^5.2.2"
   }

apps/api/scripts/get-bots.ts

@@ -0,0 +1,29 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import yaml from 'js-yaml';
+
+async function main() {
+  // Get document, or throw exception on error
+  try {
+    const data = await fetch(
+      'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
+    ).then((res) => res.text());
+
+    fs.writeFileSync(
+      path.resolve(__dirname, '../src/bots/bots.ts'),
+      [
+        '// This file is generated by the script get-bots.ts',
+        '',
+        '// The data is fetch from device-detector https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
+        '',
+        `const bots = ${JSON.stringify(yaml.load(data))} as const;`,
+        'export default bots;',
+      ].join('\n'),
+      'utf-8',
+    );
+  } catch (e) {
+    console.log(e);
+  }
+}
+
+main();

apps/api/scripts/get-organizations.ts

@@ -1,189 +0,0 @@
-// import { clerkClient } from '@clerk/fastify';
-
-import { db } from '@openpanel/db';
-
-// import { db } from '@openpanel/db';
-
-// type Fn<T = unknown> = (args: { limit: number; offset: number }) => Promise<{
-//   data: T[];
-//   totalCount: number;
-// }>;
-
-// function getAllDataByPagination<T extends Fn>(
-//   cb: T
-// ): Promise<Awaited<ReturnType<T>>['data']> {
-//   const data: Awaited<ReturnType<T>>['data'] = [];
-//   async function getData(page = 0) {
-//     console.log(`getData with offset ${page * 100}`);
-//     const response = await cb({
-//       limit: 100,
-//       offset: page * 100,
-//     });
-//     if (response.data.length !== 0) {
-//       data.push(...response.data);
-//       await getData(page + 1);
-//     }
-//     await new Promise((resolve) => setTimeout(resolve, 100));
-//   }
-
-//   return getData().then(() => data);
-// }
-
-// async function main() {
-//   const organizations = await getAllDataByPagination(
-//     clerkClient.organizations.getOrganizationList.bind(
-//       clerkClient.organizations
-//     )
-//   );
-//   const users = await getAllDataByPagination(
-//     clerkClient.users.getUserList.bind(clerkClient.users)
-//   );
-
-//   console.log(`Found ${organizations.length} organizations`);
-//   console.log(`Found ${users.length} users`);
-
-//   for (const user of users.slice(-10)) {
-//     const email = user.primaryEmailAddress?.emailAddress;
-//     console.log('Check', email);
-
-//     try {
-//       if (email) {
-//         const exists = await db.user.findUnique({
-//           where: {
-//             id: user.id,
-//           },
-//         });
-
-//         if (exists) {
-//           console.log('already exists');
-//         } else {
-//           await db.user.create({
-//             data: {
-//               id: user.id,
-//               email: email,
-//               firstName: user.firstName,
-//               lastName: user.lastName,
-//             },
-//           });
-//         }
-//       } else {
-//         console.log('No email?', user);
-//       }
-//     } catch (e) {
-//       console.log('ERROR');
-//       console.log('');
-//       console.log('');
-//       console.dir(user, { depth: null });
-
-//       console.log('');
-//       console.log('');
-//       console.log('');
-//     }
-//   }
-
-//   for (const org of organizations.slice(-20)) {
-//     try {
-//       if (org.slug) {
-//         const exists = await db.organization.findUnique({
-//           where: {
-//             id: org.slug,
-//           },
-//         });
-
-//         if (exists) {
-//           console.log('already exists org');
-//         } else {
-//           const clerkOrgMembers =
-//             await clerkClient.organizations.getOrganizationMembershipList({
-//               organizationId: org.id,
-//             });
-
-//           const members = clerkOrgMembers.data.map((member) => {
-//             const user = users.find(
-//               (u) => u.id === member.publicUserData?.userId
-//             );
-//             return {
-//               userId: member.publicUserData?.userId,
-//               role: member.role,
-//               email: user!.primaryEmailAddress!.emailAddress,
-//             };
-//           });
-
-//           await db.organization.create({
-//             data: {
-//               id: org.slug,
-//               name: org.name,
-//               createdBy: {
-//                 connect: {
-//                   id: org.createdBy,
-//                 },
-//               },
-//               members: {
-//                 create: members,
-//               },
-//             },
-//           });
-
-//           const invites =
-//             await clerkClient.organizations.getOrganizationInvitationList({
-//               organizationId: org.id,
-//               status: ['pending'],
-//             });
-
-//           for (const invite of invites.data) {
-//             await db.member.create({
-//               data: {
-//                 email: invite.emailAddress,
-//                 organizationId: org.slug,
-//                 role: invite.role,
-//                 userId: null,
-//                 meta: {
-//                   access: invite.publicMetadata?.access as string[],
-//                   invitationId: invite.id,
-//                 },
-//               },
-//             });
-//           }
-//         }
-//       } else {
-//         console.log('org does not have any slug', org);
-//       }
-//     } catch (e) {
-//       console.log('ERROR');
-//       console.log('');
-//       console.log('');
-//       console.dir(org, { depth: null });
-//       console.log('');
-//       console.log('');
-//       console.log('');
-//     }
-//   }
-
-//   process.exit(0);
-// }
-
-// main();
-
-async function main() {
-  const organization = await db.organization.findUnique({
-    where: {
-      id: 'openpanel-dev',
-      members: {
-        some: {
-          userId: 'user_2cEoI8b1SuEFbZERGEAyVvC676F',
-        },
-      },
-    },
-    include: {
-      members: {
-        select: {
-          role: true,
-          user: true,
-        },
-      },
-    },
-  });
-
-  console.dir(organization, { depth: null });
-}
-main();

apps/api/scripts/migrate-origins.ts

@@ -1,65 +0,0 @@
-import { TABLE_NAMES, ch, chQuery } from '@openpanel/db';
-
-async function main() {
-  const projects = await chQuery(
-    `SELECT distinct project_id FROM ${TABLE_NAMES.events} ORDER BY project_id`,
-  );
-  const withOrigin = [];
-
-  for (const project of projects) {
-    try {
-      const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([
-        await chQuery(
-          `SELECT * FROM ${TABLE_NAMES.events} WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`,
-        ),
-        await chQuery(
-          `SELECT * FROM ${TABLE_NAMES.events} WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`,
-        ),
-      ]);
-
-      if (eventWithOrigin[0] && eventWithoutOrigin[0]) {
-        console.log(`Project ${project.project_id} as events without origin`);
-        console.log(`- Origin: ${eventWithOrigin[0].origin}`);
-        withOrigin.push(project.project_id);
-        const events = await chQuery(
-          `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`,
-        );
-        console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`);
-        await ch.command({
-          query: `ALTER TABLE events UPDATE origin = '${eventWithOrigin[0].origin}' WHERE project_id = '${project.project_id}' AND path != '' AND origin = '';`,
-          clickhouse_settings: {
-            wait_end_of_query: 1,
-          },
-        });
-      }
-
-      if (!eventWithOrigin[0] && eventWithoutOrigin[0]) {
-        console.log(
-          `😧 Project ${project.project_id} has no events with origin (last event ${eventWithoutOrigin[0].created_at})`,
-        );
-        console.log('- NO ORIGIN');
-      }
-
-      if (!eventWithOrigin[0] && !eventWithoutOrigin[0]) {
-        console.log(
-          `🔥 WARNING: Project ${project.project_id} has no events at all?!?!?!`,
-        );
-      }
-
-      if (eventWithOrigin[0] && !eventWithoutOrigin[0]) {
-        console.log(
-          `✅ Project ${project.project_id} has all events with origin!!!`,
-        );
-      }
-      console.log('');
-      console.log('');
-
-      await new Promise((resolve) => setTimeout(resolve, 500));
-    } catch (e) {
-      console.log('🥵 ERROR ORRROR');
-      console.log('Error for project', project.project_id);
-    }
-  }
-  process.exit(0);
-}
-main();

apps/worker/package.json

@@ -22,7 +22,6 @@
     "prom-client": "^15.1.3",
     "ramda": "^0.29.1",
     "sqlstring": "^2.3.3",
-    "ua-parser-js": "^1.0.37",
     "uuid": "^9.0.1"
   },
   "devDependencies": {
@@ -30,7 +29,6 @@
     "@types/express": "^4.17.21",
     "@types/ramda": "^0.29.6",
     "@types/sqlstring": "^2.3.2",
-    "@types/ua-parser-js": "^0.7.39",
     "@types/uuid": "^9.0.8",
     "tsup": "^7.2.0",
     "typescript": "^5.2.2"

pnpm-lock.yaml

@@ -121,6 +121,9 @@ importers:
       '@openpanel/tsconfig':
         specifier: workspace:*
         version: link:../../tooling/typescript
+      '@types/js-yaml':
+        specifier: ^4.0.9
+        version: 4.0.9
       '@types/jsonwebtoken':
         specifier: ^9.0.6
         version: 9.0.6
@@ -139,6 +142,9 @@ importers:
       '@types/ws':
         specifier: ^8.5.10
         version: 8.5.10
+      js-yaml:
+        specifier: ^4.1.0
+        version: 4.1.0
       tsup:
         specifier: ^7.2.0
         version: 7.3.0(typescript@5.3.3)
@@ -8444,7 +8450,6 @@ packages:
 
   /@types/js-yaml@4.0.9:
     resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==}
-    dev: false
 
   /@types/json5@0.0.29:
     resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==}
@@ -9059,7 +9064,6 @@ packages:
 
   /argparse@2.0.1:
     resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
-    dev: false
 
   /aria-hidden@1.2.3:
     resolution: {integrity: sha512-xcLxITLe2HYa1cnYnwCjkOO1PqUHQpozB8x9AR0OgWN2woOBi5kSDVxKfd0b7sb1hw5qFeJhXm9H1nu3xSfLeQ==}
@@ -13946,7 +13950,6 @@ packages:
     hasBin: true
     dependencies:
       argparse: 2.0.1
-    dev: false
 
   /jsc-android@250231.0.0:
     resolution: {integrity: sha512-rS46PvsjYmdmuz1OAWXY/1kCYG7pnf1TBqeTiOJr1iDz7s5DLxxC9n/ZMknLDxzYzNVfI7R95MH10emSSG1Wuw==}