chore(api): update bots and referrers
This commit is contained in:
Carl-Gerhard Lindesvärd
@@ -6,6 +6,8 @@
|
||||
"testing": "API_PORT=3333 pnpm dev",
|
||||
"start": "node dist/index.js",
|
||||
"build": "rm -rf dist && tsup",
|
||||
"gen:referrers": "jiti scripts/get-referrers.ts && biome format --write src/referrers/index.ts",
|
||||
"gen:bots": "jiti scripts/get-bots.ts && biome format --write src/bots/bots.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -16,8 +18,8 @@
|
||||
"@fastify/websocket": "^8.3.1",
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/logger": "workspace:*",
|
||||
"@openpanel/integrations": "workspace:^",
|
||||
"@openpanel/logger": "workspace:*",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"@openpanel/trpc": "workspace:*",
|
||||
@@ -33,7 +35,6 @@
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^1.13.3",
|
||||
"svix": "^1.24.0",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"url-metadata": "^4.1.0",
|
||||
"uuid": "^9.0.1",
|
||||
"zod": "^3.22.4"
|
||||
@@ -42,13 +43,14 @@
|
||||
"@faker-js/faker": "^9.0.1",
|
||||
"@openpanel/sdk": "workspace:*",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/js-yaml": "^4.0.9",
|
||||
"@types/jsonwebtoken": "^9.0.6",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/request-ip": "^0.0.41",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"@types/ua-parser-js": "^0.7.39",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"@types/ws": "^8.5.10",
|
||||
"js-yaml": "^4.1.0",
|
||||
"tsup": "^7.2.0",
|
||||
"typescript": "^5.2.2"
|
||||
}
|
||||
|
||||
29
apps/api/scripts/get-bots.ts
Normal file
29
apps/api/scripts/get-bots.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
|
||||
async function main() {
|
||||
// Get document, or throw exception on error
|
||||
try {
|
||||
const data = await fetch(
|
||||
'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
|
||||
).then((res) => res.text());
|
||||
|
||||
fs.writeFileSync(
|
||||
path.resolve(__dirname, '../src/bots/bots.ts'),
|
||||
[
|
||||
'// This file is generated by the script get-bots.ts',
|
||||
'',
|
||||
'// The data is fetch from device-detector https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
|
||||
'',
|
||||
`const bots = ${JSON.stringify(yaml.load(data))} as const;`,
|
||||
'export default bots;',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -1,189 +0,0 @@
|
||||
// import { clerkClient } from '@clerk/fastify';
|
||||
|
||||
import { db } from '@openpanel/db';
|
||||
|
||||
// import { db } from '@openpanel/db';
|
||||
|
||||
// type Fn<T = unknown> = (args: { limit: number; offset: number }) => Promise<{
|
||||
// data: T[];
|
||||
// totalCount: number;
|
||||
// }>;
|
||||
|
||||
// function getAllDataByPagination<T extends Fn>(
|
||||
// cb: T
|
||||
// ): Promise<Awaited<ReturnType<T>>['data']> {
|
||||
// const data: Awaited<ReturnType<T>>['data'] = [];
|
||||
// async function getData(page = 0) {
|
||||
// console.log(`getData with offset ${page * 100}`);
|
||||
// const response = await cb({
|
||||
// limit: 100,
|
||||
// offset: page * 100,
|
||||
// });
|
||||
// if (response.data.length !== 0) {
|
||||
// data.push(...response.data);
|
||||
// await getData(page + 1);
|
||||
// }
|
||||
// await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
// }
|
||||
|
||||
// return getData().then(() => data);
|
||||
// }
|
||||
|
||||
// async function main() {
|
||||
// const organizations = await getAllDataByPagination(
|
||||
// clerkClient.organizations.getOrganizationList.bind(
|
||||
// clerkClient.organizations
|
||||
// )
|
||||
// );
|
||||
// const users = await getAllDataByPagination(
|
||||
// clerkClient.users.getUserList.bind(clerkClient.users)
|
||||
// );
|
||||
|
||||
// console.log(`Found ${organizations.length} organizations`);
|
||||
// console.log(`Found ${users.length} users`);
|
||||
|
||||
// for (const user of users.slice(-10)) {
|
||||
// const email = user.primaryEmailAddress?.emailAddress;
|
||||
// console.log('Check', email);
|
||||
|
||||
// try {
|
||||
// if (email) {
|
||||
// const exists = await db.user.findUnique({
|
||||
// where: {
|
||||
// id: user.id,
|
||||
// },
|
||||
// });
|
||||
|
||||
// if (exists) {
|
||||
// console.log('already exists');
|
||||
// } else {
|
||||
// await db.user.create({
|
||||
// data: {
|
||||
// id: user.id,
|
||||
// email: email,
|
||||
// firstName: user.firstName,
|
||||
// lastName: user.lastName,
|
||||
// },
|
||||
// });
|
||||
// }
|
||||
// } else {
|
||||
// console.log('No email?', user);
|
||||
// }
|
||||
// } catch (e) {
|
||||
// console.log('ERROR');
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// console.dir(user, { depth: null });
|
||||
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// }
|
||||
// }
|
||||
|
||||
// for (const org of organizations.slice(-20)) {
|
||||
// try {
|
||||
// if (org.slug) {
|
||||
// const exists = await db.organization.findUnique({
|
||||
// where: {
|
||||
// id: org.slug,
|
||||
// },
|
||||
// });
|
||||
|
||||
// if (exists) {
|
||||
// console.log('already exists org');
|
||||
// } else {
|
||||
// const clerkOrgMembers =
|
||||
// await clerkClient.organizations.getOrganizationMembershipList({
|
||||
// organizationId: org.id,
|
||||
// });
|
||||
|
||||
// const members = clerkOrgMembers.data.map((member) => {
|
||||
// const user = users.find(
|
||||
// (u) => u.id === member.publicUserData?.userId
|
||||
// );
|
||||
// return {
|
||||
// userId: member.publicUserData?.userId,
|
||||
// role: member.role,
|
||||
// email: user!.primaryEmailAddress!.emailAddress,
|
||||
// };
|
||||
// });
|
||||
|
||||
// await db.organization.create({
|
||||
// data: {
|
||||
// id: org.slug,
|
||||
// name: org.name,
|
||||
// createdBy: {
|
||||
// connect: {
|
||||
// id: org.createdBy,
|
||||
// },
|
||||
// },
|
||||
// members: {
|
||||
// create: members,
|
||||
// },
|
||||
// },
|
||||
// });
|
||||
|
||||
// const invites =
|
||||
// await clerkClient.organizations.getOrganizationInvitationList({
|
||||
// organizationId: org.id,
|
||||
// status: ['pending'],
|
||||
// });
|
||||
|
||||
// for (const invite of invites.data) {
|
||||
// await db.member.create({
|
||||
// data: {
|
||||
// email: invite.emailAddress,
|
||||
// organizationId: org.slug,
|
||||
// role: invite.role,
|
||||
// userId: null,
|
||||
// meta: {
|
||||
// access: invite.publicMetadata?.access as string[],
|
||||
// invitationId: invite.id,
|
||||
// },
|
||||
// },
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// console.log('org does not have any slug', org);
|
||||
// }
|
||||
// } catch (e) {
|
||||
// console.log('ERROR');
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// console.dir(org, { depth: null });
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// }
|
||||
// }
|
||||
|
||||
// process.exit(0);
|
||||
// }
|
||||
|
||||
// main();
|
||||
|
||||
async function main() {
|
||||
const organization = await db.organization.findUnique({
|
||||
where: {
|
||||
id: 'openpanel-dev',
|
||||
members: {
|
||||
some: {
|
||||
userId: 'user_2cEoI8b1SuEFbZERGEAyVvC676F',
|
||||
},
|
||||
},
|
||||
},
|
||||
include: {
|
||||
members: {
|
||||
select: {
|
||||
role: true,
|
||||
user: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
console.dir(organization, { depth: null });
|
||||
}
|
||||
main();
|
||||
@@ -1,65 +0,0 @@
|
||||
import { TABLE_NAMES, ch, chQuery } from '@openpanel/db';
|
||||
|
||||
async function main() {
|
||||
const projects = await chQuery(
|
||||
`SELECT distinct project_id FROM ${TABLE_NAMES.events} ORDER BY project_id`,
|
||||
);
|
||||
const withOrigin = [];
|
||||
|
||||
for (const project of projects) {
|
||||
try {
|
||||
const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([
|
||||
await chQuery(
|
||||
`SELECT * FROM ${TABLE_NAMES.events} WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`,
|
||||
),
|
||||
await chQuery(
|
||||
`SELECT * FROM ${TABLE_NAMES.events} WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`,
|
||||
),
|
||||
]);
|
||||
|
||||
if (eventWithOrigin[0] && eventWithoutOrigin[0]) {
|
||||
console.log(`Project ${project.project_id} as events without origin`);
|
||||
console.log(`- Origin: ${eventWithOrigin[0].origin}`);
|
||||
withOrigin.push(project.project_id);
|
||||
const events = await chQuery(
|
||||
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`,
|
||||
);
|
||||
console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`);
|
||||
await ch.command({
|
||||
query: `ALTER TABLE events UPDATE origin = '${eventWithOrigin[0].origin}' WHERE project_id = '${project.project_id}' AND path != '' AND origin = '';`,
|
||||
clickhouse_settings: {
|
||||
wait_end_of_query: 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (!eventWithOrigin[0] && eventWithoutOrigin[0]) {
|
||||
console.log(
|
||||
`😧 Project ${project.project_id} has no events with origin (last event ${eventWithoutOrigin[0].created_at})`,
|
||||
);
|
||||
console.log('- NO ORIGIN');
|
||||
}
|
||||
|
||||
if (!eventWithOrigin[0] && !eventWithoutOrigin[0]) {
|
||||
console.log(
|
||||
`🔥 WARNING: Project ${project.project_id} has no events at all?!?!?!`,
|
||||
);
|
||||
}
|
||||
|
||||
if (eventWithOrigin[0] && !eventWithoutOrigin[0]) {
|
||||
console.log(
|
||||
`✅ Project ${project.project_id} has all events with origin!!!`,
|
||||
);
|
||||
}
|
||||
console.log('');
|
||||
console.log('');
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
} catch (e) {
|
||||
console.log('🥵 ERROR ORRROR');
|
||||
console.log('Error for project', project.project_id);
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
main();
|
||||
File diff suppressed because it is too large
Load Diff
@@ -22,7 +22,6 @@
|
||||
"prom-client": "^15.1.3",
|
||||
"ramda": "^0.29.1",
|
||||
"sqlstring": "^2.3.3",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"uuid": "^9.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -30,7 +29,6 @@
|
||||
"@types/express": "^4.17.21",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"@types/ua-parser-js": "^0.7.39",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"tsup": "^7.2.0",
|
||||
"typescript": "^5.2.2"
|
||||
|
||||
9
pnpm-lock.yaml
generated
9
pnpm-lock.yaml
generated
@@ -121,6 +121,9 @@ importers:
|
||||
'@openpanel/tsconfig':
|
||||
specifier: workspace:*
|
||||
version: link:../../tooling/typescript
|
||||
'@types/js-yaml':
|
||||
specifier: ^4.0.9
|
||||
version: 4.0.9
|
||||
'@types/jsonwebtoken':
|
||||
specifier: ^9.0.6
|
||||
version: 9.0.6
|
||||
@@ -139,6 +142,9 @@ importers:
|
||||
'@types/ws':
|
||||
specifier: ^8.5.10
|
||||
version: 8.5.10
|
||||
js-yaml:
|
||||
specifier: ^4.1.0
|
||||
version: 4.1.0
|
||||
tsup:
|
||||
specifier: ^7.2.0
|
||||
version: 7.3.0(typescript@5.3.3)
|
||||
@@ -8444,7 +8450,6 @@ packages:
|
||||
|
||||
/@types/js-yaml@4.0.9:
|
||||
resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==}
|
||||
dev: false
|
||||
|
||||
/@types/json5@0.0.29:
|
||||
resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==}
|
||||
@@ -9059,7 +9064,6 @@ packages:
|
||||
|
||||
/argparse@2.0.1:
|
||||
resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
|
||||
dev: false
|
||||
|
||||
/aria-hidden@1.2.3:
|
||||
resolution: {integrity: sha512-xcLxITLe2HYa1cnYnwCjkOO1PqUHQpozB8x9AR0OgWN2woOBi5kSDVxKfd0b7sb1hw5qFeJhXm9H1nu3xSfLeQ==}
|
||||
@@ -13946,7 +13950,6 @@ packages:
|
||||
hasBin: true
|
||||
dependencies:
|
||||
argparse: 2.0.1
|
||||
dev: false
|
||||
|
||||
/jsc-android@250231.0.0:
|
||||
resolution: {integrity: sha512-rS46PvsjYmdmuz1OAWXY/1kCYG7pnf1TBqeTiOJr1iDz7s5DLxxC9n/ZMknLDxzYzNVfI7R95MH10emSSG1Wuw==}
|
||||
|
||||
Reference in New Issue
Block a user