zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

chore(root): migrate to biome

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-09-16 12:20:40 +02:00
parent 1f6e198336
commit 32e91959f6
383 changed files with 1943 additions and 3085 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/worker/Dockerfile
View File

@@ -29,8 +29,6 @@ COPY packages/queue/package.json ./packages/queue/
COPY packages/logger/package.json ./packages/logger/
COPY packages/common/package.json ./packages/common/
COPY packages/constants/package.json ./packages/constants/
# Patches
COPY patches ./patches
# BUILD
FROM base AS build

17
apps/worker/package.json
View File

@@ -6,8 +6,6 @@
"testing": "WORKER_PORT=9999 pnpm dev",
"start": "node dist/index.js",
"build": "rm -rf dist && tsup",
"lint": "eslint .",
"format": "prettier --check \"**/*.{mjs,ts,md,json}\"",
"typecheck": "tsc --noEmit"
},
"dependencies": {
@@ -27,24 +25,13 @@
"uuid": "^9.0.1"
},
"devDependencies": {
"@openpanel/eslint-config": "workspace:*",
"@openpanel/prettier-config": "workspace:*",
"@openpanel/tsconfig": "workspace:*",
"@types/express": "^4.17.21",
"@types/ramda": "^0.29.6",
"@types/sqlstring": "^2.3.2",
"@types/ua-parser-js": "^0.7.39",
"@types/uuid": "^9.0.8",
"eslint": "^8.48.0",
"prettier": "^3.0.3",
"tsup": "^7.2.0",
"typescript": "^5.2.2"
},
"eslintConfig": {
"root": true,
"extends": [
"@openpanel/eslint-config/base"
]
},
"prettier": "@openpanel/prettier-config"
}
}
}

150
apps/worker/scripts/debug.ts
View File

@@ -1,150 +0,0 @@
import { escape } from 'sqlstring';
import type { IClickhouseEvent } from '@openpanel/db';
import { chQuery, eventBuffer, TABLE_NAMES } from '@openpanel/db';
import { sessionsQueue } from '@openpanel/queue/src/queues';
import { getRedisCache, getRedisQueue } from '@openpanel/redis';
async function debugStalledEvents() {
const keys = await getRedisQueue().keys('bull:sessions:sessionEnd*');
const delayedZRange = await getRedisQueue().zrange(
'bull:sessions:delayed',
0,
-1,
'WITHSCORES'
);
const delayedValues = delayedZRange.reduce(
(acc, item, index, array) => {
if (index % 2 === 0) {
acc[item] = Number(array[index + 1]) / 0x1000;
}
return acc;
},
{} as Record<string, number>
);
const opKeys = await getRedisCache().keys('op:*');
const stalledEvents = await getRedisCache().lrange(
'op:buffer:events:stalled',
0,
-1
);
// keys.forEach((key) => {
// console.log(key);
// });
// console.log('--------------------');
const queue = await eventBuffer.getQueue(-1);
queue
.filter((item) => item.event.name === 'screen_view')
.forEach((item) => {
const date = new Date(item.event.created_at.replace(' ', 'T') + 'Z');
const match = keys.find((key) => {
return item.event.device_id && key.includes(item.event.device_id);
});
if (match) {
// console.log(
// date.toISOString(),
// item.event.name,
// item.event.device,
// item.event.session_id,
// item.event.profile_id,
// item.event.device_id,
// match
// );
} else {
console.log(
'NO MATCH FOUND!',
date.toISOString(),
item.event.name,
'[SID]',
item.event.session_id,
'[PID]',
item.event.profile_id,
'[DID]',
item.event.device_id
);
console.log(item.event);
console.log('');
// console.log('Not in queue!');
// log§
}
});
if (stalledEvents.length > 0) {
const res = await chQuery(
`SELECT * FROM ${TABLE_NAMES.events} WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})`
);
stalledEvents.forEach((item) => {
const event = JSON.parse(item) as IClickhouseEvent;
const date = new Date(event.created_at.replace(' ', 'T') + 'Z');
console.log(
'STALLED!',
date.toISOString(),
event.name,
'[IN_DB]',
res.find((item) => item.id === event.id) ? 'YES' : 'NO',
'[ID]',
event.id,
'[SID]',
event.session_id,
'[PID]',
event.profile_id,
'[DID]',
event.device_id
);
// console.log(event);
});
}
console.log('OP Keys', opKeys);
console.log('Queue', queue.length);
console.log('Session Ends', keys.length);
console.log('Stalled Events', stalledEvents.length);
// keys.forEach((key) => {
// if (key.includes('e1b233e69bcd2132ec7bf343004d4b01')) {
// console.log(key);
// }
// });
const delayedJobs = await sessionsQueue.getDelayed();
console.log('delayedJobs', delayedJobs.length);
delayedJobs.sort((a, b) => a.timestamp + a.delay - (b.timestamp + b.delay));
let delayedJobsCount = 0;
delayedJobs.forEach((job) => {
if (!job.id) {
return;
}
const timestamp = delayedValues[job.id];
if (!timestamp) {
return;
}
const date = new Date(timestamp);
// if date is in the past
// if (date.getTime() - 1000 * 60 * 5 < Date.now()) {
if (date.getTime() < Date.now()) {
delayedJobsCount++;
console.log(
date.toLocaleString('sv-SE'),
'https://op.coderax.se/worker/queue/sessions/' +
encodeURIComponent(job.id)
);
}
});
console.log('delayedJobsCount', delayedJobsCount);
}
async function main() {
if (process.argv[2] === 'stalled') {
await debugStalledEvents();
}
process.exit(0);
}
main();

22
apps/worker/src/index.ts
View File

@@ -15,14 +15,14 @@ import { sessionsJob } from './jobs/sessions';
import { register } from './metrics';
import { logger } from './utils/logger';
const PORT = parseInt(process.env.WORKER_PORT || '3000', 10);
const PORT = Number.parseInt(process.env.WORKER_PORT || '3000', 10);
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath('/');
const app = express();
const workerOptions: WorkerOptions = {
connection: getRedisQueue(),
concurrency: parseInt(process.env.CONCURRENCY || '1', 10),
concurrency: Number.parseInt(process.env.CONCURRENCY || '1', 10),
};
async function start() {
@@ -30,7 +30,7 @@ async function start() {
const sessionsWorker = new Worker(
sessionsQueue.name,
sessionsJob,
workerOptions
workerOptions,
);
const cronWorker = new Worker(cronQueue.name, cronJob, workerOptions);
@@ -110,7 +110,9 @@ async function start() {
});
}
process.exit(isNaN(+evtOrExitCodeOrError) ? 1 : +evtOrExitCodeOrError);
process.exit(
Number.isNaN(+evtOrExitCodeOrError) ? 1 : +evtOrExitCodeOrError,
);
}
[
@@ -132,7 +134,7 @@ async function start() {
].forEach((evt) =>
process.on(evt, (evt) => {
exitHandler(evt);
})
}),
);
await cronQueue.add(
@@ -147,7 +149,7 @@ async function start() {
utc: true,
pattern: '0 0 * * *',
},
}
},
);
await cronQueue.add(
@@ -160,10 +162,10 @@ async function start() {
jobId: 'flushEvents',
repeat: {
every: process.env.BATCH_INTERVAL
? parseInt(process.env.BATCH_INTERVAL, 10)
? Number.parseInt(process.env.BATCH_INTERVAL, 10)
: 1000 * 10,
},
}
},
);
await cronQueue.add(
@@ -177,7 +179,7 @@ async function start() {
repeat: {
every: 2 * 1000,
},
}
},
);
if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {
@@ -192,7 +194,7 @@ async function start() {
repeat: {
pattern: '0 0 * * *',
},
}
},
);
}

4
apps/worker/src/jobs/cron.ping.ts
View File

@@ -1,8 +1,8 @@
import { chQuery, TABLE_NAMES } from '@openpanel/db';
import { TABLE_NAMES, chQuery } from '@openpanel/db';
export async function ping() {
const [res] = await chQuery<{ count: number }>(
`SELECT COUNT(*) as count FROM ${TABLE_NAMES.events}`
`SELECT COUNT(*) as count FROM ${TABLE_NAMES.events}`,
);
if (typeof res?.count === 'number') {

2
apps/worker/src/jobs/cron.salt.ts
View File

@@ -1,4 +1,4 @@
import { generateSalt } from '@openpanel/common';
import { generateSalt } from '@openpanel/common/server';
import { db, getCurrentSalt } from '@openpanel/db';
export async function salt() {

10
apps/worker/src/jobs/events.create-session-end.ts
View File

@@ -3,19 +3,19 @@ import { last } from 'ramda';
import { getTime } from '@openpanel/common';
import {
TABLE_NAMES,
createEvent,
eventBuffer,
getEvents,
TABLE_NAMES,
} from '@openpanel/db';
import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';
export async function createSessionEnd(
job: Job<EventsQueuePayloadCreateSessionEnd>
job: Job<EventsQueuePayloadCreateSessionEnd>,
) {
const payload = job.data.payload;
const eventsInBuffer = await eventBuffer.findMany(
(item) => item.event.session_id === payload.sessionId
(item) => item.event.session_id === payload.sessionId,
);
const sql = `
@@ -39,7 +39,7 @@ export async function createSessionEnd(
const eventsInDb = await getEvents(sql);
// sort last inserted first
const events = [...eventsInBuffer, ...eventsInDb].sort(
(a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()
(a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
);
events.map((event, index) => {
@@ -51,7 +51,7 @@ export async function createSessionEnd(
`DeviceId: ${event.deviceId}`,
`Profile: ${event.profileId}`,
`Path: ${event.path}`,
].join('\n')
].join('\n'),
);
});

10
apps/worker/src/jobs/events.incoming-event.ts
View File

@@ -128,7 +128,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
{
delay: SESSION_END_TIMEOUT,
jobId: sessionEndJobId,
}
},
);
}
@@ -178,7 +178,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
function getSessionEndWithPriority(
priority: boolean,
count = 0
count = 0,
): typeof getSessionEnd {
return async (args) => {
const res = await getSessionEnd(args);
@@ -208,13 +208,13 @@ async function getSessionEnd({
previousDeviceId: string;
}) {
const sessionEndKeys = await getRedisQueue().keys(
`*:sessionEnd:${projectId}:*`
`*:sessionEnd:${projectId}:*`,
);
const sessionEndJobCurrentDeviceId = await findJobByPrefix(
sessionsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${currentDeviceId}:`
`sessionEnd:${projectId}:${currentDeviceId}:`,
);
if (sessionEndJobCurrentDeviceId) {
return { deviceId: currentDeviceId, job: sessionEndJobCurrentDeviceId };
@@ -223,7 +223,7 @@ async function getSessionEnd({
const sessionEndJobPreviousDeviceId = await findJobByPrefix(
sessionsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${previousDeviceId}:`
`sessionEnd:${projectId}:${previousDeviceId}:`,
);
if (sessionEndJobPreviousDeviceId) {
return { deviceId: previousDeviceId, job: sessionEndJobPreviousDeviceId };

6
apps/worker/src/jobs/events.ts
View File

@@ -1,7 +1,7 @@
import type { Job } from 'bullmq';
import { escape } from 'sqlstring';
import { chQuery, db, TABLE_NAMES } from '@openpanel/db';
import { TABLE_NAMES, chQuery, db } from '@openpanel/db';
import type {
EventsQueuePayload,
EventsQueuePayloadCreateSessionEnd,
@@ -24,7 +24,7 @@ export async function eventsJob(job: Job<EventsQueuePayload>) {
}
return await createSessionEnd(
job as Job<EventsQueuePayloadCreateSessionEnd>
job as Job<EventsQueuePayloadCreateSessionEnd>,
);
}
}
@@ -32,7 +32,7 @@ export async function eventsJob(job: Job<EventsQueuePayload>) {
async function updateEventsCount(projectId: string) {
const res = await chQuery<{ count: number }>(
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`,
);
const count = res[0]?.count;
if (count) {

16
apps/worker/src/metrics.ts
View File

@@ -18,7 +18,7 @@ queues.forEach((queue) => {
const metric = await queue.getActiveCount();
this.set(metric);
},
})
}),
);
register.registerMetric(
@@ -29,7 +29,7 @@ queues.forEach((queue) => {
const metric = await queue.getDelayedCount();
this.set(metric);
},
})
}),
);
register.registerMetric(
@@ -40,7 +40,7 @@ queues.forEach((queue) => {
const metric = await queue.getFailedCount();
this.set(metric);
},
})
}),
);
register.registerMetric(
@@ -51,7 +51,7 @@ queues.forEach((queue) => {
const metric = await queue.getCompletedCount();
this.set(metric);
},
})
}),
);
register.registerMetric(
@@ -62,7 +62,7 @@ queues.forEach((queue) => {
const metric = await queue.getWaitingCount();
this.set(metric);
},
})
}),
);
});
@@ -78,7 +78,7 @@ buffers.forEach((buffer) => {
const metric = await getRedisCache().llen(`op:buffer:${buffer}`);
this.set(metric);
},
})
}),
);
register.registerMetric(
@@ -87,10 +87,10 @@ buffers.forEach((buffer) => {
help: 'Number of users in the users array',
async collect() {
const metric = await getRedisCache().llen(
`op:buffer:${buffer}:stalled`
`op:buffer:${buffer}:stalled`,
);
this.set(metric);
},
})
}),
);
});

4
apps/worker/src/utils/parse-referrer.ts
View File

@@ -26,7 +26,7 @@ export function parseReferrer(url: string | undefined) {
}
export function getReferrerWithQuery(
query: Record<string, string> | undefined
query: Record<string, string> | undefined,
) {
if (!query) {
return null;
@@ -40,7 +40,7 @@ export function getReferrerWithQuery(
const match =
Object.values(referrers).find(
(referrer) => referrer.name.toLowerCase() === source.toLowerCase()
(referrer) => referrer.name.toLowerCase() === source.toLowerCase(),
) || referrers[source];
if (match) {

8
apps/worker/src/utils/parse-user-agent.ts
View File

@@ -71,7 +71,7 @@ const userAgentServerList = [
function isServer(userAgent: string) {
const match = userAgentServerList.some((server) =>
userAgent.toLowerCase().includes(server.toLowerCase())
userAgent.toLowerCase().includes(server.toLowerCase()),
);
if (match) {
return true;
@@ -83,15 +83,15 @@ function isServer(userAgent: string) {
export function getDevice(ua: string) {
const mobile1 =
/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(
ua
ua,
);
const mobile2 =
/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test(
ua.slice(0, 4)
ua.slice(0, 4),
);
const tablet =
/tablet|ipad|android(?!.*mobile)|xoom|sch-i800|kindle|silk|playbook/i.test(
ua
ua,
);
if (mobile1 || mobile2) {