zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-10 10:09:59 +01:00
parent 37246f57f0
commit bb0e413b06
9 changed files with 76 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
apps/api/src/bots/index.ts
View File

@@ -1,3 +1,4 @@
import { cacheable } from '@openpanel/redis';
import bots from './bots';
// Pre-compile regex patterns at module load time
@@ -14,59 +15,31 @@ const compiledBots = bots.map((bot) => {
const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
const includesBots = compiledBots.filter((bot) => 'includes' in bot);
// Common legitimate browser patterns - if UA matches these, it's very likely a real browser
// This provides ultra-fast early exit for ~95% of real traffic
const legitimateBrowserPatterns = [
'Mozilla/5.0', // Nearly all modern browsers
'Chrome/', // Chrome/Chromium browsers
'Safari/', // Safari and Chrome-based browsers
'Firefox/', // Firefox
'Edg/', // Edge
];
const mobilePatterns = ['iPhone', 'Android', 'iPad'];
const desktopOSPatterns = ['Windows NT', 'Macintosh', 'X11; Linux'];
export function isBot(ua: string) {
// Ultra-fast early exit: check if this looks like a legitimate browser
// Real browsers typically have Mozilla/5.0 + browser name + OS
if (ua.includes('Mozilla/5.0')) {
// Check for browser signature
const hasBrowser = legitimateBrowserPatterns.some((pattern) =>
ua.includes(pattern),
);
// Check for OS signature (mobile or desktop)
const hasOS =
mobilePatterns.some((pattern) => ua.includes(pattern)) ||
desktopOSPatterns.some((pattern) => ua.includes(pattern));
// If it has Mozilla/5.0, a known browser, and an OS, it's very likely legitimate
if (hasBrowser && hasOS) {
return null;
export const isBot = cacheable(
'is-bot',
(ua: string) => {
// Check simple string patterns first (fast)
for (const bot of includesBots) {
if (ua.includes(bot.includes)) {
return {
name: bot.name,
type: 'category' in bot ? bot.category : 'Unknown',
};
}
}
}
// Check simple string patterns first (fast)
for (const bot of includesBots) {
if (ua.includes(bot.includes)) {
return {
name: bot.name,
type: 'category' in bot ? bot.category : 'Unknown',
};
// Check regex patterns (slower)
for (const bot of regexBots) {
if (bot.compiledRegex.test(ua)) {
return {
name: bot.name,
type: 'category' in bot ? bot.category : 'Unknown',
};
}
}
}
// Check regex patterns (slower)
for (const bot of regexBots) {
if (bot.compiledRegex.test(ua)) {
return {
name: bot.name,
type: 'category' in bot ? bot.category : 'Unknown',
};
}
}
return null;
}
return null;
},
60 * 60, // 1 hour
'lru',
);

22
apps/api/src/controllers/track.controller.ts
View File

@@ -1,14 +1,11 @@
import type { FastifyReply, FastifyRequest } from 'fastify';
import { assocPath, pathOr, pick } from 'ramda';
import { logger } from '@/utils/logger';
import { generateId } from '@openpanel/common';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import type { ILogger } from '@openpanel/logger';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import type {
DecrementPayload,
IdentifyPayload,
@@ -241,25 +238,6 @@ async function track({
const jobId = [payload.name, timestamp, projectId, currentDeviceId, groupId]
.filter(Boolean)
.join('-');
await getRedisCache().incr('track:counter');
log('track handler', {
jobId: jobId,
groupId: groupId,
timestamp: timestamp,
data: {
projectId,
headers,
event: {
...payload,
timestamp,
isTimestampFromThePast,
},
uaInfo,
geo,
currentDeviceId,
previousDeviceId,
},
});
await getEventsGroupQueueShard(groupId).add({
orderMs: timestamp,
data: {

2
apps/api/src/routes/track.router.ts
View File

@@ -6,9 +6,9 @@ import { duplicateHook } from '@/hooks/duplicate.hook';
import { isBotHook } from '@/hooks/is-bot.hook';
const trackRouter: FastifyPluginCallback = async (fastify) => {
fastify.addHook('preHandler', isBotHook);
fastify.addHook('preValidation', duplicateHook);
fastify.addHook('preHandler', clientHook);
fastify.addHook('preHandler', isBotHook);
fastify.route({
method: 'POST',

17
apps/worker/src/boot-workers.ts
View File

@@ -216,10 +216,10 @@ export async function bootWorkers() {
(worker as Worker).on('failed', (job) => {
if (job) {
if (job.processedOn && job.finishedOn) {
const duration = job.finishedOn - job.processedOn;
const elapsed = job.finishedOn - job.processedOn;
eventsGroupJobDuration.observe(
{ queue_shard: worker.name, status: 'failed' },
duration,
{ name: worker.name, status: 'failed' },
elapsed,
);
}
logger.error('job failed', {
@@ -235,10 +235,15 @@ export async function bootWorkers() {
(worker as Worker).on('completed', (job) => {
if (job) {
if (job.processedOn && job.finishedOn) {
const duration = job.finishedOn - job.processedOn;
const elapsed = job.finishedOn - job.processedOn;
logger.info('job completed', {
jobId: job.id,
worker: worker.name,
elapsed,
});
eventsGroupJobDuration.observe(
{ queue_shard: worker.name, status: 'success' },
duration,
{ name: worker.name, status: 'success' },
elapsed,
);
}
}

6
apps/worker/src/metrics.ts
View File

@@ -16,9 +16,9 @@ const queues = [sessionsQueue, cronQueue, ...eventsGroupQueues];
// Histogram to track job processing time for eventsGroupQueues
export const eventsGroupJobDuration = new client.Histogram({
name: 'events_group_job_duration_ms',
help: 'Duration of job processing in eventsGroupQueues (in ms)',
labelNames: ['queue_shard', 'status'],
name: 'job_duration_ms',
help: 'Duration of job processing (in ms)',
labelNames: ['name', 'status'],
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
});

6
packages/db/src/services/clients.service.ts
View File

@@ -34,4 +34,8 @@ export async function getClientById(
});
}
export const getClientByIdCached = cacheable(getClientById, 60 * 60 * 24, true);
export const getClientByIdCached = cacheable(
getClientById,
60 * 60 * 24,
'both',
);

6
packages/db/src/services/event.service.ts
View File

@@ -340,7 +340,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
sdk_version: payload.sdkVersion ?? '',
};
await Promise.all([sessionBuffer.add(event), eventBuffer.add(event)]);
const promises = [sessionBuffer.add(event), eventBuffer.add(event)];
if (payload.profileId) {
const profile: IServiceUpsertProfile = {
@@ -371,10 +371,12 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
profile.isExternal ||
(profile.isExternal === false && payload.name === 'session_start')
) {
await upsertProfile(profile, true);
promises.push(upsertProfile(profile, true));
}
}
await Promise.all(promises);
return {
document: event,
};

4
packages/db/src/services/salt.service.ts
View File

@@ -1,6 +1,6 @@
import { generateSalt } from '@openpanel/common/server';
import { cacheable, getRedisCache } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import { db } from '../prisma-client';
export async function getCurrentSalt() {
@@ -43,7 +43,7 @@ export const getSalts = cacheable(
return salts;
},
60 * 10,
true,
'both',
);
export async function createInitialSalts() {

43
packages/redis/cachable.ts
View File

@@ -128,11 +128,13 @@ function hasResult(result: unknown): boolean {
return true;
}
type CacheMode = 'lru' | 'redis' | 'both';
// Overload 1: cacheable(fn, expireInSec, lruCache?)
export function cacheable<T extends (...args: any) => any>(
fn: T,
expireInSec: number,
lruCache?: boolean,
cacheMode?: CacheMode,
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => Promise<number>;
@@ -146,7 +148,7 @@ export function cacheable<T extends (...args: any) => any>(
name: string,
fn: T,
expireInSec: number,
lruCache?: boolean,
cacheMode?: CacheMode,
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => Promise<number>;
@@ -159,8 +161,8 @@ export function cacheable<T extends (...args: any) => any>(
export function cacheable<T extends (...args: any) => any>(
fnOrName: T | string,
fnOrExpireInSec: number | T,
_expireInSecOrLruCache?: number | boolean,
_lruCache?: boolean,
_expireInSecOrCacheMode?: number | CacheMode,
_cacheMode?: CacheMode,
) {
const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
const fn =
@@ -171,23 +173,23 @@ export function cacheable<T extends (...args: any) => any>(
: null;
let expireInSec: number | null = null;
let useLruCache = false;
let cacheMode = 'redis';
// Parse parameters based on function signature
if (typeof fnOrName === 'function') {
// Overload 1: cacheable(fn, expireInSec, lruCache?)
expireInSec = typeof fnOrExpireInSec === 'number' ? fnOrExpireInSec : null;
useLruCache =
typeof _expireInSecOrLruCache === 'boolean'
? _expireInSecOrLruCache
: false;
cacheMode =
typeof _expireInSecOrCacheMode === 'boolean'
? _expireInSecOrCacheMode
: 'redis';
} else {
// Overload 2: cacheable(name, fn, expireInSec, lruCache?)
expireInSec =
typeof _expireInSecOrLruCache === 'number'
? _expireInSecOrLruCache
typeof _expireInSecOrCacheMode === 'number'
? _expireInSecOrCacheMode
: null;
useLruCache = typeof _lruCache === 'boolean' ? _lruCache : false;
cacheMode = typeof _cacheMode === 'string' ? _cacheMode : 'redis';
}
if (typeof fn !== 'function') {
@@ -203,12 +205,13 @@ export function cacheable<T extends (...args: any) => any>(
`${cachePrefix}:${stringify(args)}`;
// Create function-specific LRU cache if enabled
const functionLruCache = useLruCache
? new LRUCache<string, any>({
max: 1000,
ttl: expireInSec * 1000, // Convert seconds to milliseconds for LRU
})
: null;
const functionLruCache =
cacheMode === 'lru' || cacheMode === 'both'
? new LRUCache<string, any>({
max: 1000,
ttl: expireInSec * 1000, // Convert seconds to milliseconds for LRU
})
: null;
const cachedFn = async (
...args: Parameters<T>
@@ -221,6 +224,10 @@ export function cacheable<T extends (...args: any) => any>(
if (lruHit !== undefined && hasResult(lruHit)) {
return lruHit;
}
if (cacheMode === 'lru') {
return null as any;
}
}
// L2 Cache: Check Redis cache (shared across instances)