zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

wip: try groupmq 2

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-13 06:43:09 +01:00
parent a672b73947
commit 9c3c1458bb
27 changed files with 343 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
packages/db/src/services/clients.service.ts
View File

@@ -1,4 +1,4 @@
import { cacheable, cacheableLru } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import type { Client, Prisma } from '../prisma-client';
import { db } from '../prisma-client';
@@ -34,7 +34,4 @@ export async function getClientById(
});
}
export const getClientByIdCached = cacheableLru(getClientById, {
maxSize: 1000,
ttl: 60 * 5,
});
export const getClientByIdCached = cacheable(getClientById, 60 * 5);

2
packages/db/src/services/notification.service.ts
View File

@@ -90,7 +90,7 @@ export const getNotificationRulesByProjectId = cacheable(
},
});
},
60 * 24
60 * 24,
);
function getIntegration(integrationId: string | null) {

9
packages/db/src/services/project.service.ts
View File

@@ -1,6 +1,6 @@
import { cacheable } from '@openpanel/redis';
import sqlstring from 'sqlstring';
import { TABLE_NAMES, chQuery } from '../clickhouse/client';
import { chQuery, TABLE_NAMES } from '../clickhouse/client';
import type { Prisma, Project } from '../prisma-client';
import { db } from '../prisma-client';
@@ -25,6 +25,7 @@ export async function getProjectById(id: string) {
return res;
}
/** L1 LRU (60s) + L2 Redis. clear() invalidates Redis + local LRU; other nodes may serve stale from LRU for up to 60s. */
export const getProjectByIdCached = cacheable(getProjectById, 60 * 60 * 24);
export async function getProjectWithClients(id: string) {
@@ -44,7 +45,7 @@ export async function getProjectWithClients(id: string) {
return res;
}
export async function getProjectsByOrganizationId(organizationId: string) {
export function getProjectsByOrganizationId(organizationId: string) {
return db.project.findMany({
where: {
organizationId,
@@ -95,7 +96,7 @@ export async function getProjects({
if (access.length > 0) {
return projects.filter((project) =>
access.some((a) => a.projectId === project.id),
access.some((a) => a.projectId === project.id)
);
}
@@ -104,7 +105,7 @@ export async function getProjects({
export const getProjectEventsCount = async (projectId: string) => {
const res = await chQuery<{ count: number }>(
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`,
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`
);
return res[0]?.count;
};

9
packages/db/src/services/salt.service.ts
View File

@@ -1,9 +1,9 @@
import { generateSalt } from '@openpanel/common/server';
import { cacheableLru } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import { db } from '../prisma-client';
export const getSalts = cacheableLru(
export const getSalts = cacheable(
'op:salt',
async () => {
const [curr, prev] = await db.salt.findMany({
@@ -24,10 +24,7 @@ export const getSalts = cacheableLru(
return salts;
},
{
maxSize: 2,
ttl: 60 * 5,
},
60 * 5,
);
export async function createInitialSalts() {

9
packages/queue/src/queues.ts
View File

@@ -6,7 +6,7 @@ import type {
} from '@openpanel/db';
import { createLogger } from '@openpanel/logger';
import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
import { Queue, QueueEvents } from 'bullmq';
import { Queue } from 'bullmq';
import { Queue as GroupQueue } from 'groupmq';
import type { ITrackPayload } from '../../validation';
@@ -66,6 +66,10 @@ export interface EventsQueuePayloadIncomingEvent {
headers: Record<string, string | undefined>;
deviceId: string;
sessionId: string;
session?: Pick<
IServiceCreateEventPayload,
'referrer' | 'referrerName' | 'referrerType'
>;
};
}
export interface EventsQueuePayloadCreateEvent {
@@ -206,9 +210,6 @@ export const sessionsQueue = new Queue<SessionsQueuePayload>(
},
}
);
export const sessionsQueueEvents = new QueueEvents(getQueueName('sessions'), {
connection: getRedisQueue(),
});
export const cronQueue = new Queue<CronQueuePayload>(getQueueName('cron'), {
connection: getRedisQueue(),

249
packages/redis/cachable.ts
View File

@@ -1,7 +1,7 @@
import { LRUCache } from 'lru-cache';
import { getRedisCache } from './redis';
export const deleteCache = async (key: string) => {
export const deleteCache = (key: string) => {
return getRedisCache().del(key);
};
@@ -15,7 +15,7 @@ export async function getCache<T>(
key: string,
expireInSec: number,
fn: () => Promise<T>,
useLruCache?: boolean,
useLruCache?: boolean
): Promise<T> {
// L1 Cache: Check global LRU cache first (in-memory, instant)
if (useLruCache) {
@@ -28,15 +28,7 @@ export async function getCache<T>(
// L2 Cache: Check Redis cache (shared across instances)
const hit = await getRedisCache().get(key);
if (hit) {
const parsed = JSON.parse(hit, (_, value) => {
if (
typeof value === 'string' &&
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/.test(value)
) {
return new Date(value);
}
return value;
});
const parsed = parseCache(hit);
// Store in LRU cache for next time
if (useLruCache) {
@@ -81,12 +73,24 @@ export function getGlobalLruCacheStats() {
}
function stringify(obj: unknown): string {
if (obj === null) return 'null';
if (obj === undefined) return 'undefined';
if (typeof obj === 'boolean') return obj ? 'true' : 'false';
if (typeof obj === 'number') return String(obj);
if (typeof obj === 'string') return obj;
if (typeof obj === 'function') return obj.toString();
if (obj === null) {
return 'null';
}
if (obj === undefined) {
return 'undefined';
}
if (typeof obj === 'boolean') {
return obj ? 'true' : 'false';
}
if (typeof obj === 'number') {
return String(obj);
}
if (typeof obj === 'string') {
return obj;
}
if (typeof obj === 'function') {
return obj.toString();
}
if (Array.isArray(obj)) {
return `[${obj.map(stringify).join(',')}]`;
@@ -128,17 +132,29 @@ function hasResult(result: unknown): boolean {
return true;
}
export interface CacheableLruOptions {
/** TTL in seconds for LRU cache */
ttl: number;
/** Maximum number of entries in LRU cache */
maxSize?: number;
}
const DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/;
const parseCache = (cached: string) => {
try {
return JSON.parse(cached, (_, value) => {
if (typeof value === 'string' && DATE_REGEX.test(value)) {
return new Date(value);
}
return value;
});
} catch (error) {
console.error('Failed to parse cache', error);
return null;
}
};
// L1 cache: short TTL to offload Redis; clear() invalidates Redis, other nodes may serve stale from LRU for up to this long
const CACHEABLE_LRU_TTL_MS = 60 * 1000; // 60 seconds
const CACHEABLE_LRU_MAX = 1000;
// Overload 1: cacheable(fn, expireInSec)
export function cacheable<T extends (...args: any) => any>(
fn: T,
expireInSec: number,
expireInSec: number
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => Promise<number>;
@@ -151,7 +167,7 @@ export function cacheable<T extends (...args: any) => any>(
export function cacheable<T extends (...args: any) => any>(
name: string,
fn: T,
expireInSec: number,
expireInSec: number
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => Promise<number>;
@@ -164,7 +180,7 @@ export function cacheable<T extends (...args: any) => any>(
export function cacheable<T extends (...args: any) => any>(
fnOrName: T | string,
fnOrExpireInSec: number | T,
_expireInSec?: number,
_expireInSec?: number
) {
const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
const fn =
@@ -195,184 +211,67 @@ export function cacheable<T extends (...args: any) => any>(
const cachePrefix = `cachable:${name}`;
const getKey = (...args: Parameters<T>) =>
`${cachePrefix}:${stringify(args)}`;
`${cachePrefix}:${stringify(args)}`.replaceAll(/\s/g, '');
// Redis-only mode: asynchronous implementation
const lruCache = new LRUCache<string, any>({
max: CACHEABLE_LRU_MAX,
ttl: CACHEABLE_LRU_TTL_MS,
});
// L1 LRU (60s) + L2 Redis. clear() deletes Redis + local LRU; other nodes may serve stale from LRU for up to 60s.
const cachedFn = async (
...args: Parameters<T>
): Promise<Awaited<ReturnType<T>>> => {
const key = getKey(...args);
// Check Redis cache (shared across instances)
// L1: in-memory LRU first (offloads Redis on hot keys)
const lruHit = lruCache.get(key);
if (lruHit !== undefined && hasResult(lruHit)) {
return lruHit as Awaited<ReturnType<T>>;
}
// L2: Redis (shared across instances)
const cached = await getRedisCache().get(key);
if (cached) {
try {
const parsed = JSON.parse(cached, (_, value) => {
if (
typeof value === 'string' &&
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/.test(value)
) {
return new Date(value);
}
return value;
});
if (hasResult(parsed)) {
return parsed;
}
} catch (e) {
console.error('Failed to parse cache', e);
const parsed = parseCache(cached);
if (hasResult(parsed)) {
lruCache.set(key, parsed);
return parsed;
}
}
// Cache miss: Execute function
// Cache miss: execute function
const result = await fn(...(args as any));
if (hasResult(result)) {
// Don't await Redis write - fire and forget for better performance
lruCache.set(key, result);
getRedisCache()
.setex(key, expireInSec, JSON.stringify(result))
.catch(() => {});
.catch(() => {
// ignore error
});
}
return result;
};
cachedFn.getKey = getKey;
cachedFn.clear = async (...args: Parameters<T>) => {
const key = getKey(...args);
return getRedisCache().del(key);
};
cachedFn.set =
(...args: Parameters<T>) =>
async (payload: Awaited<ReturnType<T>>) => {
const key = getKey(...args);
return getRedisCache()
.setex(key, expireInSec, JSON.stringify(payload))
.catch(() => {});
};
return cachedFn;
}
// Overload 1: cacheableLru(fn, options)
export function cacheableLru<T extends (...args: any) => any>(
fn: T,
options: CacheableLruOptions,
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => boolean;
set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
};
// Overload 2: cacheableLru(name, fn, options)
export function cacheableLru<T extends (...args: any) => any>(
name: string,
fn: T,
options: CacheableLruOptions,
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => boolean;
set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
};
// Implementation for cacheableLru (LRU-only - synchronous)
export function cacheableLru<T extends (...args: any) => any>(
fnOrName: T | string,
fnOrOptions: T | CacheableLruOptions,
_options?: CacheableLruOptions,
) {
const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
const fn =
typeof fnOrName === 'function'
? fnOrName
: typeof fnOrOptions === 'function'
? fnOrOptions
: null;
let options: CacheableLruOptions;
// Parse parameters based on function signature
if (typeof fnOrName === 'function') {
// Overload 1: cacheableLru(fn, options)
options =
typeof fnOrOptions === 'object' && fnOrOptions !== null
? fnOrOptions
: ({} as CacheableLruOptions);
} else {
// Overload 2: cacheableLru(name, fn, options)
options =
typeof _options === 'object' && _options !== null
? _options
: ({} as CacheableLruOptions);
}
if (typeof fn !== 'function') {
throw new Error('fn is not a function');
}
if (typeof options.ttl !== 'number') {
throw new Error('options.ttl is required and must be a number');
}
const cachePrefix = `cachable:${name}`;
const getKey = (...args: Parameters<T>) =>
`${cachePrefix}:${stringify(args)}`;
const maxSize = options.maxSize ?? 1000;
const ttl = options.ttl;
// Create function-specific LRU cache
const functionLruCache = new LRUCache<string, any>({
max: maxSize,
ttl: ttl * 1000, // Convert seconds to milliseconds for LRU
});
// LRU-only mode: synchronous implementation (or returns promise if fn is async)
const cachedFn = ((...args: Parameters<T>): ReturnType<T> => {
const key = getKey(...args);
// Check LRU cache
const lruHit = functionLruCache.get(key);
if (lruHit !== undefined && hasResult(lruHit)) {
return lruHit as ReturnType<T>;
}
// Cache miss: Execute function
const result = fn(...(args as any)) as ReturnType<T>;
// If result is a Promise, handle it asynchronously but cache the resolved value
if (result && typeof (result as any).then === 'function') {
return (result as Promise<any>).then((resolved: any) => {
if (hasResult(resolved)) {
functionLruCache.set(key, resolved);
}
return resolved;
}) as ReturnType<T>;
}
// Synchronous result: cache and return
if (hasResult(result)) {
functionLruCache.set(key, result);
}
return result;
}) as T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => boolean;
set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
};
cachedFn.getKey = getKey;
cachedFn.clear = (...args: Parameters<T>) => {
const key = getKey(...args);
return functionLruCache.delete(key);
lruCache.delete(key);
return getRedisCache().del(key);
};
cachedFn.set =
(...args: Parameters<T>) =>
(payload: ReturnType<T>) => {
(payload: Awaited<ReturnType<T>>) => {
const key = getKey(...args);
if (hasResult(payload)) {
functionLruCache.set(key, payload);
lruCache.set(key, payload);
return getRedisCache()
.setex(key, expireInSec, JSON.stringify(payload))
.catch(() => {
// ignore error
});
}
};

13
packages/sdks/react-native/index.ts
View File

@@ -1,13 +1,13 @@
import type { OpenPanelOptions, TrackProperties } from '@openpanel/sdk';
import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
import * as Application from 'expo-application';
import Constants from 'expo-constants';
import { AppState, Platform } from 'react-native';
import type { OpenPanelOptions, TrackProperties } from '@openpanel/sdk';
import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
export * from '@openpanel/sdk';
export class OpenPanel extends OpenPanelBase {
private lastPath = '';
constructor(public options: OpenPanelOptions) {
super({
...options,
@@ -37,7 +37,12 @@ export class OpenPanel extends OpenPanelBase {
});
}
public screenView(route: string, properties?: TrackProperties): void {
track(name: string, properties?: TrackProperties) {
return super.track(name, { ...properties, __path: this.lastPath });
}
screenView(route: string, properties?: TrackProperties): void {
this.lastPath = route;
super.track('screen_view', {
...properties,
__path: route,

21
packages/sdks/web/src/index.ts
View File

@@ -58,7 +58,7 @@ export type OpenPanelOptions = OpenPanelBaseOptions & {
function toCamelCase(str: string) {
return str.replace(/([-_][a-z])/gi, ($1) =>
$1.toUpperCase().replace('-', '').replace('_', ''),
$1.toUpperCase().replace('-', '').replace('_', '')
);
}
@@ -114,7 +114,9 @@ export class OpenPanel extends OpenPanelBase {
const sampled = Math.random() < sampleRate;
if (sampled) {
this.loadReplayModule().then((mod) => {
if (!mod) return;
if (!mod) {
return;
}
mod.startReplayRecorder(this.options.sessionReplay!, (chunk) => {
// Replay chunks go through send() and are queued when disabled or waitForProfile
// until ready() is called (base SDK also queues replay until sessionId is set).
@@ -153,7 +155,10 @@ export class OpenPanel extends OpenPanelBase {
// dead-code-eliminated in the library build.
if (typeof __OPENPANEL_REPLAY_URL__ !== 'undefined') {
const scriptEl = _replayScriptRef;
const url = this.options.sessionReplay?.scriptUrl || scriptEl?.src?.replace('.js', '-replay.js') || 'https://openpanel.dev/op1-replay.js';
const url =
this.options.sessionReplay?.scriptUrl ||
scriptEl?.src?.replace('.js', '-replay.js') ||
'https://openpanel.dev/op1-replay.js';
// Already loaded (e.g. user included the script manually)
if ((window as any).__openpanel_replay) {
@@ -287,11 +292,15 @@ export class OpenPanel extends OpenPanelBase {
});
}
track(name: string, properties?: TrackProperties) {
return super.track(name, { ...properties, __path: this.lastPath });
}
screenView(properties?: TrackProperties): void;
screenView(path: string, properties?: TrackProperties): void;
screenView(
pathOrProperties?: string | TrackProperties,
propertiesOrUndefined?: TrackProperties,
propertiesOrUndefined?: TrackProperties
): void {
if (this.isServer()) {
return;
@@ -322,7 +331,7 @@ export class OpenPanel extends OpenPanelBase {
async flushRevenue() {
const promises = this.pendingRevenues.map((pending) =>
super.revenue(pending.amount, pending.properties),
super.revenue(pending.amount, pending.properties)
);
await Promise.all(promises);
this.clearRevenue();
@@ -343,7 +352,7 @@ export class OpenPanel extends OpenPanelBase {
try {
sessionStorage.setItem(
'openpanel-pending-revenues',
JSON.stringify(this.pendingRevenues),
JSON.stringify(this.pendingRevenues)
);
} catch {}
}

4
packages/trpc/src/routers/project.ts
View File

@@ -96,9 +96,7 @@ export const projectRouter = createTRPCRouter({
});
await Promise.all([
getProjectByIdCached.clear(input.id),
res.clients.map((client) => {
getClientByIdCached.clear(client.id);
}),
...res.clients.map((client) => getClientByIdCached.clear(client.id)),
]);
return res;
}),