zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: optimize event buffer (#278)

Browse Source 
* fix: how we fetch profiles in the buffer

* perf: optimize event buffer

* remove unused file

* fix

* wip

* wip: try groupmq 2

* try simplified event buffer with duration calculation on the fly instead
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-16 13:29:40 +01:00
committed by GitHub
parent 4736f8509d
commit 4483e464d1
46 changed files with 887 additions and 1841 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/package.json
View File

@@ -30,7 +30,6 @@
"@openpanel/logger": "workspace:*",
"@openpanel/payments": "workspace:*",
"@openpanel/queue": "workspace:*",
"groupmq": "catalog:",
"@openpanel/redis": "workspace:*",
"@openpanel/trpc": "workspace:*",
"@openpanel/validation": "workspace:*",
@@ -40,6 +39,7 @@
"fastify": "^5.6.1",
"fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0",
"groupmq": "catalog:",
"jsonwebtoken": "^9.0.2",
"ramda": "^0.29.1",
"sharp": "^0.33.5",

9
apps/api/src/bots/index.ts
View File

@@ -1,4 +1,4 @@
import { cacheable, cacheableLru } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import bots from './bots';
// Pre-compile regex patterns at module load time
@@ -15,7 +15,7 @@ const compiledBots = bots.map((bot) => {
const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
const includesBots = compiledBots.filter((bot) => 'includes' in bot);
export const isBot = cacheableLru(
export const isBot = cacheable(
'is-bot',
(ua: string) => {
// Check simple string patterns first (fast)
@@ -40,8 +40,5 @@ export const isBot = cacheableLru(
return null;
},
{
maxSize: 1000,
ttl: 60 * 5,
},
60 * 5
);

95
apps/api/src/controllers/live.controller.ts
View File

@@ -1,12 +1,5 @@
import type { FastifyRequest } from 'fastify';
import superjson from 'superjson';
import type { WebSocket } from '@fastify/websocket';
import {
eventBuffer,
getProfileById,
transformMinimalEvent,
} from '@openpanel/db';
import { eventBuffer } from '@openpanel/db';
import { setSuperJson } from '@openpanel/json';
import {
psubscribeToPublishedEvent,
@@ -14,10 +7,7 @@ import {
} from '@openpanel/redis';
import { getProjectAccess } from '@openpanel/trpc';
import { getOrganizationAccess } from '@openpanel/trpc/src/access';
export function getLiveEventInfo(key: string) {
return key.split(':').slice(2) as [string, string];
}
import type { FastifyRequest } from 'fastify';
export function wsVisitors(
socket: WebSocket,
@@ -25,27 +15,38 @@ export function wsVisitors(
Params: {
projectId: string;
};
}>,
}>
) {
const { params } = req;
const unsubscribe = subscribeToPublishedEvent('events', 'saved', (event) => {
if (event?.projectId === params.projectId) {
eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
const sendCount = () => {
eventBuffer
.getActiveVisitorCount(params.projectId)
.then((count) => {
socket.send(String(count));
})
.catch(() => {
socket.send('0');
});
};
const unsubscribe = subscribeToPublishedEvent(
'events',
'batch',
({ projectId }) => {
if (projectId === params.projectId) {
sendCount();
}
}
});
);
const punsubscribe = psubscribeToPublishedEvent(
'__keyevent@0__:expired',
(key) => {
const [projectId] = getLiveEventInfo(key);
if (projectId && projectId === params.projectId) {
eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
socket.send(String(count));
});
const [, , projectId] = key.split(':');
if (projectId === params.projectId) {
sendCount();
}
},
}
);
socket.on('close', () => {
@@ -62,18 +63,10 @@ export async function wsProjectEvents(
};
Querystring: {
token?: string;
type?: 'saved' | 'received';
};
}>,
}>
) {
const { params, query } = req;
const type = query.type || 'saved';
if (!['saved', 'received'].includes(type)) {
socket.send('Invalid type');
socket.close();
return;
}
const { params } = req;
const userId = req.session?.userId;
if (!userId) {
@@ -87,24 +80,20 @@ export async function wsProjectEvents(
projectId: params.projectId,
});
if (!access) {
socket.send('No access');
socket.close();
return;
}
const unsubscribe = subscribeToPublishedEvent(
'events',
type,
async (event) => {
if (event.projectId === params.projectId) {
const profile = await getProfileById(event.profileId, event.projectId);
socket.send(
superjson.stringify(
access
? {
...event,
profile,
}
: transformMinimalEvent(event),
),
);
'batch',
({ projectId, count }) => {
if (projectId === params.projectId) {
socket.send(setSuperJson({ count }));
}
},
}
);
socket.on('close', () => unsubscribe());
@@ -116,7 +105,7 @@ export async function wsProjectNotifications(
Params: {
projectId: string;
};
}>,
}>
) {
const { params } = req;
const userId = req.session?.userId;
@@ -143,9 +132,9 @@ export async function wsProjectNotifications(
'created',
(notification) => {
if (notification.projectId === params.projectId) {
socket.send(superjson.stringify(notification));
socket.send(setSuperJson(notification));
}
},
}
);
socket.on('close', () => unsubscribe());
@@ -157,7 +146,7 @@ export async function wsOrganizationEvents(
Params: {
organizationId: string;
};
}>,
}>
) {
const { params } = req;
const userId = req.session?.userId;
@@ -184,7 +173,7 @@ export async function wsOrganizationEvents(
'subscription_updated',
(message) => {
socket.send(setSuperJson(message));
},
}
);
socket.on('close', () => unsubscribe());

42
apps/api/src/controllers/manage.controller.ts
View File

@@ -1,5 +1,4 @@
import crypto from 'node:crypto';
import { HttpError } from '@/utils/errors';
import { stripTrailingSlash } from '@openpanel/common';
import { hashPassword } from '@openpanel/common/server';
import {
@@ -10,6 +9,7 @@ import {
} from '@openpanel/db';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { z } from 'zod';
import { HttpError } from '@/utils/errors';
// Validation schemas
const zCreateProject = z.object({
@@ -57,7 +57,7 @@ const zUpdateReference = z.object({
// Projects CRUD
export async function listProjects(
request: FastifyRequest,
reply: FastifyReply,
reply: FastifyReply
) {
const projects = await db.project.findMany({
where: {
@@ -74,7 +74,7 @@ export async function listProjects(
export async function getProject(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const project = await db.project.findFirst({
where: {
@@ -92,7 +92,7 @@ export async function getProject(
export async function createProject(
request: FastifyRequest<{ Body: z.infer<typeof zCreateProject> }>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zCreateProject.safeParse(request.body);
@@ -139,12 +139,9 @@ export async function createProject(
},
});
// Clear cache
await Promise.all([
getProjectByIdCached.clear(project.id),
project.clients.map((client) => {
getClientByIdCached.clear(client.id);
}),
...project.clients.map((client) => getClientByIdCached.clear(client.id)),
]);
reply.send({
@@ -165,7 +162,7 @@ export async function updateProject(
Params: { id: string };
Body: z.infer<typeof zUpdateProject>;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zUpdateProject.safeParse(request.body);
@@ -223,12 +220,9 @@ export async function updateProject(
data: updateData,
});
// Clear cache
await Promise.all([
getProjectByIdCached.clear(project.id),
existing.clients.map((client) => {
getClientByIdCached.clear(client.id);
}),
...existing.clients.map((client) => getClientByIdCached.clear(client.id)),
]);
reply.send({ data: project });
@@ -236,7 +230,7 @@ export async function updateProject(
export async function deleteProject(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const project = await db.project.findFirst({
where: {
@@ -266,7 +260,7 @@ export async function deleteProject(
// Clients CRUD
export async function listClients(
request: FastifyRequest<{ Querystring: { projectId?: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const where: any = {
organizationId: request.client!.organizationId,
@@ -300,7 +294,7 @@ export async function listClients(
export async function getClient(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const client = await db.client.findFirst({
where: {
@@ -318,7 +312,7 @@ export async function getClient(
export async function createClient(
request: FastifyRequest<{ Body: z.infer<typeof zCreateClient> }>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zCreateClient.safeParse(request.body);
@@ -374,7 +368,7 @@ export async function updateClient(
Params: { id: string };
Body: z.infer<typeof zUpdateClient>;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zUpdateClient.safeParse(request.body);
@@ -417,7 +411,7 @@ export async function updateClient(
export async function deleteClient(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const client = await db.client.findFirst({
where: {
@@ -444,7 +438,7 @@ export async function deleteClient(
// References CRUD
export async function listReferences(
request: FastifyRequest<{ Querystring: { projectId?: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const where: any = {};
@@ -488,7 +482,7 @@ export async function listReferences(
export async function getReference(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const reference = await db.reference.findUnique({
where: {
@@ -516,7 +510,7 @@ export async function getReference(
export async function createReference(
request: FastifyRequest<{ Body: z.infer<typeof zCreateReference> }>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zCreateReference.safeParse(request.body);
@@ -559,7 +553,7 @@ export async function updateReference(
Params: { id: string };
Body: z.infer<typeof zUpdateReference>;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zUpdateReference.safeParse(request.body);
@@ -616,7 +610,7 @@ export async function updateReference(
export async function deleteReference(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const reference = await db.reference.findUnique({
where: {

33
apps/api/src/controllers/track.controller.ts
View File

@@ -7,7 +7,10 @@ import {
upsertProfile,
} from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import {
type EventsQueuePayloadIncomingEvent,
getEventsGroupQueueShard,
} from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import {
type IDecrementPayload,
@@ -112,6 +115,7 @@ interface TrackContext {
identity?: IIdentifyPayload;
deviceId: string;
sessionId: string;
session?: EventsQueuePayloadIncomingEvent['payload']['session'];
geo: GeoLocation;
}
@@ -141,19 +145,21 @@ async function buildContext(
validatedBody.payload.profileId = profileId;
}
const overrideDeviceId =
validatedBody.type === 'track' &&
typeof validatedBody.payload?.properties?.__deviceId === 'string'
? validatedBody.payload?.properties.__deviceId
: undefined;
// Get geo location (needed for track and identify)
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
const { deviceId, sessionId } = await getDeviceId({
const deviceIdResult = await getDeviceId({
projectId,
ip,
ua,
salts,
overrideDeviceId:
validatedBody.type === 'track' &&
typeof validatedBody.payload?.properties?.__deviceId === 'string'
? validatedBody.payload?.properties.__deviceId
: undefined,
overrideDeviceId,
});
return {
@@ -166,8 +172,9 @@ async function buildContext(
isFromPast: timestamp.isTimestampFromThePast,
},
identity,
deviceId,
sessionId,
deviceId: deviceIdResult.deviceId,
sessionId: deviceIdResult.sessionId,
session: deviceIdResult.session,
geo,
};
}
@@ -176,13 +183,14 @@ async function handleTrack(
payload: ITrackPayload,
context: TrackContext
): Promise<void> {
const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;
const { projectId, deviceId, geo, headers, timestamp, sessionId, session } =
context;
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
const groupId = uaInfo.isServer
? payload.profileId
? `${projectId}:${payload.profileId}`
: `${projectId}:${generateId()}`
: undefined
: deviceId;
const jobId = [
slug(payload.name),
@@ -203,7 +211,7 @@ async function handleTrack(
}
promises.push(
getEventsGroupQueueShard(groupId).add({
getEventsGroupQueueShard(groupId || generateId()).add({
orderMs: timestamp.value,
data: {
projectId,
@@ -217,6 +225,7 @@ async function handleTrack(
geo,
deviceId,
sessionId,
session,
},
groupId,
jobId,

9
apps/api/src/hooks/is-bot.hook.ts
View File

@@ -1,20 +1,19 @@
import { isBot } from '@/bots';
import { createBotEvent } from '@openpanel/db';
import type {
DeprecatedPostEventPayload,
ITrackHandlerPayload,
} from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { isBot } from '@/bots';
export async function isBotHook(
req: FastifyRequest<{
Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const bot = req.headers['user-agent']
? isBot(req.headers['user-agent'])
? await isBot(req.headers['user-agent'])
: null;
if (bot && req.client?.projectId) {
@@ -44,6 +43,6 @@ export async function isBotHook(
}
}
return reply.status(202).send();
return reply.status(202).send({ bot });
}
}

5
apps/api/src/routes/track.router.ts
View File

@@ -1,6 +1,5 @@
import { fetchDeviceId, handler } from '@/controllers/track.controller';
import type { FastifyPluginCallback } from 'fastify';
import { fetchDeviceId, handler } from '@/controllers/track.controller';
import { clientHook } from '@/hooks/client.hook';
import { duplicateHook } from '@/hooks/duplicate.hook';
import { isBotHook } from '@/hooks/is-bot.hook';
@@ -13,7 +12,7 @@ const trackRouter: FastifyPluginCallback = async (fastify) => {
fastify.route({
method: 'POST',
url: '/',
handler: handler,
handler,
});
fastify.route({

41
apps/api/src/utils/ids.ts
View File

@@ -1,7 +1,12 @@
import crypto from 'node:crypto';
import { generateDeviceId } from '@openpanel/common/server';
import { getSafeJson } from '@openpanel/json';
import type {
EventsQueuePayloadCreateSessionEnd,
EventsQueuePayloadIncomingEvent,
} from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import { pick } from 'ramda';
export async function getDeviceId({
projectId,
@@ -37,14 +42,20 @@ export async function getDeviceId({
ua,
});
return await getDeviceIdFromSession({
return await getInfoFromSession({
projectId,
currentDeviceId,
previousDeviceId,
});
}
async function getDeviceIdFromSession({
interface DeviceIdResult {
deviceId: string;
sessionId: string;
session?: EventsQueuePayloadIncomingEvent['payload']['session'];
}
async function getInfoFromSession({
projectId,
currentDeviceId,
previousDeviceId,
@@ -52,7 +63,7 @@ async function getDeviceIdFromSession({
projectId: string;
currentDeviceId: string;
previousDeviceId: string;
}) {
}): Promise<DeviceIdResult> {
try {
const multi = getRedisCache().multi();
multi.hget(
@@ -65,21 +76,33 @@ async function getDeviceIdFromSession({
);
const res = await multi.exec();
if (res?.[0]?.[1]) {
const data = getSafeJson<{ payload: { sessionId: string } }>(
const data = getSafeJson<EventsQueuePayloadCreateSessionEnd>(
(res?.[0]?.[1] as string) ?? ''
);
if (data) {
const sessionId = data.payload.sessionId;
return { deviceId: currentDeviceId, sessionId };
return {
deviceId: currentDeviceId,
sessionId: data.payload.sessionId,
session: pick(
['referrer', 'referrerName', 'referrerType'],
data.payload
),
};
}
}
if (res?.[1]?.[1]) {
const data = getSafeJson<{ payload: { sessionId: string } }>(
const data = getSafeJson<EventsQueuePayloadCreateSessionEnd>(
(res?.[1]?.[1] as string) ?? ''
);
if (data) {
const sessionId = data.payload.sessionId;
return { deviceId: previousDeviceId, sessionId };
return {
deviceId: previousDeviceId,
sessionId: data.payload.sessionId,
session: pick(
['referrer', 'referrerName', 'referrerType'],
data.payload
),
};
}
}
} catch (error) {

141
apps/start/src/components/events/event-list-item.tsx
View File

@@ -1,25 +1,20 @@
import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
import { Link } from '@tanstack/react-router';
import { SerieIcon } from '../report-chart/common/serie-icon';
import { EventIcon } from './event-icon';
import { Tooltiper } from '@/components/ui/tooltip';
import { useAppParams } from '@/hooks/use-app-params';
import { useNumber } from '@/hooks/use-numer-formatter';
import { pushModal } from '@/modals';
import { cn } from '@/utils/cn';
import { getProfileName } from '@/utils/getters';
import { Link } from '@tanstack/react-router';
import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
import { SerieIcon } from '../report-chart/common/serie-icon';
import { EventIcon } from './event-icon';
type EventListItemProps = IServiceEventMinimal | IServiceEvent;
export function EventListItem(props: EventListItemProps) {
const { organizationId, projectId } = useAppParams();
const { createdAt, name, path, duration, meta } = props;
const { createdAt, name, path, meta } = props;
const profile = 'profile' in props ? props.profile : null;
const number = useNumber();
const renderName = () => {
if (name === 'screen_view') {
if (path.includes('/')) {
@@ -32,83 +27,65 @@ export function EventListItem(props: EventListItemProps) {
return name.replace(/_/g, ' ');
};
const renderDuration = () => {
if (name === 'screen_view') {
return (
<span className="text-muted-foreground">
{number.shortWithUnit(duration / 1000, 'min')}
</span>
);
}
return null;
};
const isMinimal = 'minimal' in props;
return (
<>
<button
type="button"
onClick={() => {
if (!isMinimal) {
pushModal('EventDetails', {
id: props.id,
projectId,
createdAt,
});
}
}}
className={cn(
'card hover:bg-light-background flex w-full items-center justify-between rounded-lg p-4 transition-colors',
meta?.conversion &&
`bg-${meta.color}-50 dark:bg-${meta.color}-900 hover:bg-${meta.color}-100 dark:hover:bg-${meta.color}-700`,
)}
>
<div>
<div className="flex items-center gap-4 text-left ">
<EventIcon size="sm" name={name} meta={meta} />
<span>
<span className="font-medium">{renderName()}</span>
{' '}
{renderDuration()}
</span>
</div>
<div className="pl-10">
<div className="flex origin-left scale-75 gap-1">
<SerieIcon name={props.country} />
<SerieIcon name={props.os} />
<SerieIcon name={props.browser} />
</div>
<button
className={cn(
'card flex w-full items-center justify-between rounded-lg p-4 transition-colors hover:bg-light-background',
meta?.conversion &&
`bg-${meta.color}-50 dark:bg-${meta.color}-900 hover:bg-${meta.color}-100 dark:hover:bg-${meta.color}-700`
)}
onClick={() => {
if (!isMinimal) {
pushModal('EventDetails', {
id: props.id,
projectId,
createdAt,
});
}
}}
type="button"
>
<div>
<div className="flex items-center gap-4 text-left">
<EventIcon meta={meta} name={name} size="sm" />
<span className="font-medium">{renderName()}</span>
</div>
<div className="pl-10">
<div className="flex origin-left scale-75 gap-1">
<SerieIcon name={props.country} />
<SerieIcon name={props.os} />
<SerieIcon name={props.browser} />
</div>
</div>
<div className="flex gap-4">
{profile && (
<Tooltiper asChild content={getProfileName(profile)}>
<Link
onClick={(e) => {
e.stopPropagation();
}}
to={'/$organizationId/$projectId/profiles/$profileId'}
params={{
organizationId,
projectId,
profileId: profile.id,
}}
className="max-w-[80px] overflow-hidden text-ellipsis whitespace-nowrap text-muted-foreground hover:underline"
>
{getProfileName(profile)}
</Link>
</Tooltiper>
)}
<Tooltiper asChild content={createdAt.toLocaleString()}>
<div className=" text-muted-foreground">
{createdAt.toLocaleTimeString()}
</div>
</div>
<div className="flex gap-4">
{profile && (
<Tooltiper asChild content={getProfileName(profile)}>
<Link
className="max-w-[80px] overflow-hidden text-ellipsis whitespace-nowrap text-muted-foreground hover:underline"
onClick={(e) => {
e.stopPropagation();
}}
params={{
organizationId,
projectId,
profileId: profile.id,
}}
to={'/$organizationId/$projectId/profiles/$profileId'}
>
{getProfileName(profile)}
</Link>
</Tooltiper>
</div>
</button>
</>
)}
<Tooltiper asChild content={createdAt.toLocaleString()}>
<div className="text-muted-foreground">
{createdAt.toLocaleTimeString()}
</div>
</Tooltiper>
</div>
</button>
);
}

37
apps/start/src/components/events/event-listener.tsx
View File

@@ -1,3 +1,4 @@
import { AnimatedNumber } from '../animated-number';
import {
Tooltip,
TooltipContent,
@@ -8,71 +9,53 @@ import { useDebounceState } from '@/hooks/use-debounce-state';
import useWS from '@/hooks/use-ws';
import { cn } from '@/utils/cn';
import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
import { useParams } from '@tanstack/react-router';
import { AnimatedNumber } from '../animated-number';
export default function EventListener({
onRefresh,
}: {
onRefresh: () => void;
}) {
const params = useParams({
strict: false,
});
const { projectId } = useAppParams();
const counter = useDebounceState(0, 1000);
useWS<IServiceEventMinimal | IServiceEvent>(
useWS<{ count: number }>(
`/live/events/${projectId}`,
(event) => {
if (event) {
const isProfilePage = !!params?.profileId;
if (isProfilePage) {
const profile = 'profile' in event ? event.profile : null;
if (profile?.id === params?.profileId) {
counter.set((prev) => prev + 1);
}
return;
}
counter.set((prev) => prev + 1);
}
({ count }) => {
counter.set((prev) => prev + count);
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
},
}
);
return (
<Tooltip>
<TooltipTrigger asChild>
<button
type="button"
className="flex h-8 items-center gap-2 rounded-md border border-border bg-card px-3 font-medium leading-none"
onClick={() => {
counter.set(0);
onRefresh();
}}
className="flex h-8 items-center gap-2 rounded-md border border-border bg-card px-3 font-medium leading-none"
type="button"
>
<div className="relative">
<div
className={cn(
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all'
)}
/>
<div
className={cn(
'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all'
)}
/>
</div>
{counter.debounced === 0 ? (
'Listening'
) : (
<AnimatedNumber value={counter.debounced} suffix=" new events" />
<AnimatedNumber suffix=" new events" value={counter.debounced} />
)}
</button>
</TooltipTrigger>

78
apps/start/src/components/events/table/columns.tsx
View File

@@ -1,15 +1,14 @@
import type { IServiceEvent } from '@openpanel/db';
import type { ColumnDef } from '@tanstack/react-table';
import { ColumnCreatedAt } from '@/components/column-created-at';
import { EventIcon } from '@/components/events/event-icon';
import { ProjectLink } from '@/components/links';
import { ProfileAvatar } from '@/components/profiles/profile-avatar';
import { SerieIcon } from '@/components/report-chart/common/serie-icon';
import { KeyValueGrid } from '@/components/ui/key-value-grid';
import { useNumber } from '@/hooks/use-numer-formatter';
import { pushModal } from '@/modals';
import { getProfileName } from '@/utils/getters';
import type { ColumnDef } from '@tanstack/react-table';
import { ColumnCreatedAt } from '@/components/column-created-at';
import { ProfileAvatar } from '@/components/profiles/profile-avatar';
import { KeyValueGrid } from '@/components/ui/key-value-grid';
import type { IServiceEvent } from '@openpanel/db';
export function useColumns() {
const number = useNumber();
@@ -28,17 +27,24 @@ export function useColumns() {
accessorKey: 'name',
header: 'Name',
cell({ row }) {
const { name, path, duration, properties, revenue } = row.original;
const { name, path, revenue } = row.original;
const fullTitle =
name === 'screen_view'
? path
: name === 'revenue' && revenue
? `${name} (${number.currency(revenue / 100)})`
: name.replace(/_/g, ' ');
const renderName = () => {
if (name === 'screen_view') {
if (path.includes('/')) {
return <span className="max-w-md truncate">{path}</span>;
return path;
}
return (
<>
<span className="text-muted-foreground">Screen: </span>
<span className="max-w-md truncate">{path}</span>
{path}
</>
);
}
@@ -50,38 +56,27 @@ export function useColumns() {
return name.replace(/_/g, ' ');
};
const renderDuration = () => {
if (name === 'screen_view') {
return (
<span className="text-muted-foreground">
{number.shortWithUnit(duration / 1000, 'min')}
</span>
);
}
return null;
};
return (
<div className="flex items-center gap-2">
<div className="flex min-w-0 items-center gap-2">
<button
type="button"
className="transition-transform hover:scale-105"
className="shrink-0 transition-transform hover:scale-105"
onClick={() => {
pushModal('EditEvent', {
id: row.original.id,
});
}}
type="button"
>
<EventIcon
size="sm"
name={row.original.name}
meta={row.original.meta}
name={row.original.name}
size="sm"
/>
</button>
<span className="flex gap-2">
<span className="flex min-w-0 flex-1 gap-2">
<button
type="button"
className="min-w-0 max-w-full truncate text-left font-medium hover:underline"
title={fullTitle}
onClick={() => {
pushModal('EventDetails', {
id: row.original.id,
@@ -89,11 +84,10 @@ export function useColumns() {
projectId: row.original.projectId,
});
}}
className="font-medium hover:underline"
type="button"
>
{renderName()}
<span className="block truncate">{renderName()}</span>
</button>
{renderDuration()}
</span>
</div>
);
@@ -107,8 +101,8 @@ export function useColumns() {
if (profile) {
return (
<ProjectLink
className="group row items-center gap-2 whitespace-nowrap font-medium hover:underline"
href={`/profiles/${encodeURIComponent(profile.id)}`}
className="group whitespace-nowrap font-medium hover:underline row items-center gap-2"
>
<ProfileAvatar size="sm" {...profile} />
{getProfileName(profile)}
@@ -119,8 +113,8 @@ export function useColumns() {
if (profileId && profileId !== deviceId) {
return (
<ProjectLink
href={`/profiles/${encodeURIComponent(profileId)}`}
className="whitespace-nowrap font-medium hover:underline"
href={`/profiles/${encodeURIComponent(profileId)}`}
>
Unknown
</ProjectLink>
@@ -130,8 +124,8 @@ export function useColumns() {
if (deviceId) {
return (
<ProjectLink
href={`/profiles/${encodeURIComponent(deviceId)}`}
className="whitespace-nowrap font-medium hover:underline"
href={`/profiles/${encodeURIComponent(deviceId)}`}
>
Anonymous
</ProjectLink>
@@ -152,10 +146,10 @@ export function useColumns() {
const { sessionId } = row.original;
return (
<ProjectLink
href={`/sessions/${encodeURIComponent(sessionId)}`}
className="whitespace-nowrap font-medium hover:underline"
href={`/sessions/${encodeURIComponent(sessionId)}`}
>
{sessionId.slice(0,6)}
{sessionId.slice(0, 6)}
</ProjectLink>
);
},
@@ -175,7 +169,7 @@ export function useColumns() {
cell({ row }) {
const { country, city } = row.original;
return (
<div className="row items-center gap-2 min-w-0">
<div className="row min-w-0 items-center gap-2">
<SerieIcon name={country} />
<span className="truncate">{city}</span>
</div>
@@ -189,7 +183,7 @@ export function useColumns() {
cell({ row }) {
const { os } = row.original;
return (
<div className="row items-center gap-2 min-w-0">
<div className="row min-w-0 items-center gap-2">
<SerieIcon name={os} />
<span className="truncate">{os}</span>
</div>
@@ -203,7 +197,7 @@ export function useColumns() {
cell({ row }) {
const { browser } = row.original;
return (
<div className="row items-center gap-2 min-w-0">
<div className="row min-w-0 items-center gap-2">
<SerieIcon name={browser} />
<span className="truncate">{browser}</span>
</div>
@@ -221,14 +215,14 @@ export function useColumns() {
const { properties } = row.original;
const filteredProperties = Object.fromEntries(
Object.entries(properties || {}).filter(
([key]) => !key.startsWith('__'),
),
([key]) => !key.startsWith('__')
)
);
const items = Object.entries(filteredProperties);
const limit = 2;
const data = items.slice(0, limit).map(([key, value]) => ({
name: key,
value: value,
value,
}));
if (items.length > limit) {
data.push({

9
apps/start/src/components/events/table/index.tsx
View File

@@ -35,6 +35,7 @@ type Props = {
>,
unknown
>;
showEventListener?: boolean;
};
const LOADING_DATA = [{}, {}, {}, {}, {}, {}, {}, {}, {}] as IServiceEvent[];
@@ -215,7 +216,7 @@ const VirtualizedEventsTable = ({
);
};
export const EventsTable = ({ query }: Props) => {
export const EventsTable = ({ query, showEventListener = false }: Props) => {
const { isLoading } = query;
const columns = useColumns();
@@ -272,7 +273,7 @@ export const EventsTable = ({ query }: Props) => {
return (
<>
<EventsTableToolbar query={query} table={table} />
<EventsTableToolbar query={query} table={table} showEventListener={showEventListener} />
<VirtualizedEventsTable table={table} data={data} isLoading={isLoading} />
<div className="w-full h-10 center-center pt-4" ref={inViewportRef}>
<div
@@ -291,9 +292,11 @@ export const EventsTable = ({ query }: Props) => {
function EventsTableToolbar({
query,
table,
showEventListener,
}: {
query: Props['query'];
table: Table<IServiceEvent>;
showEventListener: boolean;
}) {
const { projectId } = useAppParams();
const [startDate, setStartDate] = useQueryState(
@@ -305,7 +308,7 @@ function EventsTableToolbar({
return (
<DataTableToolbarContainer>
<div className="flex flex-1 flex-wrap items-center gap-2">
<EventListener onRefresh={() => query.refetch()} />
{showEventListener && <EventListener onRefresh={() => query.refetch()} />}
<Button
variant="outline"
size="sm"

32
apps/start/src/components/onboarding/onboarding-verify-listener.tsx
View File

@@ -1,31 +1,13 @@
import type {
IServiceClient,
IServiceEvent,
IServiceProject,
} from '@openpanel/db';
import type { IServiceEvent } from '@openpanel/db';
import { CheckCircle2Icon, CheckIcon, Loader2 } from 'lucide-react';
import { useState } from 'react';
import useWS from '@/hooks/use-ws';
import { cn } from '@/utils/cn';
import { timeAgo } from '@/utils/date';
interface Props {
project: IServiceProject;
client: IServiceClient | null;
events: IServiceEvent[];
onVerified: (verified: boolean) => void;
}
const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
const [events, setEvents] = useState<IServiceEvent[]>(_events ?? []);
useWS<IServiceEvent>(
`/live/events/${client?.projectId}?type=received`,
(data) => {
setEvents((prev) => [...prev, data]);
onVerified(true);
}
);
const VerifyListener = ({ events }: Props) => {
const isConnected = events.length > 0;
const renderIcon = () => {
@@ -49,16 +31,18 @@ const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
<div
className={cn(
'flex gap-6 rounded-xl p-4 md:p-6',
isConnected ? 'bg-emerald-100 dark:bg-emerald-700' : 'bg-blue-500/10'
isConnected
? 'bg-emerald-100 dark:bg-emerald-700/10'
: 'bg-blue-500/10'
)}
>
{renderIcon()}
<div className="flex-1">
<div className="font-semibold text-foreground/90 text-lg leading-normal">
{isConnected ? 'Success' : 'Waiting for events'}
{isConnected ? 'Successfully connected' : 'Waiting for events'}
</div>
{isConnected ? (
<div className="flex flex-col-reverse">
<div className="mt-2 flex flex-col-reverse gap-1">
{events.length > 5 && (
<div className="flex items-center gap-2">
<CheckIcon size={14} />{' '}
@@ -69,7 +53,7 @@ const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
<div className="flex items-center gap-2" key={event.id}>
<CheckIcon size={14} />{' '}
<span className="font-medium">{event.name}</span>{' '}
<span className="ml-auto text-emerald-800">
<span className="ml-auto text-foreground/50 text-sm">
{timeAgo(event.createdAt, 'round')}
</span>
</div>

98
apps/start/src/components/realtime/realtime-active-sessions.tsx
View File

@@ -1,11 +1,9 @@
import { useTRPC } from '@/integrations/trpc/react';
import { useQuery } from '@tanstack/react-query';
import { AnimatePresence, motion } from 'framer-motion';
import { useEffect, useState } from 'react';
import useWS from '@/hooks/use-ws';
import type { IServiceEvent } from '@openpanel/db';
import { EventItem } from '../events/table/item';
import { ProjectLink } from '../links';
import { SerieIcon } from '../report-chart/common/serie-icon';
import { useTRPC } from '@/integrations/trpc/react';
import { formatTimeAgoOrDateTime } from '@/utils/date';
interface RealtimeActiveSessionsProps {
projectId: string;
@@ -17,64 +15,52 @@ export function RealtimeActiveSessions({
limit = 10,
}: RealtimeActiveSessionsProps) {
const trpc = useTRPC();
const activeSessionsQuery = useQuery(
trpc.realtime.activeSessions.queryOptions({
projectId,
}),
const { data: sessions = [] } = useQuery(
trpc.realtime.activeSessions.queryOptions(
{ projectId },
{ refetchInterval: 5000 }
)
);
const [state, setState] = useState<IServiceEvent[]>([]);
// Update state when initial data loads
useEffect(() => {
if (activeSessionsQuery.data && state.length === 0) {
setState(activeSessionsQuery.data);
}
}, [activeSessionsQuery.data, state]);
// Set up WebSocket connection for real-time updates
useWS<IServiceEvent>(
`/live/events/${projectId}`,
(session) => {
setState((prev) => {
// Add new session and remove duplicates, keeping most recent
const filtered = prev.filter((s) => s.id !== session.id);
return [session, ...filtered].slice(0, limit);
});
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
},
);
const sessions = state.length > 0 ? state : (activeSessionsQuery.data ?? []);
return (
<div className="col h-full max-md:hidden">
<div className="hide-scrollbar h-full overflow-y-auto pb-10">
<AnimatePresence mode="popLayout" initial={false}>
<div className="col gap-4">
{sessions.map((session) => (
<div className="col card h-full max-md:hidden">
<div className="hide-scrollbar h-full overflow-y-auto">
<AnimatePresence initial={false} mode="popLayout">
<div className="col divide-y">
{sessions.slice(0, limit).map((session) => (
<motion.div
key={session.id}
layout
// initial={{ opacity: 0, x: -200, scale: 0.8 }}
animate={{ opacity: 1, x: 0, scale: 1 }}
exit={{ opacity: 0, x: 200, scale: 0.8 }}
key={session.id}
layout
transition={{ duration: 0.4, type: 'spring', stiffness: 300 }}
>
<EventItem
event={session}
viewOptions={{
properties: false,
origin: false,
queryString: false,
}}
className="w-full"
/>
<ProjectLink
className="relative block p-4 py-3 pr-14"
href={`/sessions/${session.sessionId}`}
>
<div className="col flex-1 gap-1">
{session.name === 'screen_view' && (
<span className="text-muted-foreground text-xs leading-normal/80">
{session.origin}
</span>
)}
<span className="font-medium text-sm leading-normal">
{session.name === 'screen_view'
? session.path
: session.name}
</span>
<span className="text-muted-foreground text-xs">
{formatTimeAgoOrDateTime(session.createdAt)}
</span>
</div>
<div className="row absolute top-1/2 right-4 origin-right -translate-y-1/2 scale-50 gap-2">
<SerieIcon name={session.referrerName} />
<SerieIcon name={session.os} />
<SerieIcon name={session.browser} />
<SerieIcon name={session.device} />
</div>
</ProjectLink>
</motion.div>
))}
</div>

2
apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.events.tsx
View File

@@ -42,5 +42,5 @@ function Component() {
),
);
return <EventsTable query={query} />;
return <EventsTable query={query} showEventListener />;
}

23
apps/start/src/routes/_app.$organizationId.$projectId.realtime.tsx
View File

@@ -1,3 +1,5 @@
import { keepPreviousData, useQuery } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { Fullscreen, FullscreenClose } from '@/components/fullscreen-toggle';
import RealtimeMap from '@/components/realtime/map';
import { RealtimeActiveSessions } from '@/components/realtime/realtime-active-sessions';
@@ -7,12 +9,10 @@ import { RealtimePaths } from '@/components/realtime/realtime-paths';
import { RealtimeReferrals } from '@/components/realtime/realtime-referrals';
import RealtimeReloader from '@/components/realtime/realtime-reloader';
import { useTRPC } from '@/integrations/trpc/react';
import { PAGE_TITLES, createProjectTitle } from '@/utils/title';
import { keepPreviousData, useQuery } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
export const Route = createFileRoute(
'/_app/$organizationId/$projectId/realtime',
'/_app/$organizationId/$projectId/realtime'
)({
component: Component,
head: () => {
@@ -36,8 +36,8 @@ function Component() {
},
{
placeholderData: keepPreviousData,
},
),
}
)
);
return (
@@ -47,7 +47,7 @@ function Component() {
<RealtimeReloader projectId={projectId} />
<div className="row relative">
<div className="overflow-hidden aspect-[4/2] w-full">
<div className="aspect-[4/2] w-full overflow-hidden">
<RealtimeMap
markers={coordinatesQuery.data ?? []}
sidebarConfig={{
@@ -56,18 +56,17 @@ function Component() {
}}
/>
</div>
<div className="absolute top-8 left-8 bottom-0 col gap-4">
<div className="card p-4 w-72 bg-background/90">
<div className="col absolute top-8 bottom-4 left-8 gap-4">
<div className="card w-72 bg-background/90 p-4">
<RealtimeLiveHistogram projectId={projectId} />
</div>
<div className="w-72 flex-1 min-h-0 relative">
<div className="relative min-h-0 w-72 flex-1">
<RealtimeActiveSessions projectId={projectId} />
<div className="absolute bottom-0 left-0 right-0 h-10 bg-gradient-to-t from-def-100 to-transparent" />
</div>
</div>
</div>
<div className="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4 p-4 pt-4 md:p-8 md:pt-0">
<div className="grid grid-cols-1 gap-4 p-4 pt-4 md:grid-cols-2 md:p-8 md:pt-0 xl:grid-cols-3">
<div>
<RealtimeGeo projectId={projectId} />
</div>

28
apps/start/src/routes/_steps.onboarding.$projectId.verify.tsx
View File

@@ -1,7 +1,6 @@
import { useQuery } from '@tanstack/react-query';
import { createFileRoute, Link, redirect } from '@tanstack/react-router';
import { BoxSelectIcon } from 'lucide-react';
import { useEffect, useState } from 'react';
import { ButtonContainer } from '@/components/button-container';
import { FullPageEmptyState } from '@/components/full-page-empty-state';
import FullPageLoadingState from '@/components/full-page-loading-state';
@@ -33,22 +32,21 @@ export const Route = createFileRoute('/_steps/onboarding/$projectId/verify')({
});
function Component() {
const [isVerified, setIsVerified] = useState(false);
const { projectId } = Route.useParams();
const trpc = useTRPC();
const { data: events, refetch } = useQuery(
trpc.event.events.queryOptions({ projectId })
const { data: events } = useQuery(
trpc.event.events.queryOptions(
{ projectId },
{
refetchInterval: 2500,
}
)
);
const isVerified = events?.data && events.data.length > 0;
const { data: project } = useQuery(
trpc.project.getProjectWithClients.queryOptions({ projectId })
);
useEffect(() => {
if (events && events.data.length > 0) {
setIsVerified(true);
}
}, [events]);
if (!project) {
return (
<FullPageEmptyState icon={BoxSelectIcon} title="Project not found" />
@@ -64,15 +62,7 @@ function Component() {
<div className="flex min-h-0 flex-1 flex-col">
<div className="scrollbar-thin flex-1 overflow-y-auto">
<div className="col gap-8 p-4">
<VerifyListener
client={client}
events={events?.data ?? []}
onVerified={() => {
refetch();
setIsVerified(true);
}}
project={project}
/>
<VerifyListener events={events?.data ?? []} />
<VerifyFaq project={project} />
</div>

2
apps/worker/package.json
View File

@@ -16,11 +16,11 @@
"@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*",
"@openpanel/email": "workspace:*",
"@openpanel/importer": "workspace:*",
"@openpanel/integrations": "workspace:^",
"@openpanel/js-runtime": "workspace:*",
"@openpanel/json": "workspace:*",
"@openpanel/logger": "workspace:*",
"@openpanel/importer": "workspace:*",
"@openpanel/payments": "workspace:*",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*",

52
apps/worker/src/boot-workers.ts
View File

@@ -1,10 +1,9 @@
import type { Queue, WorkerOptions } from 'bullmq';
import { Worker } from 'bullmq';
import { performance } from 'node:perf_hooks';
import { setTimeout as sleep } from 'node:timers/promises';
import {
cronQueue,
EVENTS_GROUP_QUEUES_SHARDS,
type EventsQueuePayloadIncomingEvent,
cronQueue,
eventsGroupQueues,
gscQueue,
importQueue,
@@ -15,14 +14,12 @@ import {
sessionsQueue,
} from '@openpanel/queue';
import { getRedisQueue } from '@openpanel/redis';
import { performance } from 'node:perf_hooks';
import { setTimeout as sleep } from 'node:timers/promises';
import type { Queue, WorkerOptions } from 'bullmq';
import { Worker } from 'bullmq';
import { Worker as GroupWorker } from 'groupmq';
import { cronJob } from './jobs/cron';
import { gscJob } from './jobs/gsc';
import { incomingEvent } from './jobs/events.incoming-event';
import { gscJob } from './jobs/gsc';
import { importJob } from './jobs/import';
import { insightsProjectJob } from './jobs/insights';
import { miscJob } from './jobs/misc';
@@ -95,7 +92,7 @@ function getConcurrencyFor(queueName: string, defaultValue = 1): number {
return defaultValue;
}
export async function bootWorkers() {
export function bootWorkers() {
const enabledQueues = getEnabledQueues();
const workers: (Worker | GroupWorker<any>)[] = [];
@@ -119,12 +116,14 @@ export async function bootWorkers() {
for (const index of eventQueuesToStart) {
const queue = eventsGroupQueues[index];
if (!queue) continue;
if (!queue) {
continue;
}
const queueName = `events_${index}`;
const concurrency = getConcurrencyFor(
queueName,
Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10),
Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10)
);
const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({
@@ -132,7 +131,7 @@ export async function bootWorkers() {
concurrency,
logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,
blockingTimeoutSec: Number.parseFloat(
process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1',
process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1'
),
handler: async (job) => {
return await incomingEvent(job.data);
@@ -172,7 +171,7 @@ export async function bootWorkers() {
const notificationWorker = new Worker(
notificationQueue.name,
notificationJob,
{ ...workerOptions, concurrency },
{ ...workerOptions, concurrency }
);
workers.push(notificationWorker);
logger.info('Started worker for notification', { concurrency });
@@ -224,7 +223,7 @@ export async function bootWorkers() {
if (workers.length === 0) {
logger.warn(
'No workers started. Check ENABLED_QUEUES environment variable.',
'No workers started. Check ENABLED_QUEUES environment variable.'
);
}
@@ -254,7 +253,7 @@ export async function bootWorkers() {
const elapsed = job.finishedOn - job.processedOn;
eventsGroupJobDuration.observe(
{ name: worker.name, status: 'failed' },
elapsed,
elapsed
);
}
logger.error('job failed', {
@@ -267,23 +266,6 @@ export async function bootWorkers() {
}
});
(worker as Worker).on('completed', (job) => {
if (job) {
if (job.processedOn && job.finishedOn) {
const elapsed = job.finishedOn - job.processedOn;
logger.info('job completed', {
jobId: job.id,
worker: worker.name,
elapsed,
});
eventsGroupJobDuration.observe(
{ name: worker.name, status: 'success' },
elapsed,
);
}
}
});
(worker as Worker).on('ioredis:close', () => {
logger.error('worker closed due to ioredis:close', {
worker: worker.name,
@@ -293,7 +275,7 @@ export async function bootWorkers() {
async function exitHandler(
eventName: string,
evtOrExitCodeOrError: number | string | Error,
evtOrExitCodeOrError: number | string | Error
) {
// Log the actual error details for unhandled rejections/exceptions
if (evtOrExitCodeOrError instanceof Error) {
@@ -339,7 +321,7 @@ export async function bootWorkers() {
process.on(evt, (code) => {
exitHandler(evt, code);
});
},
}
);
return workers;

2
apps/worker/src/jobs/cron.salt.ts
View File

@@ -33,7 +33,7 @@ async function generateNewSalt() {
return created;
});
getSalts.clear();
await getSalts.clear();
return newSalt;
}

131
apps/worker/src/jobs/events.incoming-event.ts
View File

@@ -1,9 +1,5 @@
import { getTime, isSameDomain, parsePath } from '@openpanel/common';
import {
getReferrerWithQuery,
parseReferrer,
parseUserAgent,
} from '@openpanel/common/server';
import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';
import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
import {
checkNotificationRulesForEvent,
@@ -14,10 +10,12 @@ import {
} from '@openpanel/db';
import type { ILogger } from '@openpanel/logger';
import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
import { getLock } from '@openpanel/redis';
import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';
import { logger as baseLogger } from '@/utils/logger';
import { createSessionEndJob, getSessionEnd } from '@/utils/session-handler';
import {
createSessionEndJob,
extendSessionEndJob,
} from '@/utils/session-handler';
const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];
@@ -93,7 +91,8 @@ export async function incomingEvent(
projectId,
deviceId,
sessionId,
uaInfo: _uaInfo,
uaInfo,
session,
} = jobPayload;
const properties = body.properties ?? {};
const reqId = headers['request-id'] ?? 'unknown';
@@ -121,16 +120,15 @@ export async function incomingEvent(
? null
: parseReferrer(getProperty('__referrer'));
const utmReferrer = getReferrerWithQuery(query);
const userAgent = headers['user-agent'];
const sdkName = headers['openpanel-sdk-name'];
const sdkVersion = headers['openpanel-sdk-version'];
// TODO: Remove both user-agent and parseUserAgent
const uaInfo = _uaInfo ?? parseUserAgent(userAgent, properties);
const baseEvent = {
const baseEvent: IServiceCreateEventPayload = {
name: body.name,
profileId,
projectId,
deviceId,
sessionId,
properties: omit(GLOBAL_PROPERTIES, {
...properties,
__hash: hash,
@@ -149,7 +147,7 @@ export async function incomingEvent(
origin,
referrer: referrer?.url || '',
referrerName: utmReferrer?.name || referrer?.name || referrer?.url,
referrerType: referrer?.type || utmReferrer?.type || '',
referrerType: utmReferrer?.type || referrer?.type || '',
os: uaInfo.os,
osVersion: uaInfo.osVersion,
browser: uaInfo.browser,
@@ -161,16 +159,17 @@ export async function incomingEvent(
body.name === 'revenue' && '__revenue' in properties
? parseRevenue(properties.__revenue)
: undefined,
} as const;
};
// if timestamp is from the past we dont want to create a new session
if (uaInfo.isServer || isTimestampFromThePast) {
const session = profileId
? await sessionBuffer.getExistingSession({
profileId,
projectId,
})
: null;
const session =
profileId && !isTimestampFromThePast
? await sessionBuffer.getExistingSession({
profileId,
projectId,
})
: null;
const payload = {
...baseEvent,
@@ -198,82 +197,48 @@ export async function incomingEvent(
return createEventAndNotify(payload as IServiceEvent, logger, projectId);
}
const sessionEnd = await getSessionEnd({
projectId,
deviceId,
profileId,
});
const activeSession = sessionEnd
? await sessionBuffer.getExistingSession({
sessionId: sessionEnd.sessionId,
})
: null;
const payload: IServiceCreateEventPayload = merge(baseEvent, {
deviceId: sessionEnd?.deviceId ?? deviceId,
sessionId: sessionEnd?.sessionId ?? sessionId,
referrer: sessionEnd?.referrer ?? baseEvent.referrer,
referrerName: sessionEnd?.referrerName ?? baseEvent.referrerName,
referrerType: sessionEnd?.referrerType ?? baseEvent.referrerType,
// if the path is not set, use the last screen view path
path: baseEvent.path || activeSession?.exit_path || '',
origin: baseEvent.origin || activeSession?.exit_origin || '',
referrer: session?.referrer ?? baseEvent.referrer,
referrerName: session?.referrerName ?? baseEvent.referrerName,
referrerType: session?.referrerType ?? baseEvent.referrerType,
} as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;
// If the triggering event is filtered, do not create session_start or the event (issue #2)
const isExcluded = await isEventExcludedByProjectFilter(payload, projectId);
if (isExcluded) {
logger.info(
'Skipping session_start and event (excluded by project filter)',
{ event: payload.name, projectId }
);
return null;
}
if (session) {
await extendSessionEndJob({
projectId,
deviceId,
}).catch((error) => {
logger.error('Error finding and extending session end job', { error });
throw error;
});
} else {
await createEventAndNotify(
{
event: payload.name,
projectId,
}
);
return null;
}
...payload,
name: 'session_start',
createdAt: new Date(getTime(payload.createdAt) - 100),
},
logger,
projectId
).catch((error) => {
logger.error('Error creating session start event', { event: payload });
throw error;
});
if (!sessionEnd) {
const locked = await getLock(
`session_start:${projectId}:${sessionId}`,
'1',
1000
);
if (locked) {
logger.info('Creating session start event', { event: payload });
await createEventAndNotify(
{
...payload,
name: 'session_start',
createdAt: new Date(getTime(payload.createdAt) - 100),
},
logger,
projectId
).catch((error) => {
logger.error('Error creating session start event', { event: payload });
throw error;
});
} else {
logger.info('Session start already claimed by another worker', {
event: payload,
});
}
}
const event = await createEventAndNotify(payload, logger, projectId);
if (!event) {
// Skip creating session end when event was excluded
return null;
}
if (!sessionEnd) {
logger.info('Creating session end job', { event: payload });
await createSessionEndJob({ payload }).catch((error) => {
logger.error('Error creating session end job', { event: payload });
throw error;
});
}
return event;
return createEventAndNotify(payload, logger, projectId);
}

5
apps/worker/src/jobs/events.incoming-events.test.ts
View File

@@ -186,6 +186,11 @@ describe('incomingEvent', () => {
projectId,
deviceId,
sessionId: 'session-123',
session: {
referrer: '',
referrerName: '',
referrerType: '',
},
};
const changeDelay = vi.fn();

43
apps/worker/src/metrics.ts
View File

@@ -1,5 +1,3 @@
import client from 'prom-client';
import {
botBuffer,
eventBuffer,
@@ -8,6 +6,7 @@ import {
sessionBuffer,
} from '@openpanel/db';
import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';
import client from 'prom-client';
const Registry = client.Registry;
@@ -20,7 +19,7 @@ export const eventsGroupJobDuration = new client.Histogram({
name: 'job_duration_ms',
help: 'Duration of job processing (in ms)',
labelNames: ['name', 'status'],
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10_000, 30_000], // 10ms to 30s
});
register.registerMetric(eventsGroupJobDuration);
@@ -28,57 +27,61 @@ register.registerMetric(eventsGroupJobDuration);
queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_active_count`,
name: `${queue.name.replace(/[{}]/g, '')}_active_count`,
help: 'Active count',
async collect() {
const metric = await queue.getActiveCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_delayed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_delayed_count`,
help: 'Delayed count',
async collect() {
const metric = await queue.getDelayedCount();
this.set(metric);
if ('getDelayedCount' in queue) {
const metric = await queue.getDelayedCount();
this.set(metric);
} else {
this.set(0);
}
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_failed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_failed_count`,
help: 'Failed count',
async collect() {
const metric = await queue.getFailedCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_completed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_completed_count`,
help: 'Completed count',
async collect() {
const metric = await queue.getCompletedCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_waiting_count`,
name: `${queue.name.replace(/[{}]/g, '')}_waiting_count`,
help: 'Waiting count',
async collect() {
const metric = await queue.getWaitingCount();
this.set(metric);
},
}),
})
);
});
@@ -90,7 +93,7 @@ register.registerMetric(
const metric = await eventBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -101,7 +104,7 @@ register.registerMetric(
const metric = await profileBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -112,7 +115,7 @@ register.registerMetric(
const metric = await botBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -123,7 +126,7 @@ register.registerMetric(
const metric = await sessionBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -134,5 +137,5 @@ register.registerMetric(
const metric = await replayBuffer.getBufferSize();
this.set(metric);
},
}),
})
);

141
apps/worker/src/utils/session-handler.ts
View File

@@ -1,13 +1,39 @@
import type { IServiceCreateEventPayload } from '@openpanel/db';
import {
type EventsQueuePayloadCreateSessionEnd,
sessionsQueue,
} from '@openpanel/queue';
import type { Job } from 'bullmq';
import { logger } from './logger';
import { sessionsQueue } from '@openpanel/queue';
export const SESSION_TIMEOUT = 1000 * 60 * 30;
const CHANGE_DELAY_THROTTLE_MS = process.env.CHANGE_DELAY_THROTTLE_MS
? Number.parseInt(process.env.CHANGE_DELAY_THROTTLE_MS, 10)
: 60_000; // 1 minute
const CHANGE_DELAY_THROTTLE_MAP = new Map<string, number>();
export async function extendSessionEndJob({
projectId,
deviceId,
}: {
projectId: string;
deviceId: string;
}) {
const last = CHANGE_DELAY_THROTTLE_MAP.get(`${projectId}:${deviceId}`) ?? 0;
const isThrottled = Date.now() - last < CHANGE_DELAY_THROTTLE_MS;
if (isThrottled) {
return;
}
const jobId = getSessionEndJobId(projectId, deviceId);
const job = await sessionsQueue.getJob(jobId);
if (!job) {
return;
}
await job.changeDelay(SESSION_TIMEOUT);
CHANGE_DELAY_THROTTLE_MAP.set(`${projectId}:${deviceId}`, Date.now());
}
const getSessionEndJobId = (projectId: string, deviceId: string) =>
`sessionEnd:${projectId}:${deviceId}`;
@@ -33,106 +59,3 @@ export function createSessionEndJob({
}
);
}
export async function getSessionEnd({
projectId,
deviceId,
profileId,
}: {
projectId: string;
deviceId: string;
profileId: string;
}) {
const sessionEnd = await getSessionEndJob({
projectId,
deviceId,
});
if (sessionEnd) {
const existingSessionIsAnonymous =
sessionEnd.job.data.payload.profileId ===
sessionEnd.job.data.payload.deviceId;
const eventIsIdentified =
profileId && sessionEnd.job.data.payload.profileId !== profileId;
if (existingSessionIsAnonymous && eventIsIdentified) {
await sessionEnd.job.updateData({
...sessionEnd.job.data,
payload: {
...sessionEnd.job.data.payload,
profileId,
},
});
}
await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
return sessionEnd.job.data.payload;
}
return null;
}
export async function getSessionEndJob(args: {
projectId: string;
deviceId: string;
retryCount?: number;
}): Promise<{
deviceId: string;
job: Job<EventsQueuePayloadCreateSessionEnd>;
} | null> {
const { retryCount = 0 } = args;
if (retryCount >= 6) {
throw new Error('Failed to get session end');
}
async function handleJobStates(
job: Job<EventsQueuePayloadCreateSessionEnd>,
deviceId: string
): Promise<{
deviceId: string;
job: Job<EventsQueuePayloadCreateSessionEnd>;
} | null> {
const state = await job.getState();
if (state !== 'delayed') {
logger.debug(`[session-handler] Session end job is in "${state}" state`, {
state,
retryCount,
jobTimestamp: new Date(job.timestamp).toISOString(),
jobDelta: Date.now() - job.timestamp,
jobId: job.id,
payload: job.data.payload,
});
}
if (state === 'delayed' || state === 'waiting') {
return { deviceId, job };
}
if (state === 'active') {
await new Promise((resolve) => setTimeout(resolve, 100));
return getSessionEndJob({
...args,
retryCount: retryCount + 1,
});
}
if (state === 'completed') {
await job.remove();
}
return null;
}
// Check current device job
const currentJob = await sessionsQueue.getJob(
getSessionEndJobId(args.projectId, args.deviceId)
);
if (currentJob) {
return await handleJobStates(currentJob, args.deviceId);
}
// Create session
return null;
}

4
biome.json
View File

@@ -42,6 +42,7 @@
"useSemanticElements": "off"
},
"style": {
"noNestedTernary": "off",
"noNonNullAssertion": "off",
"noParameterAssign": "error",
"useAsConstAssertion": "error",
@@ -70,7 +71,8 @@
"noDangerouslySetInnerHtml": "off"
},
"complexity": {
"noForEach": "off"
"noForEach": "off",
"noExcessiveCognitiveComplexity": "off"
}
}
},

366
packages/db/src/buffers/event-buffer.test.ts
View File

@@ -2,42 +2,8 @@ import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { ch } from '../clickhouse/client';
// Mock transformEvent to avoid circular dependency with buffers -> services -> buffers
vi.mock('../services/event.service', () => ({
transformEvent: (event: any) => ({
id: event.id ?? 'id',
name: event.name,
deviceId: event.device_id,
profileId: event.profile_id,
projectId: event.project_id,
sessionId: event.session_id,
properties: event.properties ?? {},
createdAt: new Date(event.created_at ?? Date.now()),
country: event.country,
city: event.city,
region: event.region,
longitude: event.longitude,
latitude: event.latitude,
os: event.os,
osVersion: event.os_version,
browser: event.browser,
browserVersion: event.browser_version,
device: event.device,
brand: event.brand,
model: event.model,
duration: event.duration ?? 0,
path: event.path ?? '',
origin: event.origin ?? '',
referrer: event.referrer,
referrerName: event.referrer_name,
referrerType: event.referrer_type,
meta: event.meta,
importedAt: undefined,
sdkName: event.sdk_name,
sdkVersion: event.sdk_version,
profile: event.profile,
}),
}));
// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
vi.mock('../services/event.service', () => ({}));
import { EventBuffer } from './event-buffer';
@@ -68,18 +34,16 @@ describe('EventBuffer', () => {
created_at: new Date().toISOString(),
} as any;
// Get initial count
const initialCount = await eventBuffer.getBufferSize();
// Add event
await eventBuffer.add(event);
eventBuffer.add(event);
await eventBuffer.flush();
// Buffer counter should increase by 1
const newCount = await eventBuffer.getBufferSize();
expect(newCount).toBe(initialCount + 1);
});
it('adds multiple screen_views - moves previous to buffer with duration', async () => {
it('adds screen_view directly to buffer queue', async () => {
const t0 = Date.now();
const sessionId = 'session_1';
@@ -99,60 +63,23 @@ describe('EventBuffer', () => {
created_at: new Date(t0 + 1000).toISOString(),
} as any;
const view3 = {
project_id: 'p1',
profile_id: 'u1',
session_id: sessionId,
name: 'screen_view',
created_at: new Date(t0 + 3000).toISOString(),
} as any;
// Add first screen_view
const count1 = await eventBuffer.getBufferSize();
await eventBuffer.add(view1);
// Should be stored as "last" but NOT in queue yet
eventBuffer.add(view1);
await eventBuffer.flush();
// screen_view goes directly to buffer
const count2 = await eventBuffer.getBufferSize();
expect(count2).toBe(count1); // No change in buffer
expect(count2).toBe(count1 + 1);
// Last screen_view should be retrievable
const last1 = await eventBuffer.getLastScreenView({
projectId: 'p1',
sessionId: sessionId,
});
expect(last1).not.toBeNull();
expect(last1!.createdAt.toISOString()).toBe(view1.created_at);
eventBuffer.add(view2);
await eventBuffer.flush();
// Add second screen_view
await eventBuffer.add(view2);
// Now view1 should be in buffer
const count3 = await eventBuffer.getBufferSize();
expect(count3).toBe(count1 + 1);
// view2 should now be the "last"
const last2 = await eventBuffer.getLastScreenView({
projectId: 'p1',
sessionId: sessionId,
});
expect(last2!.createdAt.toISOString()).toBe(view2.created_at);
// Add third screen_view
await eventBuffer.add(view3);
// Now view2 should also be in buffer
const count4 = await eventBuffer.getBufferSize();
expect(count4).toBe(count1 + 2);
// view3 should now be the "last"
const last3 = await eventBuffer.getLastScreenView({
projectId: 'p1',
sessionId: sessionId,
});
expect(last3!.createdAt.toISOString()).toBe(view3.created_at);
expect(count3).toBe(count1 + 2);
});
it('adds session_end - moves last screen_view and session_end to buffer', async () => {
it('adds session_end directly to buffer queue', async () => {
const t0 = Date.now();
const sessionId = 'session_2';
@@ -172,148 +99,44 @@ describe('EventBuffer', () => {
created_at: new Date(t0 + 5000).toISOString(),
} as any;
// Add screen_view
const count1 = await eventBuffer.getBufferSize();
await eventBuffer.add(view);
// Should be stored as "last", not in buffer yet
eventBuffer.add(view);
eventBuffer.add(sessionEnd);
await eventBuffer.flush();
const count2 = await eventBuffer.getBufferSize();
expect(count2).toBe(count1);
// Add session_end
await eventBuffer.add(sessionEnd);
// Both should now be in buffer (+2)
const count3 = await eventBuffer.getBufferSize();
expect(count3).toBe(count1 + 2);
// Last screen_view should be cleared
const last = await eventBuffer.getLastScreenView({
projectId: 'p2',
sessionId: sessionId,
});
expect(last).toBeNull();
});
it('session_end with no previous screen_view - only adds session_end to buffer', async () => {
const sessionId = 'session_3';
const sessionEnd = {
project_id: 'p3',
profile_id: 'u3',
session_id: sessionId,
name: 'session_end',
created_at: new Date().toISOString(),
} as any;
const count1 = await eventBuffer.getBufferSize();
await eventBuffer.add(sessionEnd);
// Only session_end should be in buffer (+1)
const count2 = await eventBuffer.getBufferSize();
expect(count2).toBe(count1 + 1);
});
it('gets last screen_view by profileId', async () => {
const view = {
project_id: 'p4',
profile_id: 'u4',
session_id: 'session_4',
name: 'screen_view',
path: '/home',
created_at: new Date().toISOString(),
} as any;
await eventBuffer.add(view);
// Query by profileId
const result = await eventBuffer.getLastScreenView({
projectId: 'p4',
profileId: 'u4',
});
expect(result).not.toBeNull();
expect(result!.name).toBe('screen_view');
expect(result!.path).toBe('/home');
});
it('gets last screen_view by sessionId', async () => {
const sessionId = 'session_5';
const view = {
project_id: 'p5',
profile_id: 'u5',
session_id: sessionId,
name: 'screen_view',
path: '/about',
created_at: new Date().toISOString(),
} as any;
await eventBuffer.add(view);
// Query by sessionId
const result = await eventBuffer.getLastScreenView({
projectId: 'p5',
sessionId: sessionId,
});
expect(result).not.toBeNull();
expect(result!.name).toBe('screen_view');
expect(result!.path).toBe('/about');
});
it('returns null for non-existent last screen_view', async () => {
const result = await eventBuffer.getLastScreenView({
projectId: 'p_nonexistent',
profileId: 'u_nonexistent',
});
expect(result).toBeNull();
expect(count2).toBe(count1 + 2);
});
it('gets buffer count correctly', async () => {
// Initially 0
expect(await eventBuffer.getBufferSize()).toBe(0);
// Add regular event
await eventBuffer.add({
eventBuffer.add({
project_id: 'p6',
name: 'event1',
created_at: new Date().toISOString(),
} as any);
await eventBuffer.flush();
expect(await eventBuffer.getBufferSize()).toBe(1);
// Add another regular event
await eventBuffer.add({
eventBuffer.add({
project_id: 'p6',
name: 'event2',
created_at: new Date().toISOString(),
} as any);
await eventBuffer.flush();
expect(await eventBuffer.getBufferSize()).toBe(2);
// Add screen_view (not counted until flushed)
await eventBuffer.add({
// screen_view also goes directly to buffer
eventBuffer.add({
project_id: 'p6',
profile_id: 'u6',
session_id: 'session_6',
name: 'screen_view',
created_at: new Date().toISOString(),
} as any);
// Still 2 (screen_view is pending)
expect(await eventBuffer.getBufferSize()).toBe(2);
// Add another screen_view (first one gets flushed)
await eventBuffer.add({
project_id: 'p6',
profile_id: 'u6',
session_id: 'session_6',
name: 'screen_view',
created_at: new Date(Date.now() + 1000).toISOString(),
} as any);
// Now 3 (2 regular + 1 flushed screen_view)
await eventBuffer.flush();
expect(await eventBuffer.getBufferSize()).toBe(3);
});
@@ -330,8 +153,9 @@ describe('EventBuffer', () => {
created_at: new Date(Date.now() + 1000).toISOString(),
} as any;
await eventBuffer.add(event1);
await eventBuffer.add(event2);
eventBuffer.add(event1);
eventBuffer.add(event2);
await eventBuffer.flush();
expect(await eventBuffer.getBufferSize()).toBe(2);
@@ -341,14 +165,12 @@ describe('EventBuffer', () => {
await eventBuffer.processBuffer();
// Should insert both events
expect(insertSpy).toHaveBeenCalled();
const callArgs = insertSpy.mock.calls[0]![0];
expect(callArgs.format).toBe('JSONEachRow');
expect(callArgs.table).toBe('events');
expect(Array.isArray(callArgs.values)).toBe(true);
// Buffer should be empty after processing
expect(await eventBuffer.getBufferSize()).toBe(0);
insertSpy.mockRestore();
@@ -359,14 +181,14 @@ describe('EventBuffer', () => {
process.env.EVENT_BUFFER_CHUNK_SIZE = '2';
const eb = new EventBuffer();
// Add 4 events
for (let i = 0; i < 4; i++) {
await eb.add({
eb.add({
project_id: 'p8',
name: `event${i}`,
created_at: new Date(Date.now() + i).toISOString(),
} as any);
}
await eb.flush();
const insertSpy = vi
.spyOn(ch, 'insert')
@@ -374,14 +196,12 @@ describe('EventBuffer', () => {
await eb.processBuffer();
// With chunk size 2 and 4 events, should be called twice
expect(insertSpy).toHaveBeenCalledTimes(2);
const call1Values = insertSpy.mock.calls[0]![0].values as any[];
const call2Values = insertSpy.mock.calls[1]![0].values as any[];
expect(call1Values.length).toBe(2);
expect(call2Values.length).toBe(2);
// Restore
if (prev === undefined) delete process.env.EVENT_BUFFER_CHUNK_SIZE;
else process.env.EVENT_BUFFER_CHUNK_SIZE = prev;
@@ -396,129 +216,61 @@ describe('EventBuffer', () => {
created_at: new Date().toISOString(),
} as any;
await eventBuffer.add(event);
eventBuffer.add(event);
await eventBuffer.flush();
const count = await eventBuffer.getActiveVisitorCount('p9');
expect(count).toBeGreaterThanOrEqual(1);
});
it('handles multiple sessions independently', async () => {
it('handles multiple sessions independently — all events go to buffer', async () => {
const t0 = Date.now();
const count1 = await eventBuffer.getBufferSize();
// Session 1
const view1a = {
eventBuffer.add({
project_id: 'p10',
profile_id: 'u10',
session_id: 'session_10a',
name: 'screen_view',
created_at: new Date(t0).toISOString(),
} as any;
const view1b = {
project_id: 'p10',
profile_id: 'u10',
session_id: 'session_10a',
name: 'screen_view',
created_at: new Date(t0 + 1000).toISOString(),
} as any;
// Session 2
const view2a = {
} as any);
eventBuffer.add({
project_id: 'p10',
profile_id: 'u11',
session_id: 'session_10b',
name: 'screen_view',
created_at: new Date(t0).toISOString(),
} as any;
const view2b = {
} as any);
eventBuffer.add({
project_id: 'p10',
profile_id: 'u10',
session_id: 'session_10a',
name: 'screen_view',
created_at: new Date(t0 + 1000).toISOString(),
} as any);
eventBuffer.add({
project_id: 'p10',
profile_id: 'u11',
session_id: 'session_10b',
name: 'screen_view',
created_at: new Date(t0 + 2000).toISOString(),
} as any;
} as any);
await eventBuffer.flush();
await eventBuffer.add(view1a);
await eventBuffer.add(view2a);
await eventBuffer.add(view1b); // Flushes view1a
await eventBuffer.add(view2b); // Flushes view2a
// Should have 2 events in buffer (one from each session)
expect(await eventBuffer.getBufferSize()).toBe(2);
// Each session should have its own "last" screen_view
const last1 = await eventBuffer.getLastScreenView({
projectId: 'p10',
sessionId: 'session_10a',
});
expect(last1!.createdAt.toISOString()).toBe(view1b.created_at);
const last2 = await eventBuffer.getLastScreenView({
projectId: 'p10',
sessionId: 'session_10b',
});
expect(last2!.createdAt.toISOString()).toBe(view2b.created_at);
// All 4 events are in buffer directly
expect(await eventBuffer.getBufferSize()).toBe(count1 + 4);
});
it('screen_view without session_id goes directly to buffer', async () => {
const view = {
it('bulk adds events to buffer', async () => {
const events = Array.from({ length: 5 }, (_, i) => ({
project_id: 'p11',
profile_id: 'u11',
name: 'screen_view',
created_at: new Date().toISOString(),
} as any;
name: `event${i}`,
created_at: new Date(Date.now() + i).toISOString(),
})) as any[];
const count1 = await eventBuffer.getBufferSize();
await eventBuffer.add(view);
eventBuffer.bulkAdd(events);
await eventBuffer.flush();
// Should go directly to buffer (no session_id)
const count2 = await eventBuffer.getBufferSize();
expect(count2).toBe(count1 + 1);
});
it('updates last screen_view when new one arrives from same profile but different session', async () => {
const t0 = Date.now();
const view1 = {
project_id: 'p12',
profile_id: 'u12',
session_id: 'session_12a',
name: 'screen_view',
path: '/page1',
created_at: new Date(t0).toISOString(),
} as any;
const view2 = {
project_id: 'p12',
profile_id: 'u12',
session_id: 'session_12b', // Different session!
name: 'screen_view',
path: '/page2',
created_at: new Date(t0 + 1000).toISOString(),
} as any;
await eventBuffer.add(view1);
await eventBuffer.add(view2);
// Both sessions should have their own "last"
const lastSession1 = await eventBuffer.getLastScreenView({
projectId: 'p12',
sessionId: 'session_12a',
});
expect(lastSession1!.path).toBe('/page1');
const lastSession2 = await eventBuffer.getLastScreenView({
projectId: 'p12',
sessionId: 'session_12b',
});
expect(lastSession2!.path).toBe('/page2');
// Profile should have the latest one
const lastProfile = await eventBuffer.getLastScreenView({
projectId: 'p12',
profileId: 'u12',
});
expect(lastProfile!.path).toBe('/page2');
expect(await eventBuffer.getBufferSize()).toBe(5);
});
});

442
packages/db/src/buffers/event-buffer.ts
View File

@@ -2,33 +2,13 @@ import { getSafeJson } from '@openpanel/json';
import {
type Redis,
getRedisCache,
getRedisPub,
publishEvent,
} from '@openpanel/redis';
import { ch } from '../clickhouse/client';
import {
type IClickhouseEvent,
type IServiceEvent,
transformEvent,
} from '../services/event.service';
import { type IClickhouseEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer';
/**
* Simplified Event Buffer
*
* Rules:
* 1. All events go into a single list buffer (event_buffer:queue)
* 2. screen_view events are handled specially:
* - Store current screen_view as "last" for the session
* - When a new screen_view arrives, flush the previous one with calculated duration
* 3. session_end events:
* - Retrieve the last screen_view (don't modify it)
* - Push both screen_view and session_end to buffer
* 4. Flush: Simply process all events from the list buffer
*/
export class EventBuffer extends BaseBuffer {
// Configurable limits
private batchSize = process.env.EVENT_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.EVENT_BUFFER_BATCH_SIZE, 10)
: 4000;
@@ -36,124 +16,26 @@ export class EventBuffer extends BaseBuffer {
? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)
: 1000;
private microBatchIntervalMs = process.env.EVENT_BUFFER_MICRO_BATCH_MS
? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_MS, 10)
: 10;
private microBatchMaxSize = process.env.EVENT_BUFFER_MICRO_BATCH_SIZE
? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_SIZE, 10)
: 100;
private pendingEvents: IClickhouseEvent[] = [];
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private isFlushing = false;
/** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0;
private activeVisitorsExpiration = 60 * 5; // 5 minutes
// LIST - Stores all events ready to be flushed
/** How often (ms) we refresh the heartbeat key + zadd per visitor. */
private heartbeatRefreshMs = 60_000; // 1 minute
private lastHeartbeat = new Map<string, number>();
private queueKey = 'event_buffer:queue';
// STRING - Tracks total buffer size incrementally
protected bufferCounterKey = 'event_buffer:total_count';
// Script SHAs for loaded Lua scripts
private scriptShas: {
addScreenView?: string;
addSessionEnd?: string;
} = {};
// Hash key for storing last screen_view per session
private getLastScreenViewKeyBySession(sessionId: string) {
return `event_buffer:last_screen_view:session:${sessionId}`;
}
// Hash key for storing last screen_view per profile
private getLastScreenViewKeyByProfile(projectId: string, profileId: string) {
return `event_buffer:last_screen_view:profile:${projectId}:${profileId}`;
}
/**
* Lua script for handling screen_view addition - RACE-CONDITION SAFE without GroupMQ
*
* Strategy: Use Redis GETDEL (atomic get-and-delete) to ensure only ONE thread
* can process the "last" screen_view at a time.
*
* KEYS[1] = last screen_view key (by session) - stores both event and timestamp as JSON
* KEYS[2] = last screen_view key (by profile, may be empty)
* KEYS[3] = queue key
* KEYS[4] = buffer counter key
* ARGV[1] = new event with timestamp as JSON: {"event": {...}, "ts": 123456}
* ARGV[2] = TTL for last screen_view (1 hour)
*/
private readonly addScreenViewScript = `
local sessionKey = KEYS[1]
local profileKey = KEYS[2]
local queueKey = KEYS[3]
local counterKey = KEYS[4]
local newEventData = ARGV[1]
local ttl = tonumber(ARGV[2])
-- GETDEL is atomic: get previous and delete in one operation
-- This ensures only ONE thread gets the previous event
local previousEventData = redis.call("GETDEL", sessionKey)
-- Store new screen_view as last for session
redis.call("SET", sessionKey, newEventData, "EX", ttl)
-- Store new screen_view as last for profile (if key provided)
if profileKey and profileKey ~= "" then
redis.call("SET", profileKey, newEventData, "EX", ttl)
end
-- If there was a previous screen_view, add it to queue with calculated duration
if previousEventData then
local prev = cjson.decode(previousEventData)
local curr = cjson.decode(newEventData)
-- Calculate duration (ensure non-negative to handle clock skew)
if prev.ts and curr.ts then
prev.event.duration = math.max(0, curr.ts - prev.ts)
end
redis.call("RPUSH", queueKey, cjson.encode(prev.event))
redis.call("INCR", counterKey)
return 1
end
return 0
`;
/**
* Lua script for handling session_end - RACE-CONDITION SAFE
*
* Uses GETDEL to atomically retrieve and delete the last screen_view
*
* KEYS[1] = last screen_view key (by session)
* KEYS[2] = last screen_view key (by profile, may be empty)
* KEYS[3] = queue key
* KEYS[4] = buffer counter key
* ARGV[1] = session_end event JSON
*/
private readonly addSessionEndScript = `
local sessionKey = KEYS[1]
local profileKey = KEYS[2]
local queueKey = KEYS[3]
local counterKey = KEYS[4]
local sessionEndJson = ARGV[1]
-- GETDEL is atomic: only ONE thread gets the last screen_view
local previousEventData = redis.call("GETDEL", sessionKey)
local added = 0
-- If there was a previous screen_view, add it to queue
if previousEventData then
local prev = cjson.decode(previousEventData)
redis.call("RPUSH", queueKey, cjson.encode(prev.event))
redis.call("INCR", counterKey)
added = added + 1
end
-- Add session_end to queue
redis.call("RPUSH", queueKey, sessionEndJson)
redis.call("INCR", counterKey)
added = added + 1
-- Delete profile key
if profileKey and profileKey ~= "" then
redis.call("DEL", profileKey)
end
return added
`;
constructor() {
super({
name: 'event',
@@ -161,170 +43,97 @@ return added
await this.processBuffer();
},
});
// Load Lua scripts into Redis on startup
this.loadScripts();
}
/**
* Load Lua scripts into Redis and cache their SHAs.
* This avoids sending the entire script on every call.
*/
private async loadScripts() {
try {
const redis = getRedisCache();
const [screenViewSha, sessionEndSha] = await Promise.all([
redis.script('LOAD', this.addScreenViewScript),
redis.script('LOAD', this.addSessionEndScript),
]);
this.scriptShas.addScreenView = screenViewSha as string;
this.scriptShas.addSessionEnd = sessionEndSha as string;
this.logger.info('Loaded Lua scripts into Redis', {
addScreenView: this.scriptShas.addScreenView,
addSessionEnd: this.scriptShas.addSessionEnd,
});
} catch (error) {
this.logger.error('Failed to load Lua scripts', { error });
}
}
bulkAdd(events: IClickhouseEvent[]) {
const redis = getRedisCache();
const multi = redis.multi();
for (const event of events) {
this.add(event, multi);
this.add(event);
}
return multi.exec();
}
/**
* Add an event into Redis buffer.
*
* Logic:
* - screen_view: Store as "last" for session, flush previous if exists
* - session_end: Flush last screen_view + session_end
* - Other events: Add directly to queue
*/
async add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {
add(event: IClickhouseEvent) {
this.pendingEvents.push(event);
if (this.pendingEvents.length >= this.microBatchMaxSize) {
this.flushLocalBuffer();
return;
}
if (!this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
public async flush() {
if (this.flushTimer) {
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
await this.flushLocalBuffer();
}
private async flushLocalBuffer() {
if (this.isFlushing || this.pendingEvents.length === 0) {
return;
}
this.isFlushing = true;
const eventsToFlush = this.pendingEvents;
this.pendingEvents = [];
try {
const redis = getRedisCache();
const eventJson = JSON.stringify(event);
const multi = _multi || redis.multi();
const multi = redis.multi();
if (event.session_id && event.name === 'screen_view') {
// Handle screen_view
const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
const profileKey = event.profile_id
? this.getLastScreenViewKeyByProfile(
event.project_id,
event.profile_id,
)
: '';
const timestamp = new Date(event.created_at || Date.now()).getTime();
// Combine event and timestamp into single JSON for atomic operations
const eventWithTimestamp = JSON.stringify({
event: event,
ts: timestamp,
});
this.evalScript(
multi,
'addScreenView',
this.addScreenViewScript,
4,
sessionKey,
profileKey,
this.queueKey,
this.bufferCounterKey,
eventWithTimestamp,
'3600', // 1 hour TTL
);
} else if (event.session_id && event.name === 'session_end') {
// Handle session_end
const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
const profileKey = event.profile_id
? this.getLastScreenViewKeyByProfile(
event.project_id,
event.profile_id,
)
: '';
this.evalScript(
multi,
'addSessionEnd',
this.addSessionEndScript,
4,
sessionKey,
profileKey,
this.queueKey,
this.bufferCounterKey,
eventJson,
);
} else {
// All other events go directly to queue
multi.rpush(this.queueKey, eventJson).incr(this.bufferCounterKey);
for (const event of eventsToFlush) {
multi.rpush(this.queueKey, JSON.stringify(event));
if (event.profile_id) {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id,
);
}
}
multi.incrby(this.bufferCounterKey, eventsToFlush.length);
if (event.profile_id) {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id,
);
}
await multi.exec();
if (!_multi) {
await multi.exec();
}
await publishEvent('events', 'received', transformEvent(event));
this.flushRetryCount = 0;
this.pruneHeartbeatMap();
} catch (error) {
this.logger.error('Failed to add event to Redis buffer', { error });
// Re-queue failed events at the front to preserve order and avoid data loss
this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
this.flushRetryCount += 1;
this.logger.warn(
'Failed to flush local buffer to Redis; events re-queued',
{
error,
eventCount: eventsToFlush.length,
flushRetryCount: this.flushRetryCount,
},
);
} finally {
this.isFlushing = false;
// Events may have accumulated while we were flushing; schedule another flush if needed
if (this.pendingEvents.length > 0 && !this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
}
/**
* Execute a Lua script using EVALSHA (cached) or fallback to EVAL.
* This avoids sending the entire script on every call.
*/
private evalScript(
multi: ReturnType<Redis['multi']>,
scriptName: keyof typeof this.scriptShas,
scriptContent: string,
numKeys: number,
...args: (string | number)[]
) {
const sha = this.scriptShas[scriptName];
if (sha) {
// Use EVALSHA with cached SHA
multi.evalsha(sha, numKeys, ...args);
} else {
// Fallback to EVAL and try to reload script
multi.eval(scriptContent, numKeys, ...args);
this.logger.warn(`Script ${scriptName} not loaded, using EVAL fallback`);
// Attempt to reload scripts in background
this.loadScripts();
}
}
/**
* Process the Redis buffer - simplified version.
*
* Simply:
* 1. Fetch events from the queue (up to batchSize)
* 2. Parse and sort them
* 3. Insert into ClickHouse in chunks
* 4. Publish saved events
* 5. Clean up processed events from queue
*/
async processBuffer() {
const redis = getRedisCache();
try {
// Fetch events from queue
const queueEvents = await redis.lrange(
this.queueKey,
0,
@@ -336,7 +145,6 @@ return added
return;
}
// Parse events
const eventsToClickhouse: IClickhouseEvent[] = [];
for (const eventStr of queueEvents) {
const event = getSafeJson<IClickhouseEvent>(eventStr);
@@ -350,14 +158,12 @@ return added
return;
}
// Sort events by creation time
eventsToClickhouse.sort(
(a, b) =>
new Date(a.created_at || 0).getTime() -
new Date(b.created_at || 0).getTime(),
);
// Insert events into ClickHouse in chunks
this.logger.info('Inserting events into ClickHouse', {
totalEvents: eventsToClickhouse.length,
chunks: Math.ceil(eventsToClickhouse.length / this.chunkSize),
@@ -371,14 +177,17 @@ return added
});
}
// Publish "saved" events
const pubMulti = getRedisPub().multi();
const countByProject = new Map<string, number>();
for (const event of eventsToClickhouse) {
await publishEvent('events', 'saved', transformEvent(event), pubMulti);
countByProject.set(
event.project_id,
(countByProject.get(event.project_id) ?? 0) + 1,
);
}
for (const [projectId, count] of countByProject) {
publishEvent('events', 'batch', { projectId, count });
}
await pubMulti.exec();
// Clean up processed events from queue
await redis
.multi()
.ltrim(this.queueKey, queueEvents.length, -1)
@@ -394,45 +203,6 @@ return added
}
}
/**
* Retrieve the latest screen_view event for a given session or profile
*/
public async getLastScreenView(
params:
| {
sessionId: string;
}
| {
projectId: string;
profileId: string;
},
): Promise<IServiceEvent | null> {
const redis = getRedisCache();
let lastScreenViewKey: string;
if ('sessionId' in params) {
lastScreenViewKey = this.getLastScreenViewKeyBySession(params.sessionId);
} else {
lastScreenViewKey = this.getLastScreenViewKeyByProfile(
params.projectId,
params.profileId,
);
}
const eventDataStr = await redis.get(lastScreenViewKey);
if (eventDataStr) {
const eventData = getSafeJson<{ event: IClickhouseEvent; ts: number }>(
eventDataStr,
);
if (eventData?.event) {
return transformEvent(eventData.event);
}
}
return null;
}
public async getBufferSize() {
return this.getBufferSizeWithCounter(async () => {
const redis = getRedisCache();
@@ -440,16 +210,32 @@ return added
});
}
private async incrementActiveVisitorCount(
private pruneHeartbeatMap() {
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
for (const [key, ts] of this.lastHeartbeat) {
if (ts < cutoff) {
this.lastHeartbeat.delete(key);
}
}
}
private incrementActiveVisitorCount(
multi: ReturnType<Redis['multi']>,
projectId: string,
profileId: string,
) {
// Track active visitors and emit expiry events when inactive for TTL
const key = `${projectId}:${profileId}`;
const now = Date.now();
const last = this.lastHeartbeat.get(key) ?? 0;
if (now - last < this.heartbeatRefreshMs) {
return;
}
this.lastHeartbeat.set(key, now);
const zsetKey = `live:visitors:${projectId}`;
const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
return multi
multi
.zadd(zsetKey, now, profileId)
.set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
}

40
packages/db/src/buffers/profile-buffer.ts
View File

@@ -1,11 +1,11 @@
import { deepMergeObjects } from '@openpanel/common';
import { getSafeJson } from '@openpanel/json';
import type { ILogger } from '@openpanel/logger';
import { type Redis, getRedisCache } from '@openpanel/redis';
import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import { omit } from 'ramda';
import sqlstring from 'sqlstring';
import { TABLE_NAMES, ch, chQuery } from '../clickhouse/client';
import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
import type { IClickhouseProfile } from '../services/profile.service';
import { BaseBuffer } from './base-buffer';
@@ -89,7 +89,7 @@ export class ProfileBuffer extends BaseBuffer {
'os_version',
'browser_version',
],
profile.properties,
profile.properties
);
}
@@ -97,16 +97,16 @@ export class ProfileBuffer extends BaseBuffer {
? deepMergeObjects(existingProfile, omit(['created_at'], profile))
: profile;
if (profile && existingProfile) {
if (
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile),
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
if (
profile &&
existingProfile &&
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile)
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
this.logger.debug('Merged profile will be inserted', {
@@ -151,11 +151,11 @@ export class ProfileBuffer extends BaseBuffer {
private async fetchProfile(
profile: IClickhouseProfile,
logger: ILogger,
logger: ILogger
): Promise<IClickhouseProfile | null> {
const existingProfile = await this.fetchFromCache(
profile.id,
profile.project_id,
profile.project_id
);
if (existingProfile) {
logger.debug('Profile found in Redis');
@@ -167,7 +167,7 @@ export class ProfileBuffer extends BaseBuffer {
public async fetchFromCache(
profileId: string,
projectId: string,
projectId: string
): Promise<IClickhouseProfile | null> {
const cacheKey = this.getProfileCacheKey({
profileId,
@@ -182,7 +182,7 @@ export class ProfileBuffer extends BaseBuffer {
private async fetchFromClickhouse(
profile: IClickhouseProfile,
logger: ILogger,
logger: ILogger
): Promise<IClickhouseProfile | null> {
logger.debug('Fetching profile from Clickhouse');
const result = await chQuery<IClickhouseProfile>(
@@ -207,7 +207,7 @@ export class ProfileBuffer extends BaseBuffer {
}
GROUP BY id, project_id
ORDER BY created_at DESC
LIMIT 1`,
LIMIT 1`
);
logger.debug('Clickhouse fetch result', {
found: !!result[0],
@@ -221,7 +221,7 @@ export class ProfileBuffer extends BaseBuffer {
const profiles = await this.redis.lrange(
this.redisKey,
0,
this.batchSize - 1,
this.batchSize - 1
);
if (profiles.length === 0) {
@@ -231,7 +231,7 @@ export class ProfileBuffer extends BaseBuffer {
this.logger.debug(`Processing ${profiles.length} profiles in buffer`);
const parsedProfiles = profiles.map((p) =>
getSafeJson<IClickhouseProfile>(p),
getSafeJson<IClickhouseProfile>(p)
);
for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {

7
packages/db/src/services/clients.service.ts
View File

@@ -1,4 +1,4 @@
import { cacheable, cacheableLru } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import type { Client, Prisma } from '../prisma-client';
import { db } from '../prisma-client';
@@ -34,7 +34,4 @@ export async function getClientById(
});
}
export const getClientByIdCached = cacheableLru(getClientById, {
maxSize: 1000,
ttl: 60 * 5,
});
export const getClientByIdCached = cacheable(getClientById, 60 * 5);

16
packages/db/src/services/event.service.ts
View File

@@ -168,7 +168,6 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
device: event.device,
brand: event.brand,
model: event.model,
duration: event.duration,
path: event.path,
origin: event.origin,
referrer: event.referrer,
@@ -216,7 +215,7 @@ export interface IServiceEvent {
device?: string | undefined;
brand?: string | undefined;
model?: string | undefined;
duration: number;
duration?: number;
path: string;
origin: string;
referrer: string | undefined;
@@ -247,7 +246,7 @@ export interface IServiceEventMinimal {
browser?: string | undefined;
device?: string | undefined;
brand?: string | undefined;
duration: number;
duration?: number;
path: string;
origin: string;
referrer: string | undefined;
@@ -379,7 +378,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
device: payload.device ?? '',
brand: payload.brand ?? '',
model: payload.model ?? '',
duration: payload.duration,
duration: payload.duration ?? 0,
referrer: payload.referrer ?? '',
referrer_name: payload.referrerName ?? '',
referrer_type: payload.referrerType ?? '',
@@ -477,7 +476,7 @@ export async function getEventList(options: GetEventListOptions) {
sb.where.cursor = `created_at < ${sqlstring.escape(formatClickhouseDate(cursor))}`;
}
if (!cursor && !(startDate && endDate)) {
if (!(cursor || (startDate && endDate))) {
sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;
}
@@ -562,9 +561,6 @@ export async function getEventList(options: GetEventListOptions) {
if (select.model) {
sb.select.model = 'model';
}
if (select.duration) {
sb.select.duration = 'duration';
}
if (select.path) {
sb.select.path = 'path';
}
@@ -771,7 +767,6 @@ class EventService {
where,
select,
limit,
orderBy,
filters,
}: {
projectId: string;
@@ -811,7 +806,6 @@ class EventService {
select.event.deviceId && 'e.device_id as device_id',
select.event.name && 'e.name as name',
select.event.path && 'e.path as path',
select.event.duration && 'e.duration as duration',
select.event.country && 'e.country as country',
select.event.city && 'e.city as city',
select.event.os && 'e.os as os',
@@ -896,7 +890,6 @@ class EventService {
select.event.deviceId && 'e.device_id as device_id',
select.event.name && 'e.name as name',
select.event.path && 'e.path as path',
select.event.duration && 'e.duration as duration',
select.event.country && 'e.country as country',
select.event.city && 'e.city as city',
select.event.os && 'e.os as os',
@@ -1032,7 +1025,6 @@ class EventService {
id: true,
name: true,
createdAt: true,
duration: true,
country: true,
city: true,
os: true,

2
packages/db/src/services/notification.service.ts
View File

@@ -90,7 +90,7 @@ export const getNotificationRulesByProjectId = cacheable(
},
});
},
60 * 24
60 * 24,
);
function getIntegration(integrationId: string | null) {

35
packages/db/src/services/overview.service.ts
View File

@@ -416,6 +416,30 @@ export class OverviewService {
const where = this.getRawWhereClause('sessions', filters);
const fillConfig = this.getFillConfig(interval, startDate, endDate);
// CTE: per-event screen_view durations via window function
const rawScreenViewDurationsQuery = clix(this.client, timezone)
.select([
`${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
`dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters));
// CTE: avg duration per date bucket
const avgDurationByDateQuery = clix(this.client, timezone)
.select([
'date',
'round(avgIf(duration, duration > 0), 2) / 1000 AS avg_session_duration',
])
.from('raw_screen_view_durations')
.groupBy(['date']);
// Session aggregation with bounce rates
const sessionAggQuery = clix(this.client, timezone)
.select([
@@ -473,6 +497,8 @@ export class OverviewService {
.where('date', '!=', rollupDate)
)
.with('overall_unique_visitors', overallUniqueVisitorsQuery)
.with('raw_screen_view_durations', rawScreenViewDurationsQuery)
.with('avg_duration_by_date', avgDurationByDateQuery)
.select<{
date: string;
bounce_rate: number;
@@ -489,8 +515,7 @@ export class OverviewService {
'dss.bounce_rate as bounce_rate',
'uniq(e.profile_id) AS unique_visitors',
'uniq(e.session_id) AS total_sessions',
'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
'coalesce(dur.avg_session_duration, 0) AS avg_session_duration',
'count(*) AS total_screen_views',
'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
'(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
@@ -502,6 +527,10 @@ export class OverviewService {
'daily_session_stats AS dss',
`${clix.toStartOf('e.created_at', interval as any)} = dss.date`
)
.leftJoin(
'avg_duration_by_date AS dur',
`${clix.toStartOf('e.created_at', interval as any)} = dur.date`
)
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
.where('e.created_at', 'BETWEEN', [
@@ -509,7 +538,7 @@ export class OverviewService {
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters))
.groupBy(['date', 'dss.bounce_rate'])
.groupBy(['date', 'dss.bounce_rate', 'dur.avg_session_duration'])
.orderBy('date', 'ASC')
.fill(fillConfig.from, fillConfig.to, fillConfig.step)
.transform({

28
packages/db/src/services/pages.service.ts
View File

@@ -52,6 +52,24 @@ export class PagesService {
.where('created_at', '>=', clix.exp('now() - INTERVAL 30 DAY'))
.groupBy(['origin', 'path']);
// CTE: compute screen_view durations via window function (leadInFrame gives next event's timestamp)
const screenViewDurationsCte = clix(this.client, timezone)
.select([
'project_id',
'session_id',
'path',
'origin',
`dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
])
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('path', '!=', '')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
]);
// Pre-filtered sessions subquery for better performance
const sessionsSubquery = clix(this.client, timezone)
.select(['id', 'project_id', 'is_bounce'])
@@ -66,6 +84,7 @@ export class PagesService {
// Main query: aggregate events and calculate bounce rate from pre-filtered sessions
const query = clix(this.client, timezone)
.with('page_titles', titlesCte)
.with('screen_view_durations', screenViewDurationsCte)
.select<ITopPage>([
'e.origin as origin',
'e.path as path',
@@ -79,20 +98,13 @@ export class PagesService {
2
) as bounce_rate`,
])
.from(`${TABLE_NAMES.events} e`, false)
.from('screen_view_durations e', false)
.leftJoin(
sessionsSubquery,
'e.session_id = s.id AND e.project_id = s.project_id',
's'
)
.leftJoin('page_titles pt', 'concat(e.origin, e.path) = pt.page_key')
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
.where('e.path', '!=', '')
.where('e.created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.when(!!search, (q) => {
const term = `%${search}%`;
q.whereGroup()

9
packages/db/src/services/project.service.ts
View File

@@ -1,6 +1,6 @@
import { cacheable } from '@openpanel/redis';
import sqlstring from 'sqlstring';
import { TABLE_NAMES, chQuery } from '../clickhouse/client';
import { chQuery, TABLE_NAMES } from '../clickhouse/client';
import type { Prisma, Project } from '../prisma-client';
import { db } from '../prisma-client';
@@ -25,6 +25,7 @@ export async function getProjectById(id: string) {
return res;
}
/** L1 LRU (60s) + L2 Redis. clear() invalidates Redis + local LRU; other nodes may serve stale from LRU for up to 60s. */
export const getProjectByIdCached = cacheable(getProjectById, 60 * 60 * 24);
export async function getProjectWithClients(id: string) {
@@ -44,7 +45,7 @@ export async function getProjectWithClients(id: string) {
return res;
}
export async function getProjectsByOrganizationId(organizationId: string) {
export function getProjectsByOrganizationId(organizationId: string) {
return db.project.findMany({
where: {
organizationId,
@@ -95,7 +96,7 @@ export async function getProjects({
if (access.length > 0) {
return projects.filter((project) =>
access.some((a) => a.projectId === project.id),
access.some((a) => a.projectId === project.id)
);
}
@@ -104,7 +105,7 @@ export async function getProjects({
export const getProjectEventsCount = async (projectId: string) => {
const res = await chQuery<{ count: number }>(
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`,
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`
);
return res[0]?.count;
};

9
packages/db/src/services/salt.service.ts
View File

@@ -1,9 +1,9 @@
import { generateSalt } from '@openpanel/common/server';
import { cacheableLru } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import { db } from '../prisma-client';
export const getSalts = cacheableLru(
export const getSalts = cacheable(
'op:salt',
async () => {
const [curr, prev] = await db.salt.findMany({
@@ -24,10 +24,7 @@ export const getSalts = cacheableLru(
return salts;
},
{
maxSize: 2,
ttl: 60 * 5,
},
60 * 5,
);
export async function createInitialSalts() {

221
packages/payments/scripts/assign-product-to-org.ts
View File

@@ -1,221 +0,0 @@
import { db } from '@openpanel/db';
import { Polar } from '@polar-sh/sdk';
import inquirer from 'inquirer';
import inquirerAutocomplete from 'inquirer-autocomplete-prompt';
import { getSuccessUrl } from '..';
// Register the autocomplete prompt
inquirer.registerPrompt('autocomplete', inquirerAutocomplete);
interface Answers {
isProduction: boolean;
polarApiKey: string;
productId: string;
organizationId: string;
}
async function promptForInput() {
// Get all organizations first
const organizations = await db.organization.findMany({
select: {
id: true,
name: true,
},
});
// Step 1: Collect Polar credentials first
const polarCredentials = await inquirer.prompt<{
isProduction: boolean;
polarApiKey: string;
polarOrganizationId: string;
}>([
{
type: 'list',
name: 'isProduction',
message: 'Is this for production?',
choices: [
{ name: 'Yes', value: true },
{ name: 'No', value: false },
],
default: true,
},
{
type: 'string',
name: 'polarApiKey',
message: 'Enter your Polar API key:',
validate: (input: string) => {
if (!input) return 'API key is required';
return true;
},
},
]);
// Step 2: Initialize Polar client and fetch products
const polar = new Polar({
accessToken: polarCredentials.polarApiKey,
server: polarCredentials.isProduction ? 'production' : 'sandbox',
});
console.log('Fetching products from Polar...');
const productsResponse = await polar.products.list({
limit: 100,
isArchived: false,
sorting: ['price_amount'],
});
const products = productsResponse.result.items;
if (products.length === 0) {
throw new Error('No products found in Polar');
}
// Step 3: Continue with product selection and organization selection
const restOfAnswers = await inquirer.prompt<{
productId: string;
organizationId: string;
}>([
{
type: 'autocomplete',
name: 'productId',
message: 'Select product:',
source: (answersSoFar: any, input = '') => {
return products
.filter(
(product) =>
product.name.toLowerCase().includes(input.toLowerCase()) ||
product.id.toLowerCase().includes(input.toLowerCase()),
)
.map((product) => {
const price = product.prices?.[0];
const priceStr =
price && 'priceAmount' in price && price.priceAmount
? `$${(price.priceAmount / 100).toFixed(2)}/${price.recurringInterval || 'month'}`
: 'No price';
return {
name: `${product.name} (${priceStr})`,
value: product.id,
};
});
},
},
{
type: 'autocomplete',
name: 'organizationId',
message: 'Select organization:',
source: (answersSoFar: any, input = '') => {
return organizations
.filter(
(org) =>
org.name.toLowerCase().includes(input.toLowerCase()) ||
org.id.toLowerCase().includes(input.toLowerCase()),
)
.map((org) => ({
name: `${org.name} (${org.id})`,
value: org.id,
}));
},
},
]);
return {
...polarCredentials,
...restOfAnswers,
};
}
async function main() {
console.log('Assigning existing product to organization...');
const input = await promptForInput();
const polar = new Polar({
accessToken: input.polarApiKey,
server: input.isProduction ? 'production' : 'sandbox',
});
const organization = await db.organization.findUniqueOrThrow({
where: {
id: input.organizationId,
},
select: {
id: true,
name: true,
createdBy: {
select: {
id: true,
email: true,
firstName: true,
lastName: true,
},
},
projects: {
select: {
id: true,
},
},
},
});
if (!organization.createdBy) {
throw new Error(
`Organization ${organization.name} does not have a creator. Cannot proceed.`,
);
}
const user = organization.createdBy;
// Fetch product details for review
const product = await polar.products.get({ id: input.productId });
const price = product.prices?.[0];
const priceStr =
price && 'priceAmount' in price && price.priceAmount
? `$${(price.priceAmount / 100).toFixed(2)}/${price.recurringInterval || 'month'}`
: 'No price';
console.log('\nReview the following settings:');
console.table({
product: product.name,
price: priceStr,
organization: organization.name,
email: user.email,
name:
[user.firstName, user.lastName].filter(Boolean).join(' ') || 'No name',
});
const { confirmed } = await inquirer.prompt([
{
type: 'confirm',
name: 'confirmed',
message: 'Do you want to proceed?',
default: false,
},
]);
if (!confirmed) {
console.log('Operation canceled');
return;
}
const checkoutLink = await polar.checkoutLinks.create({
paymentProcessor: 'stripe',
productId: input.productId,
allowDiscountCodes: false,
metadata: {
organizationId: organization.id,
userId: user.id,
},
successUrl: getSuccessUrl(
input.isProduction
? 'https://dashboard.openpanel.dev'
: 'http://localhost:3000',
organization.id,
),
});
console.log('\nCheckout link created:');
console.table(checkoutLink);
console.log('\nProduct assigned successfully!');
}
main()
.catch(console.error)
.finally(() => db.$disconnect());

9
packages/queue/src/queues.ts
View File

@@ -6,7 +6,7 @@ import type {
} from '@openpanel/db';
import { createLogger } from '@openpanel/logger';
import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
import { Queue, QueueEvents } from 'bullmq';
import { Queue } from 'bullmq';
import { Queue as GroupQueue } from 'groupmq';
import type { ITrackPayload } from '../../validation';
@@ -66,6 +66,10 @@ export interface EventsQueuePayloadIncomingEvent {
headers: Record<string, string | undefined>;
deviceId: string;
sessionId: string;
session?: Pick<
IServiceCreateEventPayload,
'referrer' | 'referrerName' | 'referrerType'
>;
};
}
export interface EventsQueuePayloadCreateEvent {
@@ -206,9 +210,6 @@ export const sessionsQueue = new Queue<SessionsQueuePayload>(
},
}
);
export const sessionsQueueEvents = new QueueEvents(getQueueName('sessions'), {
connection: getRedisQueue(),
});
export const cronQueue = new Queue<CronQueuePayload>(getQueueName('cron'), {
connection: getRedisQueue(),

249
packages/redis/cachable.ts
View File

@@ -1,7 +1,7 @@
import { LRUCache } from 'lru-cache';
import { getRedisCache } from './redis';
export const deleteCache = async (key: string) => {
export const deleteCache = (key: string) => {
return getRedisCache().del(key);
};
@@ -15,7 +15,7 @@ export async function getCache<T>(
key: string,
expireInSec: number,
fn: () => Promise<T>,
useLruCache?: boolean,
useLruCache?: boolean
): Promise<T> {
// L1 Cache: Check global LRU cache first (in-memory, instant)
if (useLruCache) {
@@ -28,15 +28,7 @@ export async function getCache<T>(
// L2 Cache: Check Redis cache (shared across instances)
const hit = await getRedisCache().get(key);
if (hit) {
const parsed = JSON.parse(hit, (_, value) => {
if (
typeof value === 'string' &&
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/.test(value)
) {
return new Date(value);
}
return value;
});
const parsed = parseCache(hit);
// Store in LRU cache for next time
if (useLruCache) {
@@ -81,12 +73,24 @@ export function getGlobalLruCacheStats() {
}
function stringify(obj: unknown): string {
if (obj === null) return 'null';
if (obj === undefined) return 'undefined';
if (typeof obj === 'boolean') return obj ? 'true' : 'false';
if (typeof obj === 'number') return String(obj);
if (typeof obj === 'string') return obj;
if (typeof obj === 'function') return obj.toString();
if (obj === null) {
return 'null';
}
if (obj === undefined) {
return 'undefined';
}
if (typeof obj === 'boolean') {
return obj ? 'true' : 'false';
}
if (typeof obj === 'number') {
return String(obj);
}
if (typeof obj === 'string') {
return obj;
}
if (typeof obj === 'function') {
return obj.toString();
}
if (Array.isArray(obj)) {
return `[${obj.map(stringify).join(',')}]`;
@@ -128,17 +132,29 @@ function hasResult(result: unknown): boolean {
return true;
}
export interface CacheableLruOptions {
/** TTL in seconds for LRU cache */
ttl: number;
/** Maximum number of entries in LRU cache */
maxSize?: number;
}
const DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/;
const parseCache = (cached: string) => {
try {
return JSON.parse(cached, (_, value) => {
if (typeof value === 'string' && DATE_REGEX.test(value)) {
return new Date(value);
}
return value;
});
} catch (error) {
console.error('Failed to parse cache', error);
return null;
}
};
// L1 cache: short TTL to offload Redis; clear() invalidates Redis, other nodes may serve stale from LRU for up to this long
const CACHEABLE_LRU_TTL_MS = 60 * 1000; // 60 seconds
const CACHEABLE_LRU_MAX = 1000;
// Overload 1: cacheable(fn, expireInSec)
export function cacheable<T extends (...args: any) => any>(
fn: T,
expireInSec: number,
expireInSec: number
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => Promise<number>;
@@ -151,7 +167,7 @@ export function cacheable<T extends (...args: any) => any>(
export function cacheable<T extends (...args: any) => any>(
name: string,
fn: T,
expireInSec: number,
expireInSec: number
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => Promise<number>;
@@ -164,7 +180,7 @@ export function cacheable<T extends (...args: any) => any>(
export function cacheable<T extends (...args: any) => any>(
fnOrName: T | string,
fnOrExpireInSec: number | T,
_expireInSec?: number,
_expireInSec?: number
) {
const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
const fn =
@@ -195,184 +211,67 @@ export function cacheable<T extends (...args: any) => any>(
const cachePrefix = `cachable:${name}`;
const getKey = (...args: Parameters<T>) =>
`${cachePrefix}:${stringify(args)}`;
`${cachePrefix}:${stringify(args)}`.replaceAll(/\s/g, '');
// Redis-only mode: asynchronous implementation
const lruCache = new LRUCache<string, any>({
max: CACHEABLE_LRU_MAX,
ttl: CACHEABLE_LRU_TTL_MS,
});
// L1 LRU (60s) + L2 Redis. clear() deletes Redis + local LRU; other nodes may serve stale from LRU for up to 60s.
const cachedFn = async (
...args: Parameters<T>
): Promise<Awaited<ReturnType<T>>> => {
const key = getKey(...args);
// Check Redis cache (shared across instances)
// L1: in-memory LRU first (offloads Redis on hot keys)
const lruHit = lruCache.get(key);
if (lruHit !== undefined && hasResult(lruHit)) {
return lruHit as Awaited<ReturnType<T>>;
}
// L2: Redis (shared across instances)
const cached = await getRedisCache().get(key);
if (cached) {
try {
const parsed = JSON.parse(cached, (_, value) => {
if (
typeof value === 'string' &&
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/.test(value)
) {
return new Date(value);
}
return value;
});
if (hasResult(parsed)) {
return parsed;
}
} catch (e) {
console.error('Failed to parse cache', e);
const parsed = parseCache(cached);
if (hasResult(parsed)) {
lruCache.set(key, parsed);
return parsed;
}
}
// Cache miss: Execute function
// Cache miss: execute function
const result = await fn(...(args as any));
if (hasResult(result)) {
// Don't await Redis write - fire and forget for better performance
lruCache.set(key, result);
getRedisCache()
.setex(key, expireInSec, JSON.stringify(result))
.catch(() => {});
.catch(() => {
// ignore error
});
}
return result;
};
cachedFn.getKey = getKey;
cachedFn.clear = async (...args: Parameters<T>) => {
const key = getKey(...args);
return getRedisCache().del(key);
};
cachedFn.set =
(...args: Parameters<T>) =>
async (payload: Awaited<ReturnType<T>>) => {
const key = getKey(...args);
return getRedisCache()
.setex(key, expireInSec, JSON.stringify(payload))
.catch(() => {});
};
return cachedFn;
}
// Overload 1: cacheableLru(fn, options)
export function cacheableLru<T extends (...args: any) => any>(
fn: T,
options: CacheableLruOptions,
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => boolean;
set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
};
// Overload 2: cacheableLru(name, fn, options)
export function cacheableLru<T extends (...args: any) => any>(
name: string,
fn: T,
options: CacheableLruOptions,
): T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => boolean;
set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
};
// Implementation for cacheableLru (LRU-only - synchronous)
export function cacheableLru<T extends (...args: any) => any>(
fnOrName: T | string,
fnOrOptions: T | CacheableLruOptions,
_options?: CacheableLruOptions,
) {
const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
const fn =
typeof fnOrName === 'function'
? fnOrName
: typeof fnOrOptions === 'function'
? fnOrOptions
: null;
let options: CacheableLruOptions;
// Parse parameters based on function signature
if (typeof fnOrName === 'function') {
// Overload 1: cacheableLru(fn, options)
options =
typeof fnOrOptions === 'object' && fnOrOptions !== null
? fnOrOptions
: ({} as CacheableLruOptions);
} else {
// Overload 2: cacheableLru(name, fn, options)
options =
typeof _options === 'object' && _options !== null
? _options
: ({} as CacheableLruOptions);
}
if (typeof fn !== 'function') {
throw new Error('fn is not a function');
}
if (typeof options.ttl !== 'number') {
throw new Error('options.ttl is required and must be a number');
}
const cachePrefix = `cachable:${name}`;
const getKey = (...args: Parameters<T>) =>
`${cachePrefix}:${stringify(args)}`;
const maxSize = options.maxSize ?? 1000;
const ttl = options.ttl;
// Create function-specific LRU cache
const functionLruCache = new LRUCache<string, any>({
max: maxSize,
ttl: ttl * 1000, // Convert seconds to milliseconds for LRU
});
// LRU-only mode: synchronous implementation (or returns promise if fn is async)
const cachedFn = ((...args: Parameters<T>): ReturnType<T> => {
const key = getKey(...args);
// Check LRU cache
const lruHit = functionLruCache.get(key);
if (lruHit !== undefined && hasResult(lruHit)) {
return lruHit as ReturnType<T>;
}
// Cache miss: Execute function
const result = fn(...(args as any)) as ReturnType<T>;
// If result is a Promise, handle it asynchronously but cache the resolved value
if (result && typeof (result as any).then === 'function') {
return (result as Promise<any>).then((resolved: any) => {
if (hasResult(resolved)) {
functionLruCache.set(key, resolved);
}
return resolved;
}) as ReturnType<T>;
}
// Synchronous result: cache and return
if (hasResult(result)) {
functionLruCache.set(key, result);
}
return result;
}) as T & {
getKey: (...args: Parameters<T>) => string;
clear: (...args: Parameters<T>) => boolean;
set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
};
cachedFn.getKey = getKey;
cachedFn.clear = (...args: Parameters<T>) => {
const key = getKey(...args);
return functionLruCache.delete(key);
lruCache.delete(key);
return getRedisCache().del(key);
};
cachedFn.set =
(...args: Parameters<T>) =>
(payload: ReturnType<T>) => {
(payload: Awaited<ReturnType<T>>) => {
const key = getKey(...args);
if (hasResult(payload)) {
functionLruCache.set(key, payload);
lruCache.set(key, payload);
return getRedisCache()
.setex(key, expireInSec, JSON.stringify(payload))
.catch(() => {
// ignore error
});
}
};

3
packages/redis/publisher.ts
View File

@@ -10,8 +10,7 @@ export type IPublishChannels = {
};
};
events: {
received: IServiceEvent;
saved: IServiceEvent;
batch: { projectId: string; count: number };
};
notification: {
created: Prisma.NotificationUncheckedCreateInput;

13
packages/sdks/react-native/index.ts
View File

@@ -1,13 +1,13 @@
import type { OpenPanelOptions, TrackProperties } from '@openpanel/sdk';
import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
import * as Application from 'expo-application';
import Constants from 'expo-constants';
import { AppState, Platform } from 'react-native';
import type { OpenPanelOptions, TrackProperties } from '@openpanel/sdk';
import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
export * from '@openpanel/sdk';
export class OpenPanel extends OpenPanelBase {
private lastPath = '';
constructor(public options: OpenPanelOptions) {
super({
...options,
@@ -37,7 +37,12 @@ export class OpenPanel extends OpenPanelBase {
});
}
public screenView(route: string, properties?: TrackProperties): void {
track(name: string, properties?: TrackProperties) {
return super.track(name, { ...properties, __path: this.lastPath });
}
screenView(route: string, properties?: TrackProperties): void {
this.lastPath = route;
super.track('screen_view', {
...properties,
__path: route,

21
packages/sdks/web/src/index.ts
View File

@@ -58,7 +58,7 @@ export type OpenPanelOptions = OpenPanelBaseOptions & {
function toCamelCase(str: string) {
return str.replace(/([-_][a-z])/gi, ($1) =>
$1.toUpperCase().replace('-', '').replace('_', ''),
$1.toUpperCase().replace('-', '').replace('_', '')
);
}
@@ -114,7 +114,9 @@ export class OpenPanel extends OpenPanelBase {
const sampled = Math.random() < sampleRate;
if (sampled) {
this.loadReplayModule().then((mod) => {
if (!mod) return;
if (!mod) {
return;
}
mod.startReplayRecorder(this.options.sessionReplay!, (chunk) => {
// Replay chunks go through send() and are queued when disabled or waitForProfile
// until ready() is called (base SDK also queues replay until sessionId is set).
@@ -153,7 +155,10 @@ export class OpenPanel extends OpenPanelBase {
// dead-code-eliminated in the library build.
if (typeof __OPENPANEL_REPLAY_URL__ !== 'undefined') {
const scriptEl = _replayScriptRef;
const url = this.options.sessionReplay?.scriptUrl || scriptEl?.src?.replace('.js', '-replay.js') || 'https://openpanel.dev/op1-replay.js';
const url =
this.options.sessionReplay?.scriptUrl ||
scriptEl?.src?.replace('.js', '-replay.js') ||
'https://openpanel.dev/op1-replay.js';
// Already loaded (e.g. user included the script manually)
if ((window as any).__openpanel_replay) {
@@ -287,11 +292,15 @@ export class OpenPanel extends OpenPanelBase {
});
}
track(name: string, properties?: TrackProperties) {
return super.track(name, { ...properties, __path: this.lastPath });
}
screenView(properties?: TrackProperties): void;
screenView(path: string, properties?: TrackProperties): void;
screenView(
pathOrProperties?: string | TrackProperties,
propertiesOrUndefined?: TrackProperties,
propertiesOrUndefined?: TrackProperties
): void {
if (this.isServer()) {
return;
@@ -322,7 +331,7 @@ export class OpenPanel extends OpenPanelBase {
async flushRevenue() {
const promises = this.pendingRevenues.map((pending) =>
super.revenue(pending.amount, pending.properties),
super.revenue(pending.amount, pending.properties)
);
await Promise.all(promises);
this.clearRevenue();
@@ -343,7 +352,7 @@ export class OpenPanel extends OpenPanelBase {
try {
sessionStorage.setItem(
'openpanel-pending-revenues',
JSON.stringify(this.pendingRevenues),
JSON.stringify(this.pendingRevenues)
);
} catch {}
}

114
packages/trpc/src/routers/chart.ts
View File

@@ -1,11 +1,7 @@
import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import { round } from '@openpanel/common';
import {
type IClickhouseProfile,
type IServiceProfile,
TABLE_NAMES,
AggregateChartEngine,
ChartEngine,
ch,
chQuery,
clix,
@@ -21,8 +17,11 @@ import {
getReportById,
getSelectPropertyKey,
getSettingsForProject,
type IClickhouseProfile,
type IServiceProfile,
onlyReportEvents,
sankeyService,
TABLE_NAMES,
validateShareAccess,
} from '@openpanel/db';
import {
@@ -33,15 +32,15 @@ import {
zReportInput,
zTimeInterval,
} from '@openpanel/validation';
import { round } from '@openpanel/common';
import { AggregateChartEngine, ChartEngine } from '@openpanel/db';
import {
differenceInDays,
differenceInMonths,
differenceInWeeks,
formatISO,
} from 'date-fns';
import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import { getProjectAccess } from '../access';
import { TRPCAccessError } from '../errors';
import {
@@ -83,7 +82,7 @@ const chartProcedure = publicProcedure.use(
session: ctx.session?.userId
? { userId: ctx.session.userId }
: undefined,
},
}
);
if (!shareValidation.isValid) {
throw TRPCAccessError('You do not have access to this share');
@@ -119,7 +118,7 @@ const chartProcedure = publicProcedure.use(
report: null,
},
});
},
}
);
export const chartRouter = createTRPCRouter({
@@ -128,7 +127,7 @@ export const chartRouter = createTRPCRouter({
.input(
z.object({
projectId: z.string(),
}),
})
)
.query(async ({ input: { projectId } }) => {
const { timezone } = await getSettingsForProject(projectId);
@@ -151,7 +150,7 @@ export const chartRouter = createTRPCRouter({
TO toStartOfDay(now())
STEP INTERVAL 1 day
SETTINGS session_timezone = '${timezone}'
`,
`
);
const metricsPromise = clix(ch, timezone)
@@ -185,7 +184,7 @@ export const chartRouter = createTRPCRouter({
? Math.round(
((metrics.months_3 - metrics.months_3_prev) /
metrics.months_3_prev) *
100,
100
)
: null;
@@ -209,12 +208,12 @@ export const chartRouter = createTRPCRouter({
.input(
z.object({
projectId: z.string(),
}),
})
)
.query(async ({ input: { projectId } }) => {
const [events, meta] = await Promise.all([
chQuery<{ name: string; count: number }>(
`SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`,
`SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`
),
getEventMetasCached(projectId),
]);
@@ -238,7 +237,7 @@ export const chartRouter = createTRPCRouter({
z.object({
event: z.string().optional(),
projectId: z.string(),
}),
})
)
.query(async ({ input: { projectId, event } }) => {
const profiles = await clix(ch, 'UTC')
@@ -252,8 +251,8 @@ export const chartRouter = createTRPCRouter({
const profileProperties = [
...new Set(
profiles.flatMap((p) =>
Object.keys(p.properties).map((k) => `profile.properties.${k}`),
),
Object.keys(p.properties).map((k) => `profile.properties.${k}`)
)
),
];
@@ -283,7 +282,6 @@ export const chartRouter = createTRPCRouter({
});
const fixedProperties = [
'duration',
'revenue',
'has_profile',
'path',
@@ -316,7 +314,7 @@ export const chartRouter = createTRPCRouter({
return pipe(
sort<string>((a, b) => a.length - b.length),
uniq,
uniq
)(properties);
}),
@@ -326,9 +324,9 @@ export const chartRouter = createTRPCRouter({
event: z.string(),
property: z.string(),
projectId: z.string(),
}),
})
)
.query(async ({ input: { event, property, projectId, ...input } }) => {
.query(async ({ input: { event, property, projectId } }) => {
if (property === 'has_profile') {
return {
values: ['true', 'false'],
@@ -378,7 +376,7 @@ export const chartRouter = createTRPCRouter({
.from(TABLE_NAMES.profiles)
.where('project_id', '=', projectId),
'profile.id = profile_id',
'profile',
'profile'
);
}
@@ -389,8 +387,8 @@ export const chartRouter = createTRPCRouter({
(data: typeof events) => map(prop('values'), data),
flatten,
uniq,
sort((a, b) => a.length - b.length),
)(events),
sort((a, b) => a.length - b.length)
)(events)
);
}
@@ -406,8 +404,8 @@ export const chartRouter = createTRPCRouter({
z.object({
shareId: z.string().optional(),
id: z.string().optional(),
}),
),
})
)
)
.query(async ({ input, ctx }) => {
const chartInput = ctx.report
@@ -448,8 +446,8 @@ export const chartRouter = createTRPCRouter({
z.object({
shareId: z.string().optional(),
id: z.string().optional(),
}),
),
})
)
)
.query(async ({ input, ctx }) => {
const chartInput = ctx.report
@@ -536,12 +534,10 @@ export const chartRouter = createTRPCRouter({
z.object({
shareId: z.string().optional(),
id: z.string().optional(),
}),
),
})
)
)
.query(async ({ input, ctx }) => {
console.log('input', input);
.query(({ input, ctx }) => {
const chartInput = ctx.report
? {
...ctx.report,
@@ -562,10 +558,10 @@ export const chartRouter = createTRPCRouter({
z.object({
shareId: z.string().optional(),
id: z.string().optional(),
}),
),
})
)
)
.query(async ({ input, ctx }) => {
.query(({ input, ctx }) => {
const chartInput = ctx.report
? {
...ctx.report,
@@ -593,7 +589,7 @@ export const chartRouter = createTRPCRouter({
range: zRange,
shareId: z.string().optional(),
id: z.string().optional(),
}),
})
)
.query(async ({ input, ctx }) => {
const projectId = ctx.report?.projectId ?? input.projectId;
@@ -647,7 +643,7 @@ export const chartRouter = createTRPCRouter({
startDate,
endDate,
},
timezone,
timezone
);
const diffInterval = {
minute: () => differenceInDays(dates.endDate, dates.startDate),
@@ -677,14 +673,14 @@ export const chartRouter = createTRPCRouter({
const usersSelect = range(0, diffInterval + 1)
.map(
(index) =>
`groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`,
`groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`
)
.join(',\n');
const countsSelect = range(0, diffInterval + 1)
.map(
(index) =>
`length(interval_${index}_users) AS interval_${index}_user_count`,
`length(interval_${index}_users) AS interval_${index}_user_count`
)
.join(',\n');
@@ -769,12 +765,10 @@ export const chartRouter = createTRPCRouter({
interval: zTimeInterval.default('day'),
series: zChartSeries,
breakdowns: z.record(z.string(), z.string()).optional(),
}),
})
)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const { projectId, date, series } = input;
const limit = 100;
const serie = series[0];
if (!serie) {
@@ -813,7 +807,7 @@ export const chartRouter = createTRPCRouter({
if (profileFields.length > 0) {
// Extract top-level field names and select only what's needed
const fieldsToSelect = uniq(
profileFields.map((f) => f.split('.')[0]),
profileFields.map((f) => f.split('.')[0])
).join(', ');
sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
@@ -836,7 +830,7 @@ export const chartRouter = createTRPCRouter({
// Fetch profile details in batches to avoid exceeding ClickHouse max_query_size
const ids = profileIds.map((p) => p.profile_id).filter(Boolean);
const BATCH_SIZE = 200;
const profiles = [];
const profiles: IServiceProfile[] = [];
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
const batch = ids.slice(i, i + BATCH_SIZE);
const batchProfiles = await getProfilesCached(batch, projectId);
@@ -859,13 +853,13 @@ export const chartRouter = createTRPCRouter({
.optional()
.default(false)
.describe(
'If true, show users who dropped off at this step. If false, show users who completed at least this step.',
'If true, show users who dropped off at this step. If false, show users who completed at least this step.'
),
funnelWindow: z.number().optional(),
funnelGroup: z.string().optional(),
breakdowns: z.array(z.object({ name: z.string() })).optional(),
range: zRange,
}),
})
)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
@@ -911,15 +905,15 @@ export const chartRouter = createTRPCRouter({
// Check for profile filters and add profile join if needed
const profileFilters = funnelService.getProfileFilters(
eventSeries as IChartEvent[],
eventSeries as IChartEvent[]
);
if (profileFilters.length > 0) {
const fieldsToSelect = uniq(
profileFilters.map((f) => f.split('.')[0]),
profileFilters.map((f) => f.split('.')[0])
).join(', ');
funnelCte.leftJoin(
`(SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile`,
'profile.id = events.profile_id',
'profile.id = events.profile_id'
);
}
@@ -934,7 +928,7 @@ export const chartRouter = createTRPCRouter({
// `max(level) AS level` alias (ILLEGAL_AGGREGATION error).
query.with(
'funnel',
'SELECT profile_id, max(level) AS level FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id',
'SELECT profile_id, max(level) AS level FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id'
);
} else {
// For session grouping: filter out level = 0 inside the CTE
@@ -969,7 +963,7 @@ export const chartRouter = createTRPCRouter({
// when there are many profile IDs to pass in the IN(...) clause
const ids = profileIdsResult.map((p) => p.profile_id).filter(Boolean);
const BATCH_SIZE = 500;
const profiles = [];
const profiles: IServiceProfile[] = [];
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
const batch = ids.slice(i, i + BATCH_SIZE);
const batchProfiles = await getProfilesCached(batch, projectId);
@@ -986,7 +980,7 @@ function processCohortData(
total_first_event_count: number;
[key: string]: any;
}>,
diffInterval: number,
diffInterval: number
) {
if (data.length === 0) {
return [];
@@ -995,13 +989,13 @@ function processCohortData(
const processed = data.map((row) => {
const sum = row.total_first_event_count;
const values = range(0, diffInterval + 1).map(
(index) => (row[`interval_${index}_user_count`] || 0) as number,
(index) => (row[`interval_${index}_user_count`] || 0) as number
);
return {
cohort_interval: row.cohort_interval,
sum,
values: values,
values,
percentages: values.map((value) => (sum > 0 ? round(value / sum, 2) : 0)),
};
});
@@ -1041,10 +1035,10 @@ function processCohortData(
cohort_interval: 'Weighted Average',
sum: round(averageData.totalSum / processed.length, 0),
percentages: averageData.percentages.map(({ sum, weightedSum }) =>
sum > 0 ? round(weightedSum / sum, 2) : 0,
sum > 0 ? round(weightedSum / sum, 2) : 0
),
values: averageData.values.map(({ sum, weightedSum }) =>
sum > 0 ? round(weightedSum / sum, 0) : 0,
sum > 0 ? round(weightedSum / sum, 0) : 0
),
};

4
packages/trpc/src/routers/project.ts
View File

@@ -96,9 +96,7 @@ export const projectRouter = createTRPCRouter({
});
await Promise.all([
getProjectByIdCached.clear(input.id),
res.clients.map((client) => {
getClientByIdCached.clear(client.id);
}),
...res.clients.map((client) => getClientByIdCached.clear(client.id)),
]);
return res;
}),

50
packages/trpc/src/routers/realtime.ts
View File

@@ -1,18 +1,15 @@
import { z } from 'zod';
import {
type EventMeta,
TABLE_NAMES,
ch,
chQuery,
clix,
db,
formatClickhouseDate,
getEventList,
type IClickhouseEvent,
TABLE_NAMES,
transformEvent,
} from '@openpanel/db';
import { subMinutes } from 'date-fns';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import { createTRPCRouter, protectedProcedure } from '../trpc';
export const realtimeRouter = createTRPCRouter({
@@ -25,7 +22,7 @@ export const realtimeRouter = createTRPCRouter({
long: number;
lat: number;
}>(
`SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`,
`SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
);
return res;
@@ -33,25 +30,18 @@ export const realtimeRouter = createTRPCRouter({
activeSessions: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ input }) => {
return getEventList({
projectId: input.projectId,
take: 30,
select: {
name: true,
path: true,
origin: true,
referrer: true,
referrerName: true,
referrerType: true,
country: true,
device: true,
os: true,
browser: true,
createdAt: true,
profile: true,
meta: true,
},
});
const rows = await chQuery<IClickhouseEvent>(
`SELECT
name, session_id, created_at, path, origin, referrer, referrer_name,
country, city, region, os, os_version, browser, browser_version,
device
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(input.projectId)}
AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}'
ORDER BY created_at DESC
LIMIT 50`
);
return rows.map(transformEvent);
}),
paths: protectedProcedure
.input(z.object({ projectId: z.string() }))
@@ -76,7 +66,7 @@ export const realtimeRouter = createTRPCRouter({
.where(
'created_at',
'>=',
formatClickhouseDate(subMinutes(new Date(), 30)),
formatClickhouseDate(subMinutes(new Date(), 30))
)
.groupBy(['path', 'origin'])
.orderBy('count', 'DESC')
@@ -106,7 +96,7 @@ export const realtimeRouter = createTRPCRouter({
.where(
'created_at',
'>=',
formatClickhouseDate(subMinutes(new Date(), 30)),
formatClickhouseDate(subMinutes(new Date(), 30))
)
.groupBy(['referrer_name'])
.orderBy('count', 'DESC')
@@ -137,7 +127,7 @@ export const realtimeRouter = createTRPCRouter({
.where(
'created_at',
'>=',
formatClickhouseDate(subMinutes(new Date(), 30)),
formatClickhouseDate(subMinutes(new Date(), 30))
)
.groupBy(['country', 'city'])
.orderBy('count', 'DESC')

20
pnpm-lock.yaml generated
View File

@@ -16,8 +16,8 @@ catalogs:
specifier: ^19.2.3
version: 19.2.3
groupmq:
specifier: 1.1.1-next.2
version: 1.1.1-next.2
specifier: 2.0.0-next.1
version: 2.0.0-next.1
react:
specifier: ^19.2.3
version: 19.2.3
@@ -198,7 +198,7 @@ importers:
version: 5.0.0
groupmq:
specifier: 'catalog:'
version: 1.1.1-next.2(ioredis@5.8.2)
version: 2.0.0-next.1(ioredis@5.8.2)
jsonwebtoken:
specifier: ^9.0.2
version: 9.0.2
@@ -936,7 +936,7 @@ importers:
version: 4.18.2
groupmq:
specifier: 'catalog:'
version: 1.1.1-next.2(ioredis@5.8.2)
version: 2.0.0-next.1(ioredis@5.8.2)
prom-client:
specifier: ^15.1.3
version: 15.1.3
@@ -1419,7 +1419,7 @@ importers:
version: 5.63.0
groupmq:
specifier: 'catalog:'
version: 1.1.1-next.2(ioredis@5.8.2)
version: 2.0.0-next.1(ioredis@5.8.2)
devDependencies:
'@openpanel/tsconfig':
specifier: workspace:*
@@ -13157,11 +13157,11 @@ packages:
glob@7.1.6:
resolution: {integrity: sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==}
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
deprecated: Glob versions prior to v9 are no longer supported
glob@7.2.3:
resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
deprecated: Glob versions prior to v9 are no longer supported
glob@9.3.5:
resolution: {integrity: sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q==}
@@ -13221,8 +13221,8 @@ packages:
resolution: {integrity: sha512-5gghUc24tP9HRznNpV2+FIoq3xKkj5dTQqf4v0CpdPbFVwFkWoxOM+o+2OC9ZSvjEMTjfmG9QT+gcvggTwW1zw==}
engines: {node: '>= 10.x'}
groupmq@1.1.1-next.2:
resolution: {integrity: sha512-5gH+P3NfSCjfCLcB2g2TAHCpmQz+rwrQkb+kAyrzB9puZuAHKQVYOUPWKVBRFjY7B9jPRGHrimDO6h9rWKGfMA==}
groupmq@2.0.0-next.1:
resolution: {integrity: sha512-xcpz29HeXXn0yP/sQTGPPNMLQAZCCrJg3x9kpOAFbtsXki5KVeBsY3mWNBt3Z+YCa9OxwkTFL6tOcrB67z127A==}
engines: {node: '>=18'}
peerDependencies:
ioredis: '>=5'
@@ -34142,7 +34142,7 @@ snapshots:
graphql@15.8.0: {}
groupmq@1.1.1-next.2(ioredis@5.8.2):
groupmq@2.0.0-next.1(ioredis@5.8.2):
dependencies:
cron-parser: 4.9.0
ioredis: 5.8.2

2
pnpm-workspace.yaml
View File

@@ -13,4 +13,4 @@ catalog:
"@types/react-dom": ^19.2.3
"@types/node": ^24.7.1
typescript: ^5.9.3
groupmq: 1.1.1-next.2
groupmq: 2.0.0-next.1