zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: optimize event buffer (#278)

Browse Source 
* fix: how we fetch profiles in the buffer

* perf: optimize event buffer

* remove unused file

* fix

* wip

* wip: try groupmq 2

* try simplified event buffer with duration calculation on the fly instead
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-16 13:29:40 +01:00
committed by GitHub
parent 4736f8509d
commit 4483e464d1
46 changed files with 887 additions and 1841 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/package.json
View File

@@ -30,7 +30,6 @@
"@openpanel/logger": "workspace:*",
"@openpanel/payments": "workspace:*",
"@openpanel/queue": "workspace:*",
"groupmq": "catalog:",
"@openpanel/redis": "workspace:*",
"@openpanel/trpc": "workspace:*",
"@openpanel/validation": "workspace:*",
@@ -40,6 +39,7 @@
"fastify": "^5.6.1",
"fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0",
"groupmq": "catalog:",
"jsonwebtoken": "^9.0.2",
"ramda": "^0.29.1",
"sharp": "^0.33.5",

9
apps/api/src/bots/index.ts
View File

@@ -1,4 +1,4 @@
import { cacheable, cacheableLru } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import bots from './bots';
// Pre-compile regex patterns at module load time
@@ -15,7 +15,7 @@ const compiledBots = bots.map((bot) => {
const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
const includesBots = compiledBots.filter((bot) => 'includes' in bot);
export const isBot = cacheableLru(
export const isBot = cacheable(
'is-bot',
(ua: string) => {
// Check simple string patterns first (fast)
@@ -40,8 +40,5 @@ export const isBot = cacheableLru(
return null;
},
{
maxSize: 1000,
ttl: 60 * 5,
},
60 * 5
);

95
apps/api/src/controllers/live.controller.ts
View File

@@ -1,12 +1,5 @@
import type { FastifyRequest } from 'fastify';
import superjson from 'superjson';
import type { WebSocket } from '@fastify/websocket';
import {
eventBuffer,
getProfileById,
transformMinimalEvent,
} from '@openpanel/db';
import { eventBuffer } from '@openpanel/db';
import { setSuperJson } from '@openpanel/json';
import {
psubscribeToPublishedEvent,
@@ -14,10 +7,7 @@ import {
} from '@openpanel/redis';
import { getProjectAccess } from '@openpanel/trpc';
import { getOrganizationAccess } from '@openpanel/trpc/src/access';
export function getLiveEventInfo(key: string) {
return key.split(':').slice(2) as [string, string];
}
import type { FastifyRequest } from 'fastify';
export function wsVisitors(
socket: WebSocket,
@@ -25,27 +15,38 @@ export function wsVisitors(
Params: {
projectId: string;
};
}>,
}>
) {
const { params } = req;
const unsubscribe = subscribeToPublishedEvent('events', 'saved', (event) => {
if (event?.projectId === params.projectId) {
eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
const sendCount = () => {
eventBuffer
.getActiveVisitorCount(params.projectId)
.then((count) => {
socket.send(String(count));
})
.catch(() => {
socket.send('0');
});
};
const unsubscribe = subscribeToPublishedEvent(
'events',
'batch',
({ projectId }) => {
if (projectId === params.projectId) {
sendCount();
}
}
});
);
const punsubscribe = psubscribeToPublishedEvent(
'__keyevent@0__:expired',
(key) => {
const [projectId] = getLiveEventInfo(key);
if (projectId && projectId === params.projectId) {
eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
socket.send(String(count));
});
const [, , projectId] = key.split(':');
if (projectId === params.projectId) {
sendCount();
}
},
}
);
socket.on('close', () => {
@@ -62,18 +63,10 @@ export async function wsProjectEvents(
};
Querystring: {
token?: string;
type?: 'saved' | 'received';
};
}>,
}>
) {
const { params, query } = req;
const type = query.type || 'saved';
if (!['saved', 'received'].includes(type)) {
socket.send('Invalid type');
socket.close();
return;
}
const { params } = req;
const userId = req.session?.userId;
if (!userId) {
@@ -87,24 +80,20 @@ export async function wsProjectEvents(
projectId: params.projectId,
});
if (!access) {
socket.send('No access');
socket.close();
return;
}
const unsubscribe = subscribeToPublishedEvent(
'events',
type,
async (event) => {
if (event.projectId === params.projectId) {
const profile = await getProfileById(event.profileId, event.projectId);
socket.send(
superjson.stringify(
access
? {
...event,
profile,
}
: transformMinimalEvent(event),
),
);
'batch',
({ projectId, count }) => {
if (projectId === params.projectId) {
socket.send(setSuperJson({ count }));
}
},
}
);
socket.on('close', () => unsubscribe());
@@ -116,7 +105,7 @@ export async function wsProjectNotifications(
Params: {
projectId: string;
};
}>,
}>
) {
const { params } = req;
const userId = req.session?.userId;
@@ -143,9 +132,9 @@ export async function wsProjectNotifications(
'created',
(notification) => {
if (notification.projectId === params.projectId) {
socket.send(superjson.stringify(notification));
socket.send(setSuperJson(notification));
}
},
}
);
socket.on('close', () => unsubscribe());
@@ -157,7 +146,7 @@ export async function wsOrganizationEvents(
Params: {
organizationId: string;
};
}>,
}>
) {
const { params } = req;
const userId = req.session?.userId;
@@ -184,7 +173,7 @@ export async function wsOrganizationEvents(
'subscription_updated',
(message) => {
socket.send(setSuperJson(message));
},
}
);
socket.on('close', () => unsubscribe());

42
apps/api/src/controllers/manage.controller.ts
View File

@@ -1,5 +1,4 @@
import crypto from 'node:crypto';
import { HttpError } from '@/utils/errors';
import { stripTrailingSlash } from '@openpanel/common';
import { hashPassword } from '@openpanel/common/server';
import {
@@ -10,6 +9,7 @@ import {
} from '@openpanel/db';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { z } from 'zod';
import { HttpError } from '@/utils/errors';
// Validation schemas
const zCreateProject = z.object({
@@ -57,7 +57,7 @@ const zUpdateReference = z.object({
// Projects CRUD
export async function listProjects(
request: FastifyRequest,
reply: FastifyReply,
reply: FastifyReply
) {
const projects = await db.project.findMany({
where: {
@@ -74,7 +74,7 @@ export async function listProjects(
export async function getProject(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const project = await db.project.findFirst({
where: {
@@ -92,7 +92,7 @@ export async function getProject(
export async function createProject(
request: FastifyRequest<{ Body: z.infer<typeof zCreateProject> }>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zCreateProject.safeParse(request.body);
@@ -139,12 +139,9 @@ export async function createProject(
},
});
// Clear cache
await Promise.all([
getProjectByIdCached.clear(project.id),
project.clients.map((client) => {
getClientByIdCached.clear(client.id);
}),
...project.clients.map((client) => getClientByIdCached.clear(client.id)),
]);
reply.send({
@@ -165,7 +162,7 @@ export async function updateProject(
Params: { id: string };
Body: z.infer<typeof zUpdateProject>;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zUpdateProject.safeParse(request.body);
@@ -223,12 +220,9 @@ export async function updateProject(
data: updateData,
});
// Clear cache
await Promise.all([
getProjectByIdCached.clear(project.id),
existing.clients.map((client) => {
getClientByIdCached.clear(client.id);
}),
...existing.clients.map((client) => getClientByIdCached.clear(client.id)),
]);
reply.send({ data: project });
@@ -236,7 +230,7 @@ export async function updateProject(
export async function deleteProject(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const project = await db.project.findFirst({
where: {
@@ -266,7 +260,7 @@ export async function deleteProject(
// Clients CRUD
export async function listClients(
request: FastifyRequest<{ Querystring: { projectId?: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const where: any = {
organizationId: request.client!.organizationId,
@@ -300,7 +294,7 @@ export async function listClients(
export async function getClient(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const client = await db.client.findFirst({
where: {
@@ -318,7 +312,7 @@ export async function getClient(
export async function createClient(
request: FastifyRequest<{ Body: z.infer<typeof zCreateClient> }>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zCreateClient.safeParse(request.body);
@@ -374,7 +368,7 @@ export async function updateClient(
Params: { id: string };
Body: z.infer<typeof zUpdateClient>;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zUpdateClient.safeParse(request.body);
@@ -417,7 +411,7 @@ export async function updateClient(
export async function deleteClient(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const client = await db.client.findFirst({
where: {
@@ -444,7 +438,7 @@ export async function deleteClient(
// References CRUD
export async function listReferences(
request: FastifyRequest<{ Querystring: { projectId?: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const where: any = {};
@@ -488,7 +482,7 @@ export async function listReferences(
export async function getReference(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const reference = await db.reference.findUnique({
where: {
@@ -516,7 +510,7 @@ export async function getReference(
export async function createReference(
request: FastifyRequest<{ Body: z.infer<typeof zCreateReference> }>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zCreateReference.safeParse(request.body);
@@ -559,7 +553,7 @@ export async function updateReference(
Params: { id: string };
Body: z.infer<typeof zUpdateReference>;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const parsed = zUpdateReference.safeParse(request.body);
@@ -616,7 +610,7 @@ export async function updateReference(
export async function deleteReference(
request: FastifyRequest<{ Params: { id: string } }>,
reply: FastifyReply,
reply: FastifyReply
) {
const reference = await db.reference.findUnique({
where: {

33
apps/api/src/controllers/track.controller.ts
View File

@@ -7,7 +7,10 @@ import {
upsertProfile,
} from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import {
type EventsQueuePayloadIncomingEvent,
getEventsGroupQueueShard,
} from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import {
type IDecrementPayload,
@@ -112,6 +115,7 @@ interface TrackContext {
identity?: IIdentifyPayload;
deviceId: string;
sessionId: string;
session?: EventsQueuePayloadIncomingEvent['payload']['session'];
geo: GeoLocation;
}
@@ -141,19 +145,21 @@ async function buildContext(
validatedBody.payload.profileId = profileId;
}
const overrideDeviceId =
validatedBody.type === 'track' &&
typeof validatedBody.payload?.properties?.__deviceId === 'string'
? validatedBody.payload?.properties.__deviceId
: undefined;
// Get geo location (needed for track and identify)
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
const { deviceId, sessionId } = await getDeviceId({
const deviceIdResult = await getDeviceId({
projectId,
ip,
ua,
salts,
overrideDeviceId:
validatedBody.type === 'track' &&
typeof validatedBody.payload?.properties?.__deviceId === 'string'
? validatedBody.payload?.properties.__deviceId
: undefined,
overrideDeviceId,
});
return {
@@ -166,8 +172,9 @@ async function buildContext(
isFromPast: timestamp.isTimestampFromThePast,
},
identity,
deviceId,
sessionId,
deviceId: deviceIdResult.deviceId,
sessionId: deviceIdResult.sessionId,
session: deviceIdResult.session,
geo,
};
}
@@ -176,13 +183,14 @@ async function handleTrack(
payload: ITrackPayload,
context: TrackContext
): Promise<void> {
const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;
const { projectId, deviceId, geo, headers, timestamp, sessionId, session } =
context;
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
const groupId = uaInfo.isServer
? payload.profileId
? `${projectId}:${payload.profileId}`
: `${projectId}:${generateId()}`
: undefined
: deviceId;
const jobId = [
slug(payload.name),
@@ -203,7 +211,7 @@ async function handleTrack(
}
promises.push(
getEventsGroupQueueShard(groupId).add({
getEventsGroupQueueShard(groupId || generateId()).add({
orderMs: timestamp.value,
data: {
projectId,
@@ -217,6 +225,7 @@ async function handleTrack(
geo,
deviceId,
sessionId,
session,
},
groupId,
jobId,

9
apps/api/src/hooks/is-bot.hook.ts
View File

@@ -1,20 +1,19 @@
import { isBot } from '@/bots';
import { createBotEvent } from '@openpanel/db';
import type {
DeprecatedPostEventPayload,
ITrackHandlerPayload,
} from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { isBot } from '@/bots';
export async function isBotHook(
req: FastifyRequest<{
Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const bot = req.headers['user-agent']
? isBot(req.headers['user-agent'])
? await isBot(req.headers['user-agent'])
: null;
if (bot && req.client?.projectId) {
@@ -44,6 +43,6 @@ export async function isBotHook(
}
}
return reply.status(202).send();
return reply.status(202).send({ bot });
}
}

5
apps/api/src/routes/track.router.ts
View File

@@ -1,6 +1,5 @@
import { fetchDeviceId, handler } from '@/controllers/track.controller';
import type { FastifyPluginCallback } from 'fastify';
import { fetchDeviceId, handler } from '@/controllers/track.controller';
import { clientHook } from '@/hooks/client.hook';
import { duplicateHook } from '@/hooks/duplicate.hook';
import { isBotHook } from '@/hooks/is-bot.hook';
@@ -13,7 +12,7 @@ const trackRouter: FastifyPluginCallback = async (fastify) => {
fastify.route({
method: 'POST',
url: '/',
handler: handler,
handler,
});
fastify.route({

41
apps/api/src/utils/ids.ts
View File

@@ -1,7 +1,12 @@
import crypto from 'node:crypto';
import { generateDeviceId } from '@openpanel/common/server';
import { getSafeJson } from '@openpanel/json';
import type {
EventsQueuePayloadCreateSessionEnd,
EventsQueuePayloadIncomingEvent,
} from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import { pick } from 'ramda';
export async function getDeviceId({
projectId,
@@ -37,14 +42,20 @@ export async function getDeviceId({
ua,
});
return await getDeviceIdFromSession({
return await getInfoFromSession({
projectId,
currentDeviceId,
previousDeviceId,
});
}
async function getDeviceIdFromSession({
interface DeviceIdResult {
deviceId: string;
sessionId: string;
session?: EventsQueuePayloadIncomingEvent['payload']['session'];
}
async function getInfoFromSession({
projectId,
currentDeviceId,
previousDeviceId,
@@ -52,7 +63,7 @@ async function getDeviceIdFromSession({
projectId: string;
currentDeviceId: string;
previousDeviceId: string;
}) {
}): Promise<DeviceIdResult> {
try {
const multi = getRedisCache().multi();
multi.hget(
@@ -65,21 +76,33 @@ async function getDeviceIdFromSession({
);
const res = await multi.exec();
if (res?.[0]?.[1]) {
const data = getSafeJson<{ payload: { sessionId: string } }>(
const data = getSafeJson<EventsQueuePayloadCreateSessionEnd>(
(res?.[0]?.[1] as string) ?? ''
);
if (data) {
const sessionId = data.payload.sessionId;
return { deviceId: currentDeviceId, sessionId };
return {
deviceId: currentDeviceId,
sessionId: data.payload.sessionId,
session: pick(
['referrer', 'referrerName', 'referrerType'],
data.payload
),
};
}
}
if (res?.[1]?.[1]) {
const data = getSafeJson<{ payload: { sessionId: string } }>(
const data = getSafeJson<EventsQueuePayloadCreateSessionEnd>(
(res?.[1]?.[1] as string) ?? ''
);
if (data) {
const sessionId = data.payload.sessionId;
return { deviceId: previousDeviceId, sessionId };
return {
deviceId: previousDeviceId,
sessionId: data.payload.sessionId,
session: pick(
['referrer', 'referrerName', 'referrerType'],
data.payload
),
};
}
}
} catch (error) {

141
apps/start/src/components/events/event-list-item.tsx
View File

@@ -1,25 +1,20 @@
import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
import { Link } from '@tanstack/react-router';
import { SerieIcon } from '../report-chart/common/serie-icon';
import { EventIcon } from './event-icon';
import { Tooltiper } from '@/components/ui/tooltip';
import { useAppParams } from '@/hooks/use-app-params';
import { useNumber } from '@/hooks/use-numer-formatter';
import { pushModal } from '@/modals';
import { cn } from '@/utils/cn';
import { getProfileName } from '@/utils/getters';
import { Link } from '@tanstack/react-router';
import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
import { SerieIcon } from '../report-chart/common/serie-icon';
import { EventIcon } from './event-icon';
type EventListItemProps = IServiceEventMinimal | IServiceEvent;
export function EventListItem(props: EventListItemProps) {
const { organizationId, projectId } = useAppParams();
const { createdAt, name, path, duration, meta } = props;
const { createdAt, name, path, meta } = props;
const profile = 'profile' in props ? props.profile : null;
const number = useNumber();
const renderName = () => {
if (name === 'screen_view') {
if (path.includes('/')) {
@@ -32,83 +27,65 @@ export function EventListItem(props: EventListItemProps) {
return name.replace(/_/g, ' ');
};
const renderDuration = () => {
if (name === 'screen_view') {
return (
<span className="text-muted-foreground">
{number.shortWithUnit(duration / 1000, 'min')}
</span>
);
}
return null;
};
const isMinimal = 'minimal' in props;
return (
<>
<button
type="button"
onClick={() => {
if (!isMinimal) {
pushModal('EventDetails', {
id: props.id,
projectId,
createdAt,
});
}
}}
className={cn(
'card hover:bg-light-background flex w-full items-center justify-between rounded-lg p-4 transition-colors',
meta?.conversion &&
`bg-${meta.color}-50 dark:bg-${meta.color}-900 hover:bg-${meta.color}-100 dark:hover:bg-${meta.color}-700`,
)}
>
<div>
<div className="flex items-center gap-4 text-left ">
<EventIcon size="sm" name={name} meta={meta} />
<span>
<span className="font-medium">{renderName()}</span>
{' '}
{renderDuration()}
</span>
</div>
<div className="pl-10">
<div className="flex origin-left scale-75 gap-1">
<SerieIcon name={props.country} />
<SerieIcon name={props.os} />
<SerieIcon name={props.browser} />
</div>
<button
className={cn(
'card flex w-full items-center justify-between rounded-lg p-4 transition-colors hover:bg-light-background',
meta?.conversion &&
`bg-${meta.color}-50 dark:bg-${meta.color}-900 hover:bg-${meta.color}-100 dark:hover:bg-${meta.color}-700`
)}
onClick={() => {
if (!isMinimal) {
pushModal('EventDetails', {
id: props.id,
projectId,
createdAt,
});
}
}}
type="button"
>
<div>
<div className="flex items-center gap-4 text-left">
<EventIcon meta={meta} name={name} size="sm" />
<span className="font-medium">{renderName()}</span>
</div>
<div className="pl-10">
<div className="flex origin-left scale-75 gap-1">
<SerieIcon name={props.country} />
<SerieIcon name={props.os} />
<SerieIcon name={props.browser} />
</div>
</div>
<div className="flex gap-4">
{profile && (
<Tooltiper asChild content={getProfileName(profile)}>
<Link
onClick={(e) => {
e.stopPropagation();
}}
to={'/$organizationId/$projectId/profiles/$profileId'}
params={{
organizationId,
projectId,
profileId: profile.id,
}}
className="max-w-[80px] overflow-hidden text-ellipsis whitespace-nowrap text-muted-foreground hover:underline"
>
{getProfileName(profile)}
</Link>
</Tooltiper>
)}
<Tooltiper asChild content={createdAt.toLocaleString()}>
<div className=" text-muted-foreground">
{createdAt.toLocaleTimeString()}
</div>
</div>
<div className="flex gap-4">
{profile && (
<Tooltiper asChild content={getProfileName(profile)}>
<Link
className="max-w-[80px] overflow-hidden text-ellipsis whitespace-nowrap text-muted-foreground hover:underline"
onClick={(e) => {
e.stopPropagation();
}}
params={{
organizationId,
projectId,
profileId: profile.id,
}}
to={'/$organizationId/$projectId/profiles/$profileId'}
>
{getProfileName(profile)}
</Link>
</Tooltiper>
</div>
</button>
</>
)}
<Tooltiper asChild content={createdAt.toLocaleString()}>
<div className="text-muted-foreground">
{createdAt.toLocaleTimeString()}
</div>
</Tooltiper>
</div>
</button>
);
}

37
apps/start/src/components/events/event-listener.tsx
View File

@@ -1,3 +1,4 @@
import { AnimatedNumber } from '../animated-number';
import {
Tooltip,
TooltipContent,
@@ -8,71 +9,53 @@ import { useDebounceState } from '@/hooks/use-debounce-state';
import useWS from '@/hooks/use-ws';
import { cn } from '@/utils/cn';
import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
import { useParams } from '@tanstack/react-router';
import { AnimatedNumber } from '../animated-number';
export default function EventListener({
onRefresh,
}: {
onRefresh: () => void;
}) {
const params = useParams({
strict: false,
});
const { projectId } = useAppParams();
const counter = useDebounceState(0, 1000);
useWS<IServiceEventMinimal | IServiceEvent>(
useWS<{ count: number }>(
`/live/events/${projectId}`,
(event) => {
if (event) {
const isProfilePage = !!params?.profileId;
if (isProfilePage) {
const profile = 'profile' in event ? event.profile : null;
if (profile?.id === params?.profileId) {
counter.set((prev) => prev + 1);
}
return;
}
counter.set((prev) => prev + 1);
}
({ count }) => {
counter.set((prev) => prev + count);
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
},
}
);
return (
<Tooltip>
<TooltipTrigger asChild>
<button
type="button"
className="flex h-8 items-center gap-2 rounded-md border border-border bg-card px-3 font-medium leading-none"
onClick={() => {
counter.set(0);
onRefresh();
}}
className="flex h-8 items-center gap-2 rounded-md border border-border bg-card px-3 font-medium leading-none"
type="button"
>
<div className="relative">
<div
className={cn(
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all'
)}
/>
<div
className={cn(
'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all'
)}
/>
</div>
{counter.debounced === 0 ? (
'Listening'
) : (
<AnimatedNumber value={counter.debounced} suffix=" new events" />
<AnimatedNumber suffix=" new events" value={counter.debounced} />
)}
</button>
</TooltipTrigger>

78
apps/start/src/components/events/table/columns.tsx
View File

@@ -1,15 +1,14 @@
import type { IServiceEvent } from '@openpanel/db';
import type { ColumnDef } from '@tanstack/react-table';
import { ColumnCreatedAt } from '@/components/column-created-at';
import { EventIcon } from '@/components/events/event-icon';
import { ProjectLink } from '@/components/links';
import { ProfileAvatar } from '@/components/profiles/profile-avatar';
import { SerieIcon } from '@/components/report-chart/common/serie-icon';
import { KeyValueGrid } from '@/components/ui/key-value-grid';
import { useNumber } from '@/hooks/use-numer-formatter';
import { pushModal } from '@/modals';
import { getProfileName } from '@/utils/getters';
import type { ColumnDef } from '@tanstack/react-table';
import { ColumnCreatedAt } from '@/components/column-created-at';
import { ProfileAvatar } from '@/components/profiles/profile-avatar';
import { KeyValueGrid } from '@/components/ui/key-value-grid';
import type { IServiceEvent } from '@openpanel/db';
export function useColumns() {
const number = useNumber();
@@ -28,17 +27,24 @@ export function useColumns() {
accessorKey: 'name',
header: 'Name',
cell({ row }) {
const { name, path, duration, properties, revenue } = row.original;
const { name, path, revenue } = row.original;
const fullTitle =
name === 'screen_view'
? path
: name === 'revenue' && revenue
? `${name} (${number.currency(revenue / 100)})`
: name.replace(/_/g, ' ');
const renderName = () => {
if (name === 'screen_view') {
if (path.includes('/')) {
return <span className="max-w-md truncate">{path}</span>;
return path;
}
return (
<>
<span className="text-muted-foreground">Screen: </span>
<span className="max-w-md truncate">{path}</span>
{path}
</>
);
}
@@ -50,38 +56,27 @@ export function useColumns() {
return name.replace(/_/g, ' ');
};
const renderDuration = () => {
if (name === 'screen_view') {
return (
<span className="text-muted-foreground">
{number.shortWithUnit(duration / 1000, 'min')}
</span>
);
}
return null;
};
return (
<div className="flex items-center gap-2">
<div className="flex min-w-0 items-center gap-2">
<button
type="button"
className="transition-transform hover:scale-105"
className="shrink-0 transition-transform hover:scale-105"
onClick={() => {
pushModal('EditEvent', {
id: row.original.id,
});
}}
type="button"
>
<EventIcon
size="sm"
name={row.original.name}
meta={row.original.meta}
name={row.original.name}
size="sm"
/>
</button>
<span className="flex gap-2">
<span className="flex min-w-0 flex-1 gap-2">
<button
type="button"
className="min-w-0 max-w-full truncate text-left font-medium hover:underline"
title={fullTitle}
onClick={() => {
pushModal('EventDetails', {
id: row.original.id,
@@ -89,11 +84,10 @@ export function useColumns() {
projectId: row.original.projectId,
});
}}
className="font-medium hover:underline"
type="button"
>
{renderName()}
<span className="block truncate">{renderName()}</span>
</button>
{renderDuration()}
</span>
</div>
);
@@ -107,8 +101,8 @@ export function useColumns() {
if (profile) {
return (
<ProjectLink
className="group row items-center gap-2 whitespace-nowrap font-medium hover:underline"
href={`/profiles/${encodeURIComponent(profile.id)}`}
className="group whitespace-nowrap font-medium hover:underline row items-center gap-2"
>
<ProfileAvatar size="sm" {...profile} />
{getProfileName(profile)}
@@ -119,8 +113,8 @@ export function useColumns() {
if (profileId && profileId !== deviceId) {
return (
<ProjectLink
href={`/profiles/${encodeURIComponent(profileId)}`}
className="whitespace-nowrap font-medium hover:underline"
href={`/profiles/${encodeURIComponent(profileId)}`}
>
Unknown
</ProjectLink>
@@ -130,8 +124,8 @@ export function useColumns() {
if (deviceId) {
return (
<ProjectLink
href={`/profiles/${encodeURIComponent(deviceId)}`}
className="whitespace-nowrap font-medium hover:underline"
href={`/profiles/${encodeURIComponent(deviceId)}`}
>
Anonymous
</ProjectLink>
@@ -152,10 +146,10 @@ export function useColumns() {
const { sessionId } = row.original;
return (
<ProjectLink
href={`/sessions/${encodeURIComponent(sessionId)}`}
className="whitespace-nowrap font-medium hover:underline"
href={`/sessions/${encodeURIComponent(sessionId)}`}
>
{sessionId.slice(0,6)}
{sessionId.slice(0, 6)}
</ProjectLink>
);
},
@@ -175,7 +169,7 @@ export function useColumns() {
cell({ row }) {
const { country, city } = row.original;
return (
<div className="row items-center gap-2 min-w-0">
<div className="row min-w-0 items-center gap-2">
<SerieIcon name={country} />
<span className="truncate">{city}</span>
</div>
@@ -189,7 +183,7 @@ export function useColumns() {
cell({ row }) {
const { os } = row.original;
return (
<div className="row items-center gap-2 min-w-0">
<div className="row min-w-0 items-center gap-2">
<SerieIcon name={os} />
<span className="truncate">{os}</span>
</div>
@@ -203,7 +197,7 @@ export function useColumns() {
cell({ row }) {
const { browser } = row.original;
return (
<div className="row items-center gap-2 min-w-0">
<div className="row min-w-0 items-center gap-2">
<SerieIcon name={browser} />
<span className="truncate">{browser}</span>
</div>
@@ -221,14 +215,14 @@ export function useColumns() {
const { properties } = row.original;
const filteredProperties = Object.fromEntries(
Object.entries(properties || {}).filter(
([key]) => !key.startsWith('__'),
),
([key]) => !key.startsWith('__')
)
);
const items = Object.entries(filteredProperties);
const limit = 2;
const data = items.slice(0, limit).map(([key, value]) => ({
name: key,
value: value,
value,
}));
if (items.length > limit) {
data.push({

9
apps/start/src/components/events/table/index.tsx
View File

@@ -35,6 +35,7 @@ type Props = {
>,
unknown
>;
showEventListener?: boolean;
};
const LOADING_DATA = [{}, {}, {}, {}, {}, {}, {}, {}, {}] as IServiceEvent[];
@@ -215,7 +216,7 @@ const VirtualizedEventsTable = ({
);
};
export const EventsTable = ({ query }: Props) => {
export const EventsTable = ({ query, showEventListener = false }: Props) => {
const { isLoading } = query;
const columns = useColumns();
@@ -272,7 +273,7 @@ export const EventsTable = ({ query }: Props) => {
return (
<>
<EventsTableToolbar query={query} table={table} />
<EventsTableToolbar query={query} table={table} showEventListener={showEventListener} />
<VirtualizedEventsTable table={table} data={data} isLoading={isLoading} />
<div className="w-full h-10 center-center pt-4" ref={inViewportRef}>
<div
@@ -291,9 +292,11 @@ export const EventsTable = ({ query }: Props) => {
function EventsTableToolbar({
query,
table,
showEventListener,
}: {
query: Props['query'];
table: Table<IServiceEvent>;
showEventListener: boolean;
}) {
const { projectId } = useAppParams();
const [startDate, setStartDate] = useQueryState(
@@ -305,7 +308,7 @@ function EventsTableToolbar({
return (
<DataTableToolbarContainer>
<div className="flex flex-1 flex-wrap items-center gap-2">
<EventListener onRefresh={() => query.refetch()} />
{showEventListener && <EventListener onRefresh={() => query.refetch()} />}
<Button
variant="outline"
size="sm"

32
apps/start/src/components/onboarding/onboarding-verify-listener.tsx
View File

@@ -1,31 +1,13 @@
import type {
IServiceClient,
IServiceEvent,
IServiceProject,
} from '@openpanel/db';
import type { IServiceEvent } from '@openpanel/db';
import { CheckCircle2Icon, CheckIcon, Loader2 } from 'lucide-react';
import { useState } from 'react';
import useWS from '@/hooks/use-ws';
import { cn } from '@/utils/cn';
import { timeAgo } from '@/utils/date';
interface Props {
project: IServiceProject;
client: IServiceClient | null;
events: IServiceEvent[];
onVerified: (verified: boolean) => void;
}
const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
const [events, setEvents] = useState<IServiceEvent[]>(_events ?? []);
useWS<IServiceEvent>(
`/live/events/${client?.projectId}?type=received`,
(data) => {
setEvents((prev) => [...prev, data]);
onVerified(true);
}
);
const VerifyListener = ({ events }: Props) => {
const isConnected = events.length > 0;
const renderIcon = () => {
@@ -49,16 +31,18 @@ const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
<div
className={cn(
'flex gap-6 rounded-xl p-4 md:p-6',
isConnected ? 'bg-emerald-100 dark:bg-emerald-700' : 'bg-blue-500/10'
isConnected
? 'bg-emerald-100 dark:bg-emerald-700/10'
: 'bg-blue-500/10'
)}
>
{renderIcon()}
<div className="flex-1">
<div className="font-semibold text-foreground/90 text-lg leading-normal">
{isConnected ? 'Success' : 'Waiting for events'}
{isConnected ? 'Successfully connected' : 'Waiting for events'}
</div>
{isConnected ? (
<div className="flex flex-col-reverse">
<div className="mt-2 flex flex-col-reverse gap-1">
{events.length > 5 && (
<div className="flex items-center gap-2">
<CheckIcon size={14} />{' '}
@@ -69,7 +53,7 @@ const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
<div className="flex items-center gap-2" key={event.id}>
<CheckIcon size={14} />{' '}
<span className="font-medium">{event.name}</span>{' '}
<span className="ml-auto text-emerald-800">
<span className="ml-auto text-foreground/50 text-sm">
{timeAgo(event.createdAt, 'round')}
</span>
</div>

98
apps/start/src/components/realtime/realtime-active-sessions.tsx
View File

@@ -1,11 +1,9 @@
import { useTRPC } from '@/integrations/trpc/react';
import { useQuery } from '@tanstack/react-query';
import { AnimatePresence, motion } from 'framer-motion';
import { useEffect, useState } from 'react';
import useWS from '@/hooks/use-ws';
import type { IServiceEvent } from '@openpanel/db';
import { EventItem } from '../events/table/item';
import { ProjectLink } from '../links';
import { SerieIcon } from '../report-chart/common/serie-icon';
import { useTRPC } from '@/integrations/trpc/react';
import { formatTimeAgoOrDateTime } from '@/utils/date';
interface RealtimeActiveSessionsProps {
projectId: string;
@@ -17,64 +15,52 @@ export function RealtimeActiveSessions({
limit = 10,
}: RealtimeActiveSessionsProps) {
const trpc = useTRPC();
const activeSessionsQuery = useQuery(
trpc.realtime.activeSessions.queryOptions({
projectId,
}),
const { data: sessions = [] } = useQuery(
trpc.realtime.activeSessions.queryOptions(
{ projectId },
{ refetchInterval: 5000 }
)
);
const [state, setState] = useState<IServiceEvent[]>([]);
// Update state when initial data loads
useEffect(() => {
if (activeSessionsQuery.data && state.length === 0) {
setState(activeSessionsQuery.data);
}
}, [activeSessionsQuery.data, state]);
// Set up WebSocket connection for real-time updates
useWS<IServiceEvent>(
`/live/events/${projectId}`,
(session) => {
setState((prev) => {
// Add new session and remove duplicates, keeping most recent
const filtered = prev.filter((s) => s.id !== session.id);
return [session, ...filtered].slice(0, limit);
});
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
},
);
const sessions = state.length > 0 ? state : (activeSessionsQuery.data ?? []);
return (
<div className="col h-full max-md:hidden">
<div className="hide-scrollbar h-full overflow-y-auto pb-10">
<AnimatePresence mode="popLayout" initial={false}>
<div className="col gap-4">
{sessions.map((session) => (
<div className="col card h-full max-md:hidden">
<div className="hide-scrollbar h-full overflow-y-auto">
<AnimatePresence initial={false} mode="popLayout">
<div className="col divide-y">
{sessions.slice(0, limit).map((session) => (
<motion.div
key={session.id}
layout
// initial={{ opacity: 0, x: -200, scale: 0.8 }}
animate={{ opacity: 1, x: 0, scale: 1 }}
exit={{ opacity: 0, x: 200, scale: 0.8 }}
key={session.id}
layout
transition={{ duration: 0.4, type: 'spring', stiffness: 300 }}
>
<EventItem
event={session}
viewOptions={{
properties: false,
origin: false,
queryString: false,
}}
className="w-full"
/>
<ProjectLink
className="relative block p-4 py-3 pr-14"
href={`/sessions/${session.sessionId}`}
>
<div className="col flex-1 gap-1">
{session.name === 'screen_view' && (
<span className="text-muted-foreground text-xs leading-normal/80">
{session.origin}
</span>
)}
<span className="font-medium text-sm leading-normal">
{session.name === 'screen_view'
? session.path
: session.name}
</span>
<span className="text-muted-foreground text-xs">
{formatTimeAgoOrDateTime(session.createdAt)}
</span>
</div>
<div className="row absolute top-1/2 right-4 origin-right -translate-y-1/2 scale-50 gap-2">
<SerieIcon name={session.referrerName} />
<SerieIcon name={session.os} />
<SerieIcon name={session.browser} />
<SerieIcon name={session.device} />
</div>
</ProjectLink>
</motion.div>
))}
</div>

2
apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.events.tsx
View File

@@ -42,5 +42,5 @@ function Component() {
),
);
return <EventsTable query={query} />;
return <EventsTable query={query} showEventListener />;
}

23
apps/start/src/routes/_app.$organizationId.$projectId.realtime.tsx
View File

@@ -1,3 +1,5 @@
import { keepPreviousData, useQuery } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { Fullscreen, FullscreenClose } from '@/components/fullscreen-toggle';
import RealtimeMap from '@/components/realtime/map';
import { RealtimeActiveSessions } from '@/components/realtime/realtime-active-sessions';
@@ -7,12 +9,10 @@ import { RealtimePaths } from '@/components/realtime/realtime-paths';
import { RealtimeReferrals } from '@/components/realtime/realtime-referrals';
import RealtimeReloader from '@/components/realtime/realtime-reloader';
import { useTRPC } from '@/integrations/trpc/react';
import { PAGE_TITLES, createProjectTitle } from '@/utils/title';
import { keepPreviousData, useQuery } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
export const Route = createFileRoute(
'/_app/$organizationId/$projectId/realtime',
'/_app/$organizationId/$projectId/realtime'
)({
component: Component,
head: () => {
@@ -36,8 +36,8 @@ function Component() {
},
{
placeholderData: keepPreviousData,
},
),
}
)
);
return (
@@ -47,7 +47,7 @@ function Component() {
<RealtimeReloader projectId={projectId} />
<div className="row relative">
<div className="overflow-hidden aspect-[4/2] w-full">
<div className="aspect-[4/2] w-full overflow-hidden">
<RealtimeMap
markers={coordinatesQuery.data ?? []}
sidebarConfig={{
@@ -56,18 +56,17 @@ function Component() {
}}
/>
</div>
<div className="absolute top-8 left-8 bottom-0 col gap-4">
<div className="card p-4 w-72 bg-background/90">
<div className="col absolute top-8 bottom-4 left-8 gap-4">
<div className="card w-72 bg-background/90 p-4">
<RealtimeLiveHistogram projectId={projectId} />
</div>
<div className="w-72 flex-1 min-h-0 relative">
<div className="relative min-h-0 w-72 flex-1">
<RealtimeActiveSessions projectId={projectId} />
<div className="absolute bottom-0 left-0 right-0 h-10 bg-gradient-to-t from-def-100 to-transparent" />
</div>
</div>
</div>
<div className="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4 p-4 pt-4 md:p-8 md:pt-0">
<div className="grid grid-cols-1 gap-4 p-4 pt-4 md:grid-cols-2 md:p-8 md:pt-0 xl:grid-cols-3">
<div>
<RealtimeGeo projectId={projectId} />
</div>

28
apps/start/src/routes/_steps.onboarding.$projectId.verify.tsx
View File

@@ -1,7 +1,6 @@
import { useQuery } from '@tanstack/react-query';
import { createFileRoute, Link, redirect } from '@tanstack/react-router';
import { BoxSelectIcon } from 'lucide-react';
import { useEffect, useState } from 'react';
import { ButtonContainer } from '@/components/button-container';
import { FullPageEmptyState } from '@/components/full-page-empty-state';
import FullPageLoadingState from '@/components/full-page-loading-state';
@@ -33,22 +32,21 @@ export const Route = createFileRoute('/_steps/onboarding/$projectId/verify')({
});
function Component() {
const [isVerified, setIsVerified] = useState(false);
const { projectId } = Route.useParams();
const trpc = useTRPC();
const { data: events, refetch } = useQuery(
trpc.event.events.queryOptions({ projectId })
const { data: events } = useQuery(
trpc.event.events.queryOptions(
{ projectId },
{
refetchInterval: 2500,
}
)
);
const isVerified = events?.data && events.data.length > 0;
const { data: project } = useQuery(
trpc.project.getProjectWithClients.queryOptions({ projectId })
);
useEffect(() => {
if (events && events.data.length > 0) {
setIsVerified(true);
}
}, [events]);
if (!project) {
return (
<FullPageEmptyState icon={BoxSelectIcon} title="Project not found" />
@@ -64,15 +62,7 @@ function Component() {
<div className="flex min-h-0 flex-1 flex-col">
<div className="scrollbar-thin flex-1 overflow-y-auto">
<div className="col gap-8 p-4">
<VerifyListener
client={client}
events={events?.data ?? []}
onVerified={() => {
refetch();
setIsVerified(true);
}}
project={project}
/>
<VerifyListener events={events?.data ?? []} />
<VerifyFaq project={project} />
</div>

2
apps/worker/package.json
View File

@@ -16,11 +16,11 @@
"@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*",
"@openpanel/email": "workspace:*",
"@openpanel/importer": "workspace:*",
"@openpanel/integrations": "workspace:^",
"@openpanel/js-runtime": "workspace:*",
"@openpanel/json": "workspace:*",
"@openpanel/logger": "workspace:*",
"@openpanel/importer": "workspace:*",
"@openpanel/payments": "workspace:*",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*",

52
apps/worker/src/boot-workers.ts
View File

@@ -1,10 +1,9 @@
import type { Queue, WorkerOptions } from 'bullmq';
import { Worker } from 'bullmq';
import { performance } from 'node:perf_hooks';
import { setTimeout as sleep } from 'node:timers/promises';
import {
cronQueue,
EVENTS_GROUP_QUEUES_SHARDS,
type EventsQueuePayloadIncomingEvent,
cronQueue,
eventsGroupQueues,
gscQueue,
importQueue,
@@ -15,14 +14,12 @@ import {
sessionsQueue,
} from '@openpanel/queue';
import { getRedisQueue } from '@openpanel/redis';
import { performance } from 'node:perf_hooks';
import { setTimeout as sleep } from 'node:timers/promises';
import type { Queue, WorkerOptions } from 'bullmq';
import { Worker } from 'bullmq';
import { Worker as GroupWorker } from 'groupmq';
import { cronJob } from './jobs/cron';
import { gscJob } from './jobs/gsc';
import { incomingEvent } from './jobs/events.incoming-event';
import { gscJob } from './jobs/gsc';
import { importJob } from './jobs/import';
import { insightsProjectJob } from './jobs/insights';
import { miscJob } from './jobs/misc';
@@ -95,7 +92,7 @@ function getConcurrencyFor(queueName: string, defaultValue = 1): number {
return defaultValue;
}
export async function bootWorkers() {
export function bootWorkers() {
const enabledQueues = getEnabledQueues();
const workers: (Worker | GroupWorker<any>)[] = [];
@@ -119,12 +116,14 @@ export async function bootWorkers() {
for (const index of eventQueuesToStart) {
const queue = eventsGroupQueues[index];
if (!queue) continue;
if (!queue) {
continue;
}
const queueName = `events_${index}`;
const concurrency = getConcurrencyFor(
queueName,
Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10),
Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10)
);
const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({
@@ -132,7 +131,7 @@ export async function bootWorkers() {
concurrency,
logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,
blockingTimeoutSec: Number.parseFloat(
process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1',
process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1'
),
handler: async (job) => {
return await incomingEvent(job.data);
@@ -172,7 +171,7 @@ export async function bootWorkers() {
const notificationWorker = new Worker(
notificationQueue.name,
notificationJob,
{ ...workerOptions, concurrency },
{ ...workerOptions, concurrency }
);
workers.push(notificationWorker);
logger.info('Started worker for notification', { concurrency });
@@ -224,7 +223,7 @@ export async function bootWorkers() {
if (workers.length === 0) {
logger.warn(
'No workers started. Check ENABLED_QUEUES environment variable.',
'No workers started. Check ENABLED_QUEUES environment variable.'
);
}
@@ -254,7 +253,7 @@ export async function bootWorkers() {
const elapsed = job.finishedOn - job.processedOn;
eventsGroupJobDuration.observe(
{ name: worker.name, status: 'failed' },
elapsed,
elapsed
);
}
logger.error('job failed', {
@@ -267,23 +266,6 @@ export async function bootWorkers() {
}
});
(worker as Worker).on('completed', (job) => {
if (job) {
if (job.processedOn && job.finishedOn) {
const elapsed = job.finishedOn - job.processedOn;
logger.info('job completed', {
jobId: job.id,
worker: worker.name,
elapsed,
});
eventsGroupJobDuration.observe(
{ name: worker.name, status: 'success' },
elapsed,
);
}
}
});
(worker as Worker).on('ioredis:close', () => {
logger.error('worker closed due to ioredis:close', {
worker: worker.name,
@@ -293,7 +275,7 @@ export async function bootWorkers() {
async function exitHandler(
eventName: string,
evtOrExitCodeOrError: number | string | Error,
evtOrExitCodeOrError: number | string | Error
) {
// Log the actual error details for unhandled rejections/exceptions
if (evtOrExitCodeOrError instanceof Error) {
@@ -339,7 +321,7 @@ export async function bootWorkers() {
process.on(evt, (code) => {
exitHandler(evt, code);
});
},
}
);
return workers;

2
apps/worker/src/jobs/cron.salt.ts
View File

@@ -33,7 +33,7 @@ async function generateNewSalt() {
return created;
});
getSalts.clear();
await getSalts.clear();
return newSalt;
}

131
apps/worker/src/jobs/events.incoming-event.ts
View File

@@ -1,9 +1,5 @@
import { getTime, isSameDomain, parsePath } from '@openpanel/common';
import {
getReferrerWithQuery,
parseReferrer,
parseUserAgent,
} from '@openpanel/common/server';
import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';
import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
import {
checkNotificationRulesForEvent,
@@ -14,10 +10,12 @@ import {
} from '@openpanel/db';
import type { ILogger } from '@openpanel/logger';
import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
import { getLock } from '@openpanel/redis';
import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';
import { logger as baseLogger } from '@/utils/logger';
import { createSessionEndJob, getSessionEnd } from '@/utils/session-handler';
import {
createSessionEndJob,
extendSessionEndJob,
} from '@/utils/session-handler';
const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];
@@ -93,7 +91,8 @@ export async function incomingEvent(
projectId,
deviceId,
sessionId,
uaInfo: _uaInfo,
uaInfo,
session,
} = jobPayload;
const properties = body.properties ?? {};
const reqId = headers['request-id'] ?? 'unknown';
@@ -121,16 +120,15 @@ export async function incomingEvent(
? null
: parseReferrer(getProperty('__referrer'));
const utmReferrer = getReferrerWithQuery(query);
const userAgent = headers['user-agent'];
const sdkName = headers['openpanel-sdk-name'];
const sdkVersion = headers['openpanel-sdk-version'];
// TODO: Remove both user-agent and parseUserAgent
const uaInfo = _uaInfo ?? parseUserAgent(userAgent, properties);
const baseEvent = {
const baseEvent: IServiceCreateEventPayload = {
name: body.name,
profileId,
projectId,
deviceId,
sessionId,
properties: omit(GLOBAL_PROPERTIES, {
...properties,
__hash: hash,
@@ -149,7 +147,7 @@ export async function incomingEvent(
origin,
referrer: referrer?.url || '',
referrerName: utmReferrer?.name || referrer?.name || referrer?.url,
referrerType: referrer?.type || utmReferrer?.type || '',
referrerType: utmReferrer?.type || referrer?.type || '',
os: uaInfo.os,
osVersion: uaInfo.osVersion,
browser: uaInfo.browser,
@@ -161,16 +159,17 @@ export async function incomingEvent(
body.name === 'revenue' && '__revenue' in properties
? parseRevenue(properties.__revenue)
: undefined,
} as const;
};
// if timestamp is from the past we dont want to create a new session
if (uaInfo.isServer || isTimestampFromThePast) {
const session = profileId
? await sessionBuffer.getExistingSession({
profileId,
projectId,
})
: null;
const session =
profileId && !isTimestampFromThePast
? await sessionBuffer.getExistingSession({
profileId,
projectId,
})
: null;
const payload = {
...baseEvent,
@@ -198,82 +197,48 @@ export async function incomingEvent(
return createEventAndNotify(payload as IServiceEvent, logger, projectId);
}
const sessionEnd = await getSessionEnd({
projectId,
deviceId,
profileId,
});
const activeSession = sessionEnd
? await sessionBuffer.getExistingSession({
sessionId: sessionEnd.sessionId,
})
: null;
const payload: IServiceCreateEventPayload = merge(baseEvent, {
deviceId: sessionEnd?.deviceId ?? deviceId,
sessionId: sessionEnd?.sessionId ?? sessionId,
referrer: sessionEnd?.referrer ?? baseEvent.referrer,
referrerName: sessionEnd?.referrerName ?? baseEvent.referrerName,
referrerType: sessionEnd?.referrerType ?? baseEvent.referrerType,
// if the path is not set, use the last screen view path
path: baseEvent.path || activeSession?.exit_path || '',
origin: baseEvent.origin || activeSession?.exit_origin || '',
referrer: session?.referrer ?? baseEvent.referrer,
referrerName: session?.referrerName ?? baseEvent.referrerName,
referrerType: session?.referrerType ?? baseEvent.referrerType,
} as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;
// If the triggering event is filtered, do not create session_start or the event (issue #2)
const isExcluded = await isEventExcludedByProjectFilter(payload, projectId);
if (isExcluded) {
logger.info(
'Skipping session_start and event (excluded by project filter)',
{ event: payload.name, projectId }
);
return null;
}
if (session) {
await extendSessionEndJob({
projectId,
deviceId,
}).catch((error) => {
logger.error('Error finding and extending session end job', { error });
throw error;
});
} else {
await createEventAndNotify(
{
event: payload.name,
projectId,
}
);
return null;
}
...payload,
name: 'session_start',
createdAt: new Date(getTime(payload.createdAt) - 100),
},
logger,
projectId
).catch((error) => {
logger.error('Error creating session start event', { event: payload });
throw error;
});
if (!sessionEnd) {
const locked = await getLock(
`session_start:${projectId}:${sessionId}`,
'1',
1000
);
if (locked) {
logger.info('Creating session start event', { event: payload });
await createEventAndNotify(
{
...payload,
name: 'session_start',
createdAt: new Date(getTime(payload.createdAt) - 100),
},
logger,
projectId
).catch((error) => {
logger.error('Error creating session start event', { event: payload });
throw error;
});
} else {
logger.info('Session start already claimed by another worker', {
event: payload,
});
}
}
const event = await createEventAndNotify(payload, logger, projectId);
if (!event) {
// Skip creating session end when event was excluded
return null;
}
if (!sessionEnd) {
logger.info('Creating session end job', { event: payload });
await createSessionEndJob({ payload }).catch((error) => {
logger.error('Error creating session end job', { event: payload });
throw error;
});
}
return event;
return createEventAndNotify(payload, logger, projectId);
}

5
apps/worker/src/jobs/events.incoming-events.test.ts
View File

@@ -186,6 +186,11 @@ describe('incomingEvent', () => {
projectId,
deviceId,
sessionId: 'session-123',
session: {
referrer: '',
referrerName: '',
referrerType: '',
},
};
const changeDelay = vi.fn();

43
apps/worker/src/metrics.ts
View File

@@ -1,5 +1,3 @@
import client from 'prom-client';
import {
botBuffer,
eventBuffer,
@@ -8,6 +6,7 @@ import {
sessionBuffer,
} from '@openpanel/db';
import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';
import client from 'prom-client';
const Registry = client.Registry;
@@ -20,7 +19,7 @@ export const eventsGroupJobDuration = new client.Histogram({
name: 'job_duration_ms',
help: 'Duration of job processing (in ms)',
labelNames: ['name', 'status'],
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10_000, 30_000], // 10ms to 30s
});
register.registerMetric(eventsGroupJobDuration);
@@ -28,57 +27,61 @@ register.registerMetric(eventsGroupJobDuration);
queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_active_count`,
name: `${queue.name.replace(/[{}]/g, '')}_active_count`,
help: 'Active count',
async collect() {
const metric = await queue.getActiveCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_delayed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_delayed_count`,
help: 'Delayed count',
async collect() {
const metric = await queue.getDelayedCount();
this.set(metric);
if ('getDelayedCount' in queue) {
const metric = await queue.getDelayedCount();
this.set(metric);
} else {
this.set(0);
}
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_failed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_failed_count`,
help: 'Failed count',
async collect() {
const metric = await queue.getFailedCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_completed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_completed_count`,
help: 'Completed count',
async collect() {
const metric = await queue.getCompletedCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_waiting_count`,
name: `${queue.name.replace(/[{}]/g, '')}_waiting_count`,
help: 'Waiting count',
async collect() {
const metric = await queue.getWaitingCount();
this.set(metric);
},
}),
})
);
});
@@ -90,7 +93,7 @@ register.registerMetric(
const metric = await eventBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -101,7 +104,7 @@ register.registerMetric(
const metric = await profileBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -112,7 +115,7 @@ register.registerMetric(
const metric = await botBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -123,7 +126,7 @@ register.registerMetric(
const metric = await sessionBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -134,5 +137,5 @@ register.registerMetric(
const metric = await replayBuffer.getBufferSize();
this.set(metric);
},
}),
})
);

141
apps/worker/src/utils/session-handler.ts
View File

@@ -1,13 +1,39 @@
import type { IServiceCreateEventPayload } from '@openpanel/db';
import {
type EventsQueuePayloadCreateSessionEnd,
sessionsQueue,
} from '@openpanel/queue';
import type { Job } from 'bullmq';
import { logger } from './logger';
import { sessionsQueue } from '@openpanel/queue';
export const SESSION_TIMEOUT = 1000 * 60 * 30;
const CHANGE_DELAY_THROTTLE_MS = process.env.CHANGE_DELAY_THROTTLE_MS
? Number.parseInt(process.env.CHANGE_DELAY_THROTTLE_MS, 10)
: 60_000; // 1 minute
const CHANGE_DELAY_THROTTLE_MAP = new Map<string, number>();
export async function extendSessionEndJob({
projectId,
deviceId,
}: {
projectId: string;
deviceId: string;
}) {
const last = CHANGE_DELAY_THROTTLE_MAP.get(`${projectId}:${deviceId}`) ?? 0;
const isThrottled = Date.now() - last < CHANGE_DELAY_THROTTLE_MS;
if (isThrottled) {
return;
}
const jobId = getSessionEndJobId(projectId, deviceId);
const job = await sessionsQueue.getJob(jobId);
if (!job) {
return;
}
await job.changeDelay(SESSION_TIMEOUT);
CHANGE_DELAY_THROTTLE_MAP.set(`${projectId}:${deviceId}`, Date.now());
}
const getSessionEndJobId = (projectId: string, deviceId: string) =>
`sessionEnd:${projectId}:${deviceId}`;
@@ -33,106 +59,3 @@ export function createSessionEndJob({
}
);
}
export async function getSessionEnd({
projectId,
deviceId,
profileId,
}: {
projectId: string;
deviceId: string;
profileId: string;
}) {
const sessionEnd = await getSessionEndJob({
projectId,
deviceId,
});
if (sessionEnd) {
const existingSessionIsAnonymous =
sessionEnd.job.data.payload.profileId ===
sessionEnd.job.data.payload.deviceId;
const eventIsIdentified =
profileId && sessionEnd.job.data.payload.profileId !== profileId;
if (existingSessionIsAnonymous && eventIsIdentified) {
await sessionEnd.job.updateData({
...sessionEnd.job.data,
payload: {
...sessionEnd.job.data.payload,
profileId,
},
});
}
await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
return sessionEnd.job.data.payload;
}
return null;
}
export async function getSessionEndJob(args: {
projectId: string;
deviceId: string;
retryCount?: number;
}): Promise<{
deviceId: string;
job: Job<EventsQueuePayloadCreateSessionEnd>;
} | null> {
const { retryCount = 0 } = args;
if (retryCount >= 6) {
throw new Error('Failed to get session end');
}
async function handleJobStates(
job: Job<EventsQueuePayloadCreateSessionEnd>,
deviceId: string
): Promise<{
deviceId: string;
job: Job<EventsQueuePayloadCreateSessionEnd>;
} | null> {
const state = await job.getState();
if (state !== 'delayed') {
logger.debug(`[session-handler] Session end job is in "${state}" state`, {
state,
retryCount,
jobTimestamp: new Date(job.timestamp).toISOString(),
jobDelta: Date.now() - job.timestamp,
jobId: job.id,
payload: job.data.payload,
});
}
if (state === 'delayed' || state === 'waiting') {
return { deviceId, job };
}
if (state === 'active') {
await new Promise((resolve) => setTimeout(resolve, 100));
return getSessionEndJob({
...args,
retryCount: retryCount + 1,
});
}
if (state === 'completed') {
await job.remove();
}
return null;
}
// Check current device job
const currentJob = await sessionsQueue.getJob(
getSessionEndJobId(args.projectId, args.deviceId)
);
if (currentJob) {
return await handleJobStates(currentJob, args.deviceId);
}
// Create session
return null;
}