zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-12-17 22:59:11 +01:00
parent bc84404235
commit ccff90829b
33 changed files with 1882 additions and 1083 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
packages/db/prisma/migrations/20251217204808_insight_payload_default/migration.sql Normal file
View File

@@ -0,0 +1,9 @@
/*
Warnings:
- Made the column `payload` on table `project_insights` required. This step will fail if there are existing NULL values in that column.
*/
-- AlterTable
ALTER TABLE "public"."project_insights" ALTER COLUMN "payload" SET NOT NULL,
ALTER COLUMN "payload" SET DEFAULT '{}';

13
packages/db/prisma/migrations/20251217210920_insights/migration.sql Normal file
View File

@@ -0,0 +1,13 @@
/*
Warnings:
- You are about to drop the column `changePct` on the `project_insights` table. All the data in the column will be lost.
- You are about to drop the column `compareValue` on the `project_insights` table. All the data in the column will be lost.
- You are about to drop the column `currentValue` on the `project_insights` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "public"."project_insights" DROP COLUMN "changePct",
DROP COLUMN "compareValue",
DROP COLUMN "currentValue",
ADD COLUMN "displayName" TEXT NOT NULL DEFAULT '';

11
packages/db/prisma/schema.prisma
View File

@@ -512,13 +512,12 @@ model ProjectInsight {
windowKind String // "yesterday" | "rolling_7d" | "rolling_30d"
state InsightState @default(active)
title String
summary String?
payload Json? // RenderedCard blocks, extra data
title String
summary String?
displayName String @default("")
/// [IPrismaProjectInsightPayload]
payload Json @default("{}") // Rendered insight payload (typed)
currentValue Float?
compareValue Float?
changePct Float?
direction String? // "up" | "down" | "flat"
impactScore Float @default(0)
severityBand String? // "low" | "moderate" | "severe"

19
packages/db/src/clickhouse/query-builder.ts
View File

@@ -43,7 +43,7 @@ class Expression {
}
export class Query<T = any> {
private _select: string[] = [];
private _select: (string | Expression)[] = [];
private _except: string[] = [];
private _from?: string | Expression;
private _where: WhereCondition[] = [];
@@ -81,17 +81,19 @@ export class Query<T = any> {
// Select methods
select<U>(
columns: (string | null | undefined | false)[],
columns: (string | Expression | null | undefined | false)[],
type: 'merge' | 'replace' = 'replace',
): Query<U> {
if (this._skipNext) return this as unknown as Query<U>;
if (type === 'merge') {
this._select = [
...this._select,
...columns.filter((col): col is string => Boolean(col)),
...columns.filter((col): col is string | Expression => Boolean(col)),
];
} else {
this._select = columns.filter((col): col is string => Boolean(col));
this._select = columns.filter((col): col is string | Expression =>
Boolean(col),
);
}
return this as unknown as Query<U>;
}
@@ -372,7 +374,14 @@ export class Query<T = any> {
if (this._select.length > 0) {
parts.push(
'SELECT',
this._select.map((col) => this.escapeDate(col)).join(', '),
this._select
// Important: Expressions are treated as raw SQL; do not run escapeDate()
// on them, otherwise any embedded date strings get double-escaped
// (e.g. ''2025-12-16 23:59:59'') which ClickHouse rejects.
.map((col) =>
col instanceof Expression ? col.toString() : this.escapeDate(col),
)
.join(', '),
);
} else {
parts.push('SELECT *');

68
packages/db/src/services/insights/cached-clix.ts Normal file
View File

@@ -0,0 +1,68 @@
import crypto from 'node:crypto';
import type { ClickHouseClient } from '@clickhouse/client';
import {
type Query,
clix as originalClix,
} from '../../clickhouse/query-builder';
/**
* Creates a cached wrapper around clix that automatically caches query results
* based on query hash. This eliminates duplicate queries within the same module/window context.
*
* @param client - ClickHouse client
* @param cache - Optional cache Map to store query results
* @param timezone - Timezone for queries (defaults to UTC)
* @returns A function that creates cached Query instances (compatible with clix API)
*/
export function createCachedClix(
client: ClickHouseClient,
cache?: Map<string, any>,
timezone?: string,
) {
function clixCached(): Query {
const query = originalClix(client, timezone);
const queryTimezone = timezone ?? 'UTC';
// Override execute() method to add caching
const originalExecute = query.execute.bind(query);
query.execute = async () => {
// Build the query SQL string
const querySQL = query.toSQL();
// Create cache key from query SQL + timezone
const cacheKey = crypto
.createHash('sha256')
.update(`${querySQL}|${queryTimezone}`)
.digest('hex');
// Check cache first
if (cache?.has(cacheKey)) {
return cache.get(cacheKey);
}
// Execute query
const result = await originalExecute();
// Cache the result
if (cache) {
cache.set(cacheKey, result);
}
return result;
};
return query;
}
// Copy static methods from original clix
clixCached.exp = originalClix.exp;
clixCached.date = originalClix.date;
clixCached.datetime = originalClix.datetime;
clixCached.dynamicDatetime = originalClix.dynamicDatetime;
clixCached.toStartOf = originalClix.toStartOf;
clixCached.toStartOfInterval = originalClix.toStartOfInterval;
clixCached.toInterval = originalClix.toInterval;
clixCached.toDate = originalClix.toDate;
return clixCached;
}

171
packages/db/src/services/insights/engine.ts
View File

@@ -1,17 +1,22 @@
import crypto from 'node:crypto';
import { createCachedClix } from './cached-clix';
import { materialDecision } from './material';
import { defaultImpactScore, severityBand } from './scoring';
import type {
Cadence,
ComputeContext,
ComputeResult,
ExplainQueue,
InsightModule,
InsightStore,
WindowKind,
} from './types';
import { resolveWindow } from './windows';
const DEFAULT_WINDOWS: WindowKind[] = [
'yesterday',
'rolling_7d',
'rolling_30d',
];
export interface EngineConfig {
keepTopNPerModuleWindow: number; // e.g. 5
closeStaleAfterDays: number; // e.g. 7
@@ -21,8 +26,6 @@ export interface EngineConfig {
minAbsDelta: number; // e.g. 80
minPct: number; // e.g. 0.15
};
enableExplain: boolean;
explainTopNPerProjectPerDay: number; // e.g. 3
}
/** Simple gating to cut noise; modules can override via thresholds. */
@@ -53,64 +56,84 @@ function chunk<T>(arr: T[], size: number): T[][] {
return out;
}
function sha256(x: string) {
return crypto.createHash('sha256').update(x).digest('hex');
}
/**
* Engine entrypoint: runs all projects for a cadence.
* Recommended: call this from a per-project worker (fanout), but it can also run directly.
*/
export function createEngine(args: {
store: InsightStore;
modules: InsightModule[];
db: any;
logger?: Pick<Console, 'info' | 'warn' | 'error'>;
explainQueue?: ExplainQueue;
config: EngineConfig;
}) {
const { store, modules, db, explainQueue, config } = args;
const { store, modules, db, config } = args;
const logger = args.logger ?? console;
async function runCadence(cadence: Cadence, now: Date): Promise<void> {
const projectIds = await store.listProjectIdsForCadence(cadence);
for (const projectId of projectIds) {
await runProject({ projectId, cadence, now });
}
function isProjectOldEnoughForWindow(
projectCreatedAt: Date | null | undefined,
baselineStart: Date,
): boolean {
if (!projectCreatedAt) return true; // best-effort; don't block if unknown
return projectCreatedAt.getTime() <= baselineStart.getTime();
}
async function runProject(opts: {
projectId: string;
cadence: Cadence;
now: Date;
projectCreatedAt?: Date | null;
}): Promise<void> {
const { projectId, cadence, now } = opts;
const { projectId, cadence, now, projectCreatedAt } = opts;
const projLogger = logger;
const eligible = modules.filter((m) => m.cadence.includes(cadence));
// Track top insights (by impact) for optional explain step across all modules/windows
const explainCandidates: Array<{
insightId: string;
impact: number;
evidence: any;
evidenceHash: string;
}> = [];
for (const mod of eligible) {
for (const windowKind of mod.windows) {
const window = resolveWindow(windowKind as WindowKind, now);
const ctx: ComputeContext = {
projectId,
window,
db,
now,
logger: projLogger,
};
const windows = mod.windows ?? DEFAULT_WINDOWS;
for (const windowKind of windows) {
let window: ReturnType<typeof resolveWindow>;
let ctx: ComputeContext;
try {
window = resolveWindow(windowKind, now);
if (
!isProjectOldEnoughForWindow(projectCreatedAt, window.baselineStart)
) {
continue;
}
// Initialize cache for this module+window combination.
// Cache is automatically garbage collected when context goes out of scope.
const cache = new Map<string, any>();
ctx = {
projectId,
window,
db,
now,
logger: projLogger,
clix: createCachedClix(db, cache),
};
} catch (e) {
projLogger.error('[insights] failed to create compute context', {
projectId,
module: mod.key,
windowKind,
err: e,
});
continue;
}
// 1) enumerate dimensions
let dims = mod.enumerateDimensions
? await mod.enumerateDimensions(ctx)
: [];
let dims: string[] = [];
try {
dims = mod.enumerateDimensions
? await mod.enumerateDimensions(ctx)
: [];
} catch (e) {
// Important: enumeration failures should not abort the whole project run.
// Also avoid lifecycle close/suppression when we didn't actually evaluate dims.
projLogger.error('[insights] module enumerateDimensions failed', {
projectId,
module: mod.key,
windowKind,
err: e,
});
continue;
}
const maxDims = mod.thresholds?.maxDims ?? 25;
if (dims.length > maxDims) dims = dims.slice(0, maxDims);
@@ -190,9 +213,6 @@ export function createEngine(args: {
window,
card,
metrics: {
currentValue: r.currentValue,
compareValue: r.compareValue,
changePct: r.changePct,
direction: r.direction,
impactScore: impact,
severityBand: sev,
@@ -241,7 +261,6 @@ export function createEngine(args: {
windowKind,
eventKind,
changeFrom: {
changePct: prev.changePct,
direction: prev.direction,
impactScore: prev.impactScore,
severityBand: prev.severityBand,
@@ -255,48 +274,6 @@ export function createEngine(args: {
now,
});
}
// 9) optional AI explain candidates (only for top-impact insights)
if (config.enableExplain && explainQueue && mod.drivers) {
// compute evidence deterministically (drivers)
try {
const drivers = await mod.drivers(r, ctx);
const evidence = {
insight: {
moduleKey: mod.key,
dimensionKey: r.dimensionKey,
windowKind,
currentValue: r.currentValue,
compareValue: r.compareValue,
changePct: r.changePct,
direction: r.direction,
},
drivers,
window: {
start: window.start.toISOString().slice(0, 10),
end: window.end.toISOString().slice(0, 10),
baselineStart: window.baselineStart
.toISOString()
.slice(0, 10),
baselineEnd: window.baselineEnd.toISOString().slice(0, 10),
},
};
const evidenceHash = sha256(JSON.stringify(evidence));
explainCandidates.push({
insightId: persisted.id,
impact,
evidence,
evidenceHash,
});
} catch (e) {
projLogger.warn('[insights] drivers() failed', {
projectId,
module: mod.key,
dimensionKey: r.dimensionKey,
err: e,
});
}
}
}
}
@@ -320,27 +297,7 @@ export function createEngine(args: {
});
}
}
// 12) enqueue explains for top insights across the whole project run
if (config.enableExplain && explainQueue) {
explainCandidates.sort((a, b) => b.impact - a.impact);
const top = explainCandidates.slice(
0,
config.explainTopNPerProjectPerDay,
);
for (const c of top) {
await explainQueue.enqueueExplain({
insightId: c.insightId,
projectId,
moduleKey: 'n/a', // optional; you can include it in evidence instead
dimensionKey: 'n/a',
windowKind: 'yesterday',
evidence: c.evidence,
evidenceHash: c.evidenceHash,
});
}
}
}
return { runCadence, runProject };
return { runProject };
}

1
packages/db/src/services/insights/index.ts
View File

@@ -4,6 +4,5 @@ export * from './scoring';
export * from './material';
export * from './engine';
export * from './store';
export * from './normalize';
export * from './utils';
export * from './modules';

374
packages/db/src/services/insights/modules/devices.module.ts
View File

@@ -1,77 +1,42 @@
import { TABLE_NAMES } from '../../../clickhouse/client';
import { clix } from '../../../clickhouse/query-builder';
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
import { computeWeekdayMedians, getEndOfDay, getWeekday } from '../utils';
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
import type {
ComputeContext,
ComputeResult,
InsightModule,
RenderedCard,
} from '../types';
import {
buildLookupMap,
computeChangePct,
computeDirection,
computeMedian,
getEndOfDay,
getWeekday,
selectTopDimensions,
} from '../utils';
function normalizeDevice(device: string): string {
const d = (device || '').toLowerCase().trim();
if (d.includes('mobile') || d === 'phone') return 'mobile';
if (d.includes('tablet')) return 'tablet';
if (d.includes('desktop')) return 'desktop';
return d || 'unknown';
}
export const devicesModule: InsightModule = {
key: 'devices',
cadence: ['daily'],
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 5 },
async enumerateDimensions(ctx) {
// Query devices from current window (limited set, no need for baseline merge)
const results = await clix(ctx.db)
.select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.where('device', '!=', '')
.groupBy(['device'])
.orderBy('cnt', 'DESC')
.execute();
// Normalize and dedupe device types
const dims = new Set<string>();
for (const r of results) {
dims.add(`device:${normalizeDevice(r.device)}`);
}
return Array.from(dims);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
// Single query for ALL current values
const currentResults = await clix(ctx.db)
.select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.groupBy(['device'])
.execute();
// Build current lookup map (normalized) and total
const currentMap = new Map<string, number>();
let totalCurrentValue = 0;
for (const r of currentResults) {
const key = normalizeDevice(r.device);
const cnt = Number(r.cnt ?? 0);
currentMap.set(key, (currentMap.get(key) ?? 0) + cnt);
totalCurrentValue += cnt;
}
// Single query for baseline
let baselineMap: Map<string, number>;
let totalBaselineValue = 0;
if (ctx.window.kind === 'yesterday') {
const baselineResults = await clix(ctx.db)
async function fetchDeviceAggregates(ctx: ComputeContext): Promise<{
currentMap: Map<string, number>;
baselineMap: Map<string, number>;
totalCurrent: number;
totalBaseline: number;
}> {
if (ctx.window.kind === 'yesterday') {
const [currentResults, baselineResults, totals] = await Promise.all([
ctx
.clix()
.select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.groupBy(['device'])
.execute(),
ctx
.clix()
.select<{ date: string; device: string; cnt: number }>([
'toDate(created_at) as date',
'device',
@@ -85,77 +50,144 @@ export const devicesModule: InsightModule = {
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['date', 'device'])
.execute();
const targetWeekday = getWeekday(ctx.window.start);
// Group by normalized device type before computing medians
const normalizedResults = baselineResults.map((r) => ({
date: r.date,
device: normalizeDevice(r.device),
cnt: r.cnt,
}));
// Aggregate by date + normalized device first
const aggregated = new Map<string, { date: string; cnt: number }[]>();
for (const r of normalizedResults) {
const key = `${r.date}|${r.device}`;
if (!aggregated.has(r.device)) {
aggregated.set(r.device, []);
}
// Find existing entry for this date+device or add new
const entries = aggregated.get(r.device)!;
const existing = entries.find((e) => e.date === r.date);
if (existing) {
existing.cnt += Number(r.cnt ?? 0);
} else {
entries.push({ date: r.date, cnt: Number(r.cnt ?? 0) });
}
}
// Compute weekday medians per device type
baselineMap = new Map<string, number>();
for (const [deviceType, entries] of aggregated) {
const sameWeekdayValues = entries
.filter((e) => getWeekday(new Date(e.date)) === targetWeekday)
.map((e) => e.cnt)
.sort((a, b) => a - b);
if (sameWeekdayValues.length > 0) {
const mid = Math.floor(sameWeekdayValues.length / 2);
const median =
sameWeekdayValues.length % 2 === 0
? ((sameWeekdayValues[mid - 1] ?? 0) +
(sameWeekdayValues[mid] ?? 0)) /
2
: (sameWeekdayValues[mid] ?? 0);
baselineMap.set(deviceType, median);
totalBaselineValue += median;
}
}
} else {
const baselineResults = await clix(ctx.db)
.select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
.execute(),
ctx
.clix()
.select<{ cur_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
getEndOfDay(ctx.window.end),
])
.groupBy(['device'])
.execute();
.execute(),
]);
baselineMap = new Map<string, number>();
for (const r of baselineResults) {
const key = normalizeDevice(r.device);
const cnt = Number(r.cnt ?? 0);
baselineMap.set(key, (baselineMap.get(key) ?? 0) + cnt);
totalBaselineValue += cnt;
const currentMap = buildLookupMap(currentResults, (r) => r.device);
const targetWeekday = getWeekday(ctx.window.start);
const aggregated = new Map<string, { date: string; cnt: number }[]>();
for (const r of baselineResults) {
if (!aggregated.has(r.device)) {
aggregated.set(r.device, []);
}
const entries = aggregated.get(r.device)!;
const existing = entries.find((e) => e.date === r.date);
if (existing) {
existing.cnt += Number(r.cnt ?? 0);
} else {
entries.push({ date: r.date, cnt: Number(r.cnt ?? 0) });
}
}
// Build results from maps
const baselineMap = new Map<string, number>();
for (const [deviceType, entries] of aggregated) {
const sameWeekdayValues = entries
.filter((e) => getWeekday(new Date(e.date)) === targetWeekday)
.map((e) => e.cnt)
.sort((a, b) => a - b);
if (sameWeekdayValues.length > 0) {
baselineMap.set(deviceType, computeMedian(sameWeekdayValues));
}
}
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline =
baselineMap.size > 0
? Array.from(baselineMap.values()).reduce((sum, val) => sum + val, 0)
: 0;
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
const curStart = formatClickhouseDate(ctx.window.start);
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
const [results, totals] = await Promise.all([
ctx
.clix()
.select<{ device: string; cur: number; base: number }>([
'device',
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.groupBy(['device'])
.execute(),
ctx
.clix()
.select<{ cur_total: number; base_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.execute(),
]);
const currentMap = buildLookupMap(
results,
(r) => r.device,
(r) => Number(r.cur ?? 0),
);
const baselineMap = buildLookupMap(
results,
(r) => r.device,
(r) => Number(r.base ?? 0),
);
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = totals[0]?.base_total ?? 0;
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
export const devicesModule: InsightModule = {
key: 'devices',
cadence: ['daily'],
thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 5 },
async enumerateDimensions(ctx) {
const { currentMap, baselineMap } = await fetchDeviceAggregates(ctx);
const topDims = selectTopDimensions(
currentMap,
baselineMap,
this.thresholds?.maxDims ?? 5,
);
return topDims.map((dim) => `device:${dim}`);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
await fetchDeviceAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
@@ -165,23 +197,12 @@ export const devicesModule: InsightModule = {
const currentValue = currentMap.get(deviceType) ?? 0;
const compareValue = baselineMap.get(deviceType) ?? 0;
const currentShare =
totalCurrentValue > 0 ? currentValue / totalCurrentValue : 0;
const compareShare =
totalBaselineValue > 0 ? compareValue / totalBaselineValue : 0;
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
// Share shift in percentage points
const shareShiftPp = (currentShare - compareShare) * 100;
const changePct =
compareShare > 0
? (currentShare - compareShare) / compareShare
: currentShare > 0
? 1
: 0;
// Direction should match the sign of the pp shift (so title + delta agree)
const direction: 'up' | 'down' | 'flat' =
shareShiftPp > 0 ? 'up' : shareShiftPp < 0 ? 'down' : 'flat';
const changePct = computeChangePct(currentValue, compareValue);
const direction = computeDirection(changePct);
results.push({
ok: true,
@@ -203,20 +224,51 @@ export const devicesModule: InsightModule = {
render(result, ctx): RenderedCard {
const device = result.dimensionKey.replace('device:', '');
const shareShiftPp = (result.extra?.shareShiftPp as number) ?? 0;
const isIncrease = shareShiftPp >= 0;
const changePct = result.changePct ?? 0;
const isIncrease = changePct >= 0;
const sessionsCurrent = result.currentValue ?? 0;
const sessionsCompare = result.compareValue ?? 0;
const shareCurrent = Number(result.extra?.currentShare ?? 0);
const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
kind: 'insight_v1',
title: `${device} ${isIncrease ? '↑' : '↓'} ${Math.abs(shareShiftPp).toFixed(1)}pp`,
summary: `${ctx.window.label}. Device share shift.`,
primaryDimension: { type: 'device', key: device, displayName: device },
tags: ['devices', ctx.window.kind, isIncrease ? 'increase' : 'decrease'],
metric: 'share',
extra: {
currentShare: result.extra?.currentShare,
compareShare: result.extra?.compareShare,
shareShiftPp: result.extra?.shareShiftPp,
title: `${device} ${isIncrease ? '↑' : '↓'} ${Math.abs(changePct * 100).toFixed(0)}%`,
summary: `${ctx.window.label}. Device traffic change.`,
displayName: device,
payload: {
kind: 'insight_v1',
dimensions: [{ key: 'device', value: device, displayName: device }],
primaryMetric: 'sessions',
metrics: {
sessions: {
current: sessionsCurrent,
compare: sessionsCompare,
delta: sessionsCurrent - sessionsCompare,
changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
direction: result.direction ?? 'flat',
unit: 'count',
},
share: {
current: shareCurrent,
compare: shareCompare,
delta: shareCurrent - shareCompare,
changePct:
shareCompare > 0
? (shareCurrent - shareCompare) / shareCompare
: null,
direction:
shareCurrent - shareCompare > 0.0005
? 'up'
: shareCurrent - shareCompare < -0.0005
? 'down'
: 'flat',
unit: 'ratio',
},
},
extra: {
// keep module-specific flags/fields if needed later
},
},
};
},

308
packages/db/src/services/insights/modules/entry-pages.module.ts
View File

@@ -1,26 +1,34 @@
import { TABLE_NAMES } from '../../../clickhouse/client';
import { clix } from '../../../clickhouse/query-builder';
import { normalizePath } from '../normalize';
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
import type {
ComputeContext,
ComputeResult,
InsightModule,
RenderedCard,
} from '../types';
import {
buildLookupMap,
computeChangePct,
computeDirection,
computeWeekdayMedians,
getEndOfDay,
getWeekday,
selectTopDimensions,
} from '../utils';
export const entryPagesModule: InsightModule = {
key: 'entry-pages',
cadence: ['daily'],
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
const DELIMITER = '|||';
async enumerateDimensions(ctx) {
// Query top entry pages from BOTH current and baseline windows
const [currentResults, baselineResults] = await Promise.all([
clix(ctx.db)
.select<{ entry_path: string; cnt: number }>([
async function fetchEntryPageAggregates(ctx: ComputeContext): Promise<{
currentMap: Map<string, number>;
baselineMap: Map<string, number>;
totalCurrent: number;
totalBaseline: number;
}> {
if (ctx.window.kind === 'yesterday') {
const [currentResults, baselineResults, totals] = await Promise.all([
ctx
.clix()
.select<{ entry_origin: string; entry_path: string; cnt: number }>([
'entry_origin',
'entry_path',
'count(*) as cnt',
])
@@ -31,12 +39,18 @@ export const entryPagesModule: InsightModule = {
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.groupBy(['entry_path'])
.orderBy('cnt', 'DESC')
.limit(this.thresholds?.maxDims ?? 100)
.groupBy(['entry_origin', 'entry_path'])
.execute(),
clix(ctx.db)
.select<{ entry_path: string; cnt: number }>([
ctx
.clix()
.select<{
date: string;
entry_origin: string;
entry_path: string;
cnt: number;
}>([
'toDate(created_at) as date',
'entry_origin',
'entry_path',
'count(*) as cnt',
])
@@ -47,104 +61,147 @@ export const entryPagesModule: InsightModule = {
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['entry_path'])
.orderBy('cnt', 'DESC')
.limit(this.thresholds?.maxDims ?? 100)
.groupBy(['date', 'entry_origin', 'entry_path'])
.execute(),
ctx
.clix()
.select<{ cur_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.execute(),
]);
// Merge both sets
const dims = new Set<string>();
for (const r of currentResults) {
dims.add(`entry:${normalizePath(r.entry_path || '/')}`);
}
for (const r of baselineResults) {
dims.add(`entry:${normalizePath(r.entry_path || '/')}`);
}
const currentMap = buildLookupMap(
currentResults,
(r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
);
return Array.from(dims);
},
const targetWeekday = getWeekday(ctx.window.start);
const baselineMap = computeWeekdayMedians(
baselineResults,
targetWeekday,
(r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
);
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
// Single query for ALL current values
const currentResults = await clix(ctx.db)
.select<{ entry_path: string; cnt: number }>([
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = Array.from(baselineMap.values()).reduce(
(sum, val) => sum + val,
0,
);
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
const curStart = formatClickhouseDate(ctx.window.start);
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
const [results, totals] = await Promise.all([
ctx
.clix()
.select<{
entry_origin: string;
entry_path: string;
cur: number;
base: number;
}>([
'entry_origin',
'entry_path',
'count(*) as cnt',
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.start,
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.groupBy(['entry_path'])
.execute();
.groupBy(['entry_origin', 'entry_path'])
.execute(),
ctx
.clix()
.select<{ cur_total: number; base_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.execute(),
]);
// Build current lookup map
const currentMap = new Map<string, number>();
for (const r of currentResults) {
const key = normalizePath(r.entry_path || '/');
currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
}
const currentMap = buildLookupMap(
results,
(r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
(r) => Number(r.cur ?? 0),
);
// Single query for baseline
let baselineMap: Map<string, number>;
const baselineMap = buildLookupMap(
results,
(r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
(r) => Number(r.base ?? 0),
);
if (ctx.window.kind === 'yesterday') {
const baselineResults = await clix(ctx.db)
.select<{ date: string; entry_path: string; cnt: number }>([
'toDate(created_at) as date',
'entry_path',
'count(*) as cnt',
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['date', 'entry_path'])
.execute();
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = totals[0]?.base_total ?? 0;
const targetWeekday = getWeekday(ctx.window.start);
baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
normalizePath(r.entry_path || '/'),
);
} else {
const baselineResults = await clix(ctx.db)
.select<{ entry_path: string; cnt: number }>([
'entry_path',
'count(*) as cnt',
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['entry_path'])
.execute();
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
baselineMap = new Map<string, number>();
for (const r of baselineResults) {
const key = normalizePath(r.entry_path || '/');
baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
}
}
export const entryPagesModule: InsightModule = {
key: 'entry-pages',
cadence: ['daily'],
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
// Build results from maps
async enumerateDimensions(ctx) {
const { currentMap, baselineMap } = await fetchEntryPageAggregates(ctx);
const topDims = selectTopDimensions(
currentMap,
baselineMap,
this.thresholds?.maxDims ?? 100,
);
return topDims.map((dim) => `entry:${dim}`);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
await fetchEntryPageAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
if (!dimKey.startsWith('entry:')) continue;
const entryPath = dimKey.replace('entry:', '');
const originPath = dimKey.replace('entry:', '');
const currentValue = currentMap.get(entryPath) ?? 0;
const compareValue = baselineMap.get(entryPath) ?? 0;
const currentValue = currentMap.get(originPath) ?? 0;
const compareValue = baselineMap.get(originPath) ?? 0;
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
const shareShiftPp = (currentShare - compareShare) * 100;
const changePct = computeChangePct(currentValue, compareValue);
const direction = computeDirection(changePct);
@@ -156,6 +213,9 @@ export const entryPagesModule: InsightModule = {
changePct,
direction,
extra: {
shareShiftPp,
currentShare,
compareShare,
isNew: compareValue === 0 && currentValue > 0,
},
});
@@ -165,28 +225,62 @@ export const entryPagesModule: InsightModule = {
},
render(result, ctx): RenderedCard {
const path = result.dimensionKey.replace('entry:', '');
const originPath = result.dimensionKey.replace('entry:', '');
const [origin, path] = originPath.split(DELIMITER);
const displayValue = origin ? `${origin}${path}` : path || '/';
const pct = ((result.changePct ?? 0) * 100).toFixed(1);
const isIncrease = (result.changePct ?? 0) >= 0;
const isNew = result.extra?.isNew as boolean | undefined;
const title = isNew
? `New entry page: ${path}`
: `Entry page ${path} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
? `New entry page: ${displayValue}`
: `Entry page ${displayValue} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
const sessionsCurrent = result.currentValue ?? 0;
const sessionsCompare = result.compareValue ?? 0;
const shareCurrent = Number(result.extra?.currentShare ?? 0);
const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
kind: 'insight_v1',
title,
summary: `${ctx.window.label}. Sessions ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
primaryDimension: { type: 'entry', key: path, displayName: path },
tags: [
'entry-pages',
ctx.window.kind,
isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
],
metric: 'sessions',
extra: {
isNew: result.extra?.isNew,
summary: `${ctx.window.label}. Sessions ${sessionsCurrent} vs ${sessionsCompare}.`,
displayName: displayValue,
payload: {
kind: 'insight_v1',
dimensions: [
{ key: 'origin', value: origin ?? '', displayName: origin ?? '' },
{ key: 'path', value: path ?? '', displayName: path ?? '' },
],
primaryMetric: 'sessions',
metrics: {
sessions: {
current: sessionsCurrent,
compare: sessionsCompare,
delta: sessionsCurrent - sessionsCompare,
changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
direction: result.direction ?? 'flat',
unit: 'count',
},
share: {
current: shareCurrent,
compare: shareCompare,
delta: shareCurrent - shareCompare,
changePct:
shareCompare > 0
? (shareCurrent - shareCompare) / shareCompare
: null,
direction:
shareCurrent - shareCompare > 0.0005
? 'up'
: shareCurrent - shareCompare < -0.0005
? 'down'
: 'flat',
unit: 'ratio',
},
},
extra: {
isNew: result.extra?.isNew,
},
},
};
},

333
packages/db/src/services/insights/modules/geo.module.ts
View File

@@ -1,18 +1,31 @@
import { TABLE_NAMES } from '../../../clickhouse/client';
import { clix } from '../../../clickhouse/query-builder';
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
import { computeWeekdayMedians, getEndOfDay, getWeekday } from '../utils';
import { getCountry } from '@openpanel/constants';
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
import type {
ComputeContext,
ComputeResult,
InsightModule,
RenderedCard,
} from '../types';
import {
buildLookupMap,
computeChangePct,
computeDirection,
computeWeekdayMedians,
getEndOfDay,
getWeekday,
selectTopDimensions,
} from '../utils';
export const geoModule: InsightModule = {
key: 'geo',
cadence: ['daily'],
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 30 },
async enumerateDimensions(ctx) {
// Query top countries from BOTH current and baseline windows
const [currentResults, baselineResults] = await Promise.all([
clix(ctx.db)
async function fetchGeoAggregates(ctx: ComputeContext): Promise<{
currentMap: Map<string, number>;
baselineMap: Map<string, number>;
totalCurrent: number;
totalBaseline: number;
}> {
if (ctx.window.kind === 'yesterday') {
const [currentResults, baselineResults, totals] = await Promise.all([
ctx
.clix()
.select<{ country: string; cnt: number }>([
'country',
'count(*) as cnt',
@@ -24,72 +37,10 @@ export const geoModule: InsightModule = {
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.where('country', '!=', '')
.groupBy(['country'])
.orderBy('cnt', 'DESC')
.limit(this.thresholds?.maxDims ?? 30)
.execute(),
clix(ctx.db)
.select<{ country: string; cnt: number }>([
'country',
'count(*) as cnt',
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
.where('country', '!=', '')
.groupBy(['country'])
.orderBy('cnt', 'DESC')
.limit(this.thresholds?.maxDims ?? 30)
.execute(),
]);
// Merge both sets
const dims = new Set<string>();
for (const r of currentResults) {
dims.add(`country:${r.country || 'unknown'}`);
}
for (const r of baselineResults) {
dims.add(`country:${r.country || 'unknown'}`);
}
return Array.from(dims);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
// Single query for ALL current values + total
const currentResults = await clix(ctx.db)
.select<{ country: string; cnt: number }>(['country', 'count(*) as cnt'])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.groupBy(['country'])
.execute();
// Build current lookup map and total
const currentMap = new Map<string, number>();
let totalCurrentValue = 0;
for (const r of currentResults) {
const key = r.country || 'unknown';
const cnt = Number(r.cnt ?? 0);
currentMap.set(key, (currentMap.get(key) ?? 0) + cnt);
totalCurrentValue += cnt;
}
// Single query for baseline
let baselineMap: Map<string, number>;
let totalBaselineValue = 0;
if (ctx.window.kind === 'yesterday') {
const baselineResults = await clix(ctx.db)
ctx
.clix()
.select<{ date: string; country: string; cnt: number }>([
'toDate(created_at) as date',
'country',
@@ -103,45 +54,127 @@ export const geoModule: InsightModule = {
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['date', 'country'])
.execute();
const targetWeekday = getWeekday(ctx.window.start);
baselineMap = computeWeekdayMedians(
baselineResults,
targetWeekday,
(r) => r.country || 'unknown',
);
// Compute total baseline from medians
for (const value of baselineMap.values()) {
totalBaselineValue += value;
}
} else {
const baselineResults = await clix(ctx.db)
.select<{ country: string; cnt: number }>([
'country',
'count(*) as cnt',
.execute(),
ctx
.clix()
.select<{ cur_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
getEndOfDay(ctx.window.end),
])
.groupBy(['country'])
.execute();
.execute(),
]);
baselineMap = new Map<string, number>();
for (const r of baselineResults) {
const key = r.country || 'unknown';
const cnt = Number(r.cnt ?? 0);
baselineMap.set(key, (baselineMap.get(key) ?? 0) + cnt);
totalBaselineValue += cnt;
}
}
const currentMap = buildLookupMap(
currentResults,
(r) => r.country || 'unknown',
);
// Build results from maps
const targetWeekday = getWeekday(ctx.window.start);
const baselineMap = computeWeekdayMedians(
baselineResults,
targetWeekday,
(r) => r.country || 'unknown',
);
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = Array.from(baselineMap.values()).reduce(
(sum, val) => sum + val,
0,
);
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
const curStart = formatClickhouseDate(ctx.window.start);
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
const [results, totals] = await Promise.all([
ctx
.clix()
.select<{ country: string; cur: number; base: number }>([
'country',
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.groupBy(['country'])
.execute(),
ctx
.clix()
.select<{ cur_total: number; base_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.execute(),
]);
const currentMap = buildLookupMap(
results,
(r) => r.country || 'unknown',
(r) => Number(r.cur ?? 0),
);
const baselineMap = buildLookupMap(
results,
(r) => r.country || 'unknown',
(r) => Number(r.base ?? 0),
);
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = totals[0]?.base_total ?? 0;
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
export const geoModule: InsightModule = {
key: 'geo',
cadence: ['daily'],
thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 30 },
async enumerateDimensions(ctx) {
const { currentMap, baselineMap } = await fetchGeoAggregates(ctx);
const topDims = selectTopDimensions(
currentMap,
baselineMap,
this.thresholds?.maxDims ?? 30,
);
return topDims.map((dim) => `country:${dim}`);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
await fetchGeoAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
@@ -151,23 +184,12 @@ export const geoModule: InsightModule = {
const currentValue = currentMap.get(country) ?? 0;
const compareValue = baselineMap.get(country) ?? 0;
const currentShare =
totalCurrentValue > 0 ? currentValue / totalCurrentValue : 0;
const compareShare =
totalBaselineValue > 0 ? compareValue / totalBaselineValue : 0;
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
// Share shift in percentage points
const shareShiftPp = (currentShare - compareShare) * 100;
const changePct =
compareShare > 0
? (currentShare - compareShare) / compareShare
: currentShare > 0
? 1
: 0;
// Direction should match the sign of the pp shift (so title + delta agree)
const direction: 'up' | 'down' | 'flat' =
shareShiftPp > 0 ? 'up' : shareShiftPp < 0 ? 'down' : 'flat';
const changePct = computeChangePct(currentValue, compareValue);
const direction = computeDirection(changePct);
results.push({
ok: true,
@@ -190,30 +212,59 @@ export const geoModule: InsightModule = {
render(result, ctx): RenderedCard {
const country = result.dimensionKey.replace('country:', '');
const shareShiftPp = (result.extra?.shareShiftPp as number) ?? 0;
const isIncrease = shareShiftPp >= 0;
const changePct = result.changePct ?? 0;
const isIncrease = changePct >= 0;
const isNew = result.extra?.isNew as boolean | undefined;
const displayName = getCountry(country);
const title = isNew
? `New traffic from: ${country}`
: `${country} ${isIncrease ? '↑' : '↓'} ${Math.abs(shareShiftPp).toFixed(1)}pp`;
? `New traffic from: ${displayName}`
: `${displayName} ${isIncrease ? '↑' : '↓'} ${Math.abs(changePct * 100).toFixed(0)}%`;
const sessionsCurrent = result.currentValue ?? 0;
const sessionsCompare = result.compareValue ?? 0;
const shareCurrent = Number(result.extra?.currentShare ?? 0);
const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
kind: 'insight_v1',
title,
summary: `${ctx.window.label}. Share shift from ${country}.`,
primaryDimension: { type: 'country', key: country, displayName: country },
tags: [
'geo',
ctx.window.kind,
isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
],
metric: 'share',
extra: {
currentShare: result.extra?.currentShare,
compareShare: result.extra?.compareShare,
shareShiftPp: result.extra?.shareShiftPp,
isNew: result.extra?.isNew,
summary: `${ctx.window.label}. Traffic change from ${displayName}.`,
displayName,
payload: {
kind: 'insight_v1',
dimensions: [
{ key: 'country', value: country, displayName: displayName },
],
primaryMetric: 'sessions',
metrics: {
sessions: {
current: sessionsCurrent,
compare: sessionsCompare,
delta: sessionsCurrent - sessionsCompare,
changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
direction: result.direction ?? 'flat',
unit: 'count',
},
share: {
current: shareCurrent,
compare: shareCompare,
delta: shareCurrent - shareCompare,
changePct:
shareCompare > 0
? (shareCurrent - shareCompare) / shareCompare
: null,
direction:
shareCurrent - shareCompare > 0.0005
? 'up'
: shareCurrent - shareCompare < -0.0005
? 'down'
: 'flat',
unit: 'ratio',
},
},
extra: {
isNew: result.extra?.isNew,
},
},
};
},

312
packages/db/src/services/insights/modules/page-trends.module.ts
View File

@@ -1,26 +1,37 @@
import { TABLE_NAMES } from '../../../clickhouse/client';
import { clix } from '../../../clickhouse/query-builder';
import { normalizePath } from '../normalize';
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
import type {
ComputeContext,
ComputeResult,
InsightModule,
RenderedCard,
} from '../types';
import {
buildLookupMap,
computeChangePct,
computeDirection,
computeWeekdayMedians,
getEndOfDay,
getWeekday,
selectTopDimensions,
} from '../utils';
export const pageTrendsModule: InsightModule = {
key: 'page-trends',
cadence: ['daily'],
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
const DELIMITER = '|||';
async enumerateDimensions(ctx) {
// Query top pages from BOTH current and baseline windows
const [currentResults, baselineResults] = await Promise.all([
clix(ctx.db)
.select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
async function fetchPageTrendAggregates(ctx: ComputeContext): Promise<{
currentMap: Map<string, number>;
baselineMap: Map<string, number>;
totalCurrent: number;
totalBaseline: number;
}> {
if (ctx.window.kind === 'yesterday') {
const [currentResults, baselineResults, totals] = await Promise.all([
ctx
.clix()
.select<{ origin: string; path: string; cnt: number }>([
'origin',
'path',
'count(*) as cnt',
])
.from(TABLE_NAMES.events)
.where('project_id', '=', ctx.projectId)
.where('name', '=', 'screen_view')
@@ -28,65 +39,13 @@ export const pageTrendsModule: InsightModule = {
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.groupBy(['path'])
.orderBy('cnt', 'DESC')
.limit(this.thresholds?.maxDims ?? 100)
.groupBy(['origin', 'path'])
.execute(),
clix(ctx.db)
.select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
.from(TABLE_NAMES.events)
.where('project_id', '=', ctx.projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['path'])
.orderBy('cnt', 'DESC')
.limit(this.thresholds?.maxDims ?? 100)
.execute(),
]);
// Merge both sets
const dims = new Set<string>();
for (const r of currentResults) {
dims.add(`page:${normalizePath(r.path || '/')}`);
}
for (const r of baselineResults) {
dims.add(`page:${normalizePath(r.path || '/')}`);
}
return Array.from(dims);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
// Single query for ALL current values
const currentResults = await clix(ctx.db)
.select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
.from(TABLE_NAMES.events)
.where('project_id', '=', ctx.projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.groupBy(['path'])
.execute();
// Build current lookup map
const currentMap = new Map<string, number>();
for (const r of currentResults) {
const key = normalizePath(r.path || '/');
currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
}
// Single query for baseline
let baselineMap: Map<string, number>;
if (ctx.window.kind === 'yesterday') {
const baselineResults = await clix(ctx.db)
.select<{ date: string; path: string; cnt: number }>([
ctx
.clix()
.select<{ date: string; origin: string; path: string; cnt: number }>([
'toDate(created_at) as date',
'origin',
'path',
'count(*) as cnt',
])
@@ -97,42 +56,142 @@ export const pageTrendsModule: InsightModule = {
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['date', 'path'])
.execute();
const targetWeekday = getWeekday(ctx.window.start);
baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
normalizePath(r.path || '/'),
);
} else {
const baselineResults = await clix(ctx.db)
.select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
.groupBy(['date', 'origin', 'path'])
.execute(),
ctx
.clix()
.select<{ cur_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
),
])
.from(TABLE_NAMES.events)
.where('project_id', '=', ctx.projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
getEndOfDay(ctx.window.end),
])
.groupBy(['path'])
.execute();
.execute(),
]);
baselineMap = new Map<string, number>();
for (const r of baselineResults) {
const key = normalizePath(r.path || '/');
baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
}
}
const currentMap = buildLookupMap(
currentResults,
(r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
);
// Build results from maps
const targetWeekday = getWeekday(ctx.window.start);
const baselineMap = computeWeekdayMedians(
baselineResults,
targetWeekday,
(r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
);
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = Array.from(baselineMap.values()).reduce(
(sum, val) => sum + val,
0,
);
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
const curStart = formatClickhouseDate(ctx.window.start);
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
const [results, totals] = await Promise.all([
ctx
.clix()
.select<{ origin: string; path: string; cur: number; base: number }>([
'origin',
'path',
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
),
])
.from(TABLE_NAMES.events)
.where('project_id', '=', ctx.projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.groupBy(['origin', 'path'])
.execute(),
ctx
.clix()
.select<{ cur_total: number; base_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
),
])
.from(TABLE_NAMES.events)
.where('project_id', '=', ctx.projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.execute(),
]);
const currentMap = buildLookupMap(
results,
(r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
(r) => Number(r.cur ?? 0),
);
const baselineMap = buildLookupMap(
results,
(r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
(r) => Number(r.base ?? 0),
);
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = totals[0]?.base_total ?? 0;
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
export const pageTrendsModule: InsightModule = {
key: 'page-trends',
cadence: ['daily'],
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
async enumerateDimensions(ctx) {
const { currentMap, baselineMap } = await fetchPageTrendAggregates(ctx);
const topDims = selectTopDimensions(
currentMap,
baselineMap,
this.thresholds?.maxDims ?? 100,
);
return topDims.map((dim) => `page:${dim}`);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
await fetchPageTrendAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
if (!dimKey.startsWith('page:')) continue;
const pagePath = dimKey.replace('page:', '');
const originPath = dimKey.replace('page:', '');
const currentValue = currentMap.get(pagePath) ?? 0;
const compareValue = baselineMap.get(pagePath) ?? 0;
const currentValue = currentMap.get(originPath) ?? 0;
const compareValue = baselineMap.get(originPath) ?? 0;
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
const shareShiftPp = (currentShare - compareShare) * 100;
const changePct = computeChangePct(currentValue, compareValue);
const direction = computeDirection(changePct);
@@ -144,6 +203,9 @@ export const pageTrendsModule: InsightModule = {
changePct,
direction,
extra: {
shareShiftPp,
currentShare,
compareShare,
isNew: compareValue === 0 && currentValue > 0,
},
});
@@ -153,28 +215,62 @@ export const pageTrendsModule: InsightModule = {
},
render(result, ctx): RenderedCard {
const path = result.dimensionKey.replace('page:', '');
const originPath = result.dimensionKey.replace('page:', '');
const [origin, path] = originPath.split(DELIMITER);
const displayValue = origin ? `${origin}${path}` : path || '/';
const pct = ((result.changePct ?? 0) * 100).toFixed(1);
const isIncrease = (result.changePct ?? 0) >= 0;
const isNew = result.extra?.isNew as boolean | undefined;
const title = isNew
? `New page getting views: ${path}`
: `Page ${path} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
? `New page getting views: ${displayValue}`
: `Page ${displayValue} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
const pageviewsCurrent = result.currentValue ?? 0;
const pageviewsCompare = result.compareValue ?? 0;
const shareCurrent = Number(result.extra?.currentShare ?? 0);
const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
kind: 'insight_v1',
title,
summary: `${ctx.window.label}. Pageviews ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
primaryDimension: { type: 'page', key: path, displayName: path },
tags: [
'page-trends',
ctx.window.kind,
isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
],
metric: 'pageviews',
extra: {
isNew: result.extra?.isNew,
summary: `${ctx.window.label}. Pageviews ${pageviewsCurrent} vs ${pageviewsCompare}.`,
displayName: displayValue,
payload: {
kind: 'insight_v1',
dimensions: [
{ key: 'origin', value: origin ?? '', displayName: origin ?? '' },
{ key: 'path', value: path ?? '', displayName: path ?? '' },
],
primaryMetric: 'pageviews',
metrics: {
pageviews: {
current: pageviewsCurrent,
compare: pageviewsCompare,
delta: pageviewsCurrent - pageviewsCompare,
changePct: pageviewsCompare > 0 ? (result.changePct ?? 0) : null,
direction: result.direction ?? 'flat',
unit: 'count',
},
share: {
current: shareCurrent,
compare: shareCompare,
delta: shareCurrent - shareCompare,
changePct:
shareCompare > 0
? (shareCurrent - shareCompare) / shareCompare
: null,
direction:
shareCurrent - shareCompare > 0.0005
? 'up'
: shareCurrent - shareCompare < -0.0005
? 'down'
: 'flat',
unit: 'ratio',
},
},
extra: {
isNew: result.extra?.isNew,
},
},
};
},

303
packages/db/src/services/insights/modules/referrers.module.ts
View File

@@ -1,26 +1,30 @@
import { TABLE_NAMES } from '../../../clickhouse/client';
import { clix } from '../../../clickhouse/query-builder';
import { normalizeReferrer } from '../normalize';
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
import type {
ComputeContext,
ComputeResult,
InsightModule,
RenderedCard,
} from '../types';
import {
buildLookupMap,
computeChangePct,
computeDirection,
computeWeekdayMedians,
getEndOfDay,
getWeekday,
selectTopDimensions,
} from '../utils';
export const referrersModule: InsightModule = {
key: 'referrers',
cadence: ['daily'],
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
thresholds: { minTotal: 100, minAbsDelta: 20, minPct: 0.15, maxDims: 50 },
async enumerateDimensions(ctx) {
// Query top referrers from BOTH current and baseline windows
// This allows detecting new sources that didn't exist in baseline
const [currentResults, baselineResults] = await Promise.all([
clix(ctx.db)
async function fetchReferrerAggregates(ctx: ComputeContext): Promise<{
currentMap: Map<string, number>;
baselineMap: Map<string, number>;
totalCurrent: number;
totalBaseline: number;
}> {
if (ctx.window.kind === 'yesterday') {
const [currentResults, baselineResults, totals] = await Promise.all([
ctx
.clix()
.select<{ referrer_name: string; cnt: number }>([
'referrer_name',
'count(*) as cnt',
@@ -33,69 +37,9 @@ export const referrersModule: InsightModule = {
getEndOfDay(ctx.window.end),
])
.groupBy(['referrer_name'])
.orderBy('cnt', 'DESC')
.limit(this.thresholds?.maxDims ?? 50)
.execute(),
clix(ctx.db)
.select<{ referrer_name: string; cnt: number }>([
'referrer_name',
'count(*) as cnt',
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['referrer_name'])
.orderBy('cnt', 'DESC')
.limit(this.thresholds?.maxDims ?? 50)
.execute(),
]);
// Merge both sets to catch new/emerging sources
const dims = new Set<string>();
for (const r of currentResults) {
dims.add(`referrer:${normalizeReferrer(r.referrer_name || 'direct')}`);
}
for (const r of baselineResults) {
dims.add(`referrer:${normalizeReferrer(r.referrer_name || 'direct')}`);
}
return Array.from(dims);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
// Single query for ALL current values (batched)
const currentResults = await clix(ctx.db)
.select<{ referrer_name: string; cnt: number }>([
'referrer_name',
'count(*) as cnt',
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.start,
getEndOfDay(ctx.window.end),
])
.groupBy(['referrer_name'])
.execute();
// Build current lookup map
const currentMap = new Map<string, number>();
for (const r of currentResults) {
const key = normalizeReferrer(r.referrer_name || 'direct');
currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
}
// Single query for baseline (with date breakdown for weekday median if needed)
let baselineMap: Map<string, number>;
if (ctx.window.kind === 'yesterday') {
// Need daily breakdown for weekday median calculation
const baselineResults = await clix(ctx.db)
ctx
.clix()
.select<{ date: string; referrer_name: string; cnt: number }>([
'toDate(created_at) as date',
'referrer_name',
@@ -109,37 +53,127 @@ export const referrersModule: InsightModule = {
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['date', 'referrer_name'])
.execute();
const targetWeekday = getWeekday(ctx.window.start);
baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
normalizeReferrer(r.referrer_name || 'direct'),
);
} else {
// Rolling windows: simple aggregate
const baselineResults = await clix(ctx.db)
.select<{ referrer_name: string; cnt: number }>([
'referrer_name',
'count(*) as cnt',
.execute(),
ctx
.clix()
.select<{ cur_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
getEndOfDay(ctx.window.end),
])
.groupBy(['referrer_name'])
.execute();
.execute(),
]);
baselineMap = new Map<string, number>();
for (const r of baselineResults) {
const key = normalizeReferrer(r.referrer_name || 'direct');
baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
}
}
const currentMap = buildLookupMap(
currentResults,
(r) => r.referrer_name || 'direct',
);
// Build results from maps (in memory, no more queries!)
const targetWeekday = getWeekday(ctx.window.start);
const baselineMap = computeWeekdayMedians(
baselineResults,
targetWeekday,
(r) => r.referrer_name || 'direct',
);
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = Array.from(baselineMap.values()).reduce(
(sum, val) => sum + val,
0,
);
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
const curStart = formatClickhouseDate(ctx.window.start);
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
const [results, totals] = await Promise.all([
ctx
.clix()
.select<{ referrer_name: string; cur: number; base: number }>([
'referrer_name',
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.groupBy(['referrer_name'])
.execute(),
ctx
.clix()
.select<{ cur_total: number; base_total: number }>([
ctx.clix.exp(
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
),
ctx.clix.exp(
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
.execute(),
]);
const currentMap = buildLookupMap(
results,
(r) => r.referrer_name || 'direct',
(r) => Number(r.cur ?? 0),
);
const baselineMap = buildLookupMap(
results,
(r) => r.referrer_name || 'direct',
(r) => Number(r.base ?? 0),
);
const totalCurrent = totals[0]?.cur_total ?? 0;
const totalBaseline = totals[0]?.base_total ?? 0;
return { currentMap, baselineMap, totalCurrent, totalBaseline };
}
export const referrersModule: InsightModule = {
key: 'referrers',
cadence: ['daily'],
thresholds: { minTotal: 100, minAbsDelta: 20, minPct: 0.15, maxDims: 50 },
async enumerateDimensions(ctx) {
const { currentMap, baselineMap } = await fetchReferrerAggregates(ctx);
const topDims = selectTopDimensions(
currentMap,
baselineMap,
this.thresholds?.maxDims ?? 50,
);
return topDims.map((dim) => `referrer:${dim}`);
},
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
await fetchReferrerAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
@@ -148,6 +182,11 @@ export const referrersModule: InsightModule = {
const currentValue = currentMap.get(referrerName) ?? 0;
const compareValue = baselineMap.get(referrerName) ?? 0;
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
const shareShiftPp = (currentShare - compareShare) * 100;
const changePct = computeChangePct(currentValue, compareValue);
const direction = computeDirection(changePct);
@@ -159,6 +198,9 @@ export const referrersModule: InsightModule = {
changePct,
direction,
extra: {
shareShiftPp,
currentShare,
compareShare,
isNew: compareValue === 0 && currentValue > 0,
isGone: currentValue === 0 && compareValue > 0,
},
@@ -178,24 +220,55 @@ export const referrersModule: InsightModule = {
? `New traffic source: ${referrer}`
: `Traffic from ${referrer} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
const sessionsCurrent = result.currentValue ?? 0;
const sessionsCompare = result.compareValue ?? 0;
const shareCurrent = Number(result.extra?.currentShare ?? 0);
const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
kind: 'insight_v1',
title,
summary: `${ctx.window.label}. Sessions ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
primaryDimension: {
type: 'referrer',
key: referrer,
displayName: referrer,
},
tags: [
'referrers',
ctx.window.kind,
isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
],
metric: 'sessions',
extra: {
isNew: result.extra?.isNew,
isGone: result.extra?.isGone,
summary: `${ctx.window.label}. Sessions ${sessionsCurrent} vs ${sessionsCompare}.`,
displayName: referrer,
payload: {
kind: 'insight_v1',
dimensions: [
{
key: 'referrer_name',
value: referrer,
displayName: referrer,
},
],
primaryMetric: 'sessions',
metrics: {
sessions: {
current: sessionsCurrent,
compare: sessionsCompare,
delta: sessionsCurrent - sessionsCompare,
changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
direction: result.direction ?? 'flat',
unit: 'count',
},
share: {
current: shareCurrent,
compare: shareCompare,
delta: shareCurrent - shareCompare,
changePct:
shareCompare > 0
? (shareCurrent - shareCompare) / shareCompare
: null,
direction:
shareCurrent - shareCompare > 0.0005
? 'up'
: shareCurrent - shareCompare < -0.0005
? 'down'
: 'flat',
unit: 'ratio',
},
},
extra: {
isNew: result.extra?.isNew,
isGone: result.extra?.isGone,
},
},
};
},

80
packages/db/src/services/insights/normalize.ts
View File

@@ -1,80 +0,0 @@
export function normalizeReferrer(name: string): string {
if (!name || name === '') return 'direct';
const normalized = name.toLowerCase().trim();
// Normalize common referrer variations
const map: Record<string, string> = {
'm.instagram.com': 'instagram',
'l.instagram.com': 'instagram',
'www.instagram.com': 'instagram',
'instagram.com': 'instagram',
't.co': 'twitter',
'twitter.com': 'twitter',
'x.com': 'twitter',
'lm.facebook.com': 'facebook',
'm.facebook.com': 'facebook',
'facebook.com': 'facebook',
'l.facebook.com': 'facebook',
'linkedin.com': 'linkedin',
'www.linkedin.com': 'linkedin',
'youtube.com': 'youtube',
'www.youtube.com': 'youtube',
'm.youtube.com': 'youtube',
'reddit.com': 'reddit',
'www.reddit.com': 'reddit',
'tiktok.com': 'tiktok',
'www.tiktok.com': 'tiktok',
};
// Check exact match first
if (map[normalized]) {
return map[normalized];
}
// Check if it contains any of the mapped domains
for (const [key, value] of Object.entries(map)) {
if (normalized.includes(key)) {
return value;
}
}
// Extract domain from URL if present
try {
const url = normalized.startsWith('http')
? normalized
: `https://${normalized}`;
const hostname = new URL(url).hostname;
// Remove www. prefix
return hostname.replace(/^www\./, '');
} catch {
// If not a valid URL, return as-is
return normalized || 'direct';
}
}
export function normalizePath(path: string): string {
if (!path || path === '') return '/';
try {
// If it's a full URL, extract pathname
const url = path.startsWith('http')
? new URL(path)
: new URL(path, 'http://x');
const pathname = url.pathname;
// Normalize trailing slash (remove unless it's root)
return pathname === '/' ? '/' : pathname.replace(/\/$/, '');
} catch {
// If not a valid URL, treat as path
return path === '/' ? '/' : path.replace(/\/$/, '') || '/';
}
}
export function normalizeUtmCombo(source: string, medium: string): string {
const s = (source || '').toLowerCase().trim();
const m = (medium || '').toLowerCase().trim();
if (!s && !m) return 'none';
if (!s) return `utm:${m}`;
if (!m) return `utm:${s}`;
return `utm:${s}/${m}`;
}

6
packages/db/src/services/insights/scoring.ts
View File

@@ -11,8 +11,8 @@ export function severityBand(
changePct?: number | null,
): 'low' | 'moderate' | 'severe' | null {
const p = Math.abs(changePct ?? 0);
if (p < 0.05) return null;
if (p < 0.15) return 'low';
if (p < 0.3) return 'moderate';
if (p < 0.1) return null;
if (p < 0.5) return 'low';
if (p < 1) return 'moderate';
return 'severe';
}

20
packages/db/src/services/insights/store.ts
View File

@@ -13,6 +13,8 @@ export const insightStore: InsightStore = {
const projects = await db.project.findMany({
where: {
deleteAt: null,
eventsCount: { gt: 10_000 },
updatedAt: { gt: new Date(Date.now() - 1000 * 60 * 60 * 24) },
organization: {
subscriptionStatus: 'active',
},
@@ -22,6 +24,14 @@ export const insightStore: InsightStore = {
return projects.map((p) => p.id);
},
async getProjectCreatedAt(projectId: string): Promise<Date | null> {
const project = await db.project.findFirst({
where: { id: projectId, deleteAt: null },
select: { createdAt: true },
});
return project?.createdAt ?? null;
},
async getActiveInsightByIdentity({
projectId,
moduleKey,
@@ -52,7 +62,6 @@ export const insightStore: InsightStore = {
lastSeenAt: insight.lastSeenAt,
lastUpdatedAt: insight.lastUpdatedAt,
direction: insight.direction,
changePct: insight.changePct,
severityBand: insight.severityBand,
};
},
@@ -68,8 +77,6 @@ export const insightStore: InsightStore = {
decision,
prev,
}): Promise<PersistedInsight> {
const payloadData = (card.payload ?? card) as Prisma.InputJsonValue;
const baseData = {
projectId,
moduleKey,
@@ -78,10 +85,8 @@ export const insightStore: InsightStore = {
state: prev?.state === 'closed' ? 'active' : (prev?.state ?? 'active'),
title: card.title,
summary: card.summary ?? null,
payload: payloadData as Prisma.InputJsonValue,
currentValue: metrics.currentValue ?? null,
compareValue: metrics.compareValue ?? null,
changePct: metrics.changePct ?? null,
displayName: card.displayName,
payload: card.payload,
direction: metrics.direction ?? null,
impactScore: metrics.impactScore,
severityBand: metrics.severityBand ?? null,
@@ -161,7 +166,6 @@ export const insightStore: InsightStore = {
lastSeenAt: insight.lastSeenAt,
lastUpdatedAt: insight.lastUpdatedAt,
direction: insight.direction,
changePct: insight.changePct,
severityBand: insight.severityBand,
};
},

65
packages/db/src/services/insights/types.ts
View File

@@ -1,4 +1,11 @@
export type Cadence = 'hourly' | 'daily' | 'weekly';
import type {
InsightDimension,
InsightMetricEntry,
InsightMetricKey,
InsightPayload,
} from '@openpanel/validation';
export type Cadence = 'daily';
export type WindowKind = 'yesterday' | 'rolling_7d' | 'rolling_30d';
@@ -17,6 +24,12 @@ export interface ComputeContext {
db: any; // your DB client
now: Date;
logger: Pick<Console, 'info' | 'warn' | 'error'>;
/**
* Cached clix function that automatically caches query results based on query hash.
* This eliminates duplicate queries within the same module+window context.
* Use this instead of importing clix directly to benefit from automatic caching.
*/
clix: ReturnType<typeof import('./cached-clix').createCachedClix>;
}
export interface ComputeResult {
@@ -29,31 +42,22 @@ export interface ComputeResult {
extra?: Record<string, unknown>; // share delta pp, rank, sparkline, etc.
}
// Types imported from @openpanel/validation:
// - InsightMetricKey
// - InsightMetricEntry
// - InsightDimension
// - InsightPayload
/**
* Render should be deterministic and safe to call multiple times.
* Raw computed values (currentValue, compareValue, changePct, direction)
* are stored in top-level DB fields. The payload only contains display
* metadata and module-specific extra data for frontend flexibility.
* Returns the shape that matches ProjectInsight create input.
* The payload contains all metric data and display metadata.
*/
export interface RenderedCard {
kind?: 'insight_v1';
title: string;
summary?: string;
tags?: string[];
primaryDimension?: { type: string; key: string; displayName?: string };
/**
* What metric this insight measures - frontend uses this to format values.
* 'sessions' | 'pageviews' for absolute counts
* 'share' for percentage-based (geo, devices)
*/
metric?: 'sessions' | 'pageviews' | 'share';
/**
* Module-specific extra data (e.g., share values for geo/devices).
* Frontend can use this based on moduleKey.
*/
extra?: Record<string, unknown>;
displayName: string;
payload: InsightPayload; // Contains dimensions, primaryMetric, metrics, extra
}
/** Optional per-module thresholds (the engine can still apply global defaults) */
@@ -67,7 +71,8 @@ export interface ModuleThresholds {
export interface InsightModule {
key: string;
cadence: Cadence[];
windows: WindowKind[];
/** Optional per-module override; engine applies a default if omitted. */
windows?: WindowKind[];
thresholds?: ModuleThresholds;
enumerateDimensions?(ctx: ComputeContext): Promise<string[]>;
/** Preferred path: batch compute many dimensions in one go. */
@@ -99,7 +104,6 @@ export interface PersistedInsight {
lastSeenAt: Date;
lastUpdatedAt: Date;
direction?: string | null;
changePct?: number | null;
severityBand?: string | null;
}
@@ -124,6 +128,8 @@ export interface MaterialDecision {
*/
export interface InsightStore {
listProjectIdsForCadence(cadence: Cadence): Promise<string[]>;
/** Used by the engine/worker to decide if a window has enough baseline history. */
getProjectCreatedAt(projectId: string): Promise<Date | null>;
getActiveInsightByIdentity(args: {
projectId: string;
moduleKey: string;
@@ -137,9 +143,6 @@ export interface InsightStore {
window: WindowRange;
card: RenderedCard;
metrics: {
currentValue?: number;
compareValue?: number;
changePct?: number;
direction?: 'up' | 'down' | 'flat';
impactScore: number;
severityBand?: string | null;
@@ -186,15 +189,3 @@ export interface InsightStore {
now: Date;
}): Promise<{ suppressed: number; unsuppressed: number }>;
}
export interface ExplainQueue {
enqueueExplain(job: {
insightId: string;
projectId: string;
moduleKey: string;
dimensionKey: string;
windowKind: WindowKind;
evidence: Record<string, unknown>;
evidenceHash: string;
}): Promise<void>;
}

76
packages/db/src/services/insights/utils.ts
View File

@@ -29,9 +29,7 @@ export function computeMedian(sortedValues: number[]): number {
* @param getDimension - Function to extract normalized dimension from row
* @returns Map of dimension -> median value
*/
export function computeWeekdayMedians<
T extends { date: string; cnt: number | string },
>(
export function computeWeekdayMedians<T>(
data: T[],
targetWeekday: number,
getDimension: (row: T) => string,
@@ -40,12 +38,12 @@ export function computeWeekdayMedians<
const byDimension = new Map<string, number[]>();
for (const row of data) {
const rowWeekday = getWeekday(new Date(row.date));
const rowWeekday = getWeekday(new Date((row as any).date));
if (rowWeekday !== targetWeekday) continue;
const dim = getDimension(row);
const values = byDimension.get(dim) ?? [];
values.push(Number(row.cnt ?? 0));
values.push(Number((row as any).cnt ?? 0));
byDimension.set(dim, values);
}
@@ -87,19 +85,6 @@ export function computeDirection(
: 'flat';
}
/**
* Merge dimension sets from current and baseline to detect new/gone dimensions
*/
export function mergeDimensionSets(
currentDims: Set<string>,
baselineDims: Set<string>,
): string[] {
const merged = new Set<string>();
for (const dim of currentDims) merged.add(dim);
for (const dim of baselineDims) merged.add(dim);
return Array.from(merged);
}
/**
* Get end of day timestamp (23:59:59.999) for a given date.
* Used to ensure BETWEEN queries include the full day.
@@ -109,3 +94,58 @@ export function getEndOfDay(date: Date): Date {
end.setUTCHours(23, 59, 59, 999);
return end;
}
/**
* Build a lookup map from query results.
* Aggregates counts by key, handling duplicate keys by summing values.
*
* @param results - Array of result rows
* @param getKey - Function to extract the key from each row
* @param getCount - Function to extract the count from each row (defaults to 'cnt' field)
* @returns Map of key -> aggregated count
*/
export function buildLookupMap<T>(
results: T[],
getKey: (row: T) => string,
getCount: (row: T) => number = (row) => Number((row as any).cnt ?? 0),
): Map<string, number> {
const map = new Map<string, number>();
for (const row of results) {
const key = getKey(row);
const cnt = getCount(row);
map.set(key, (map.get(key) ?? 0) + cnt);
}
return map;
}
/**
* Select top-N dimensions by ranking on greatest(current, baseline).
* This preserves union behavior: dimensions with high values in either period are included.
*
* @param currentMap - Map of dimension -> current value
* @param baselineMap - Map of dimension -> baseline value
* @param maxDims - Maximum number of dimensions to return
* @returns Array of dimension keys, ranked by greatest(current, baseline)
*/
export function selectTopDimensions(
currentMap: Map<string, number>,
baselineMap: Map<string, number>,
maxDims: number,
): string[] {
// Merge all dimensions from both maps
const allDims = new Set<string>();
for (const dim of currentMap.keys()) allDims.add(dim);
for (const dim of baselineMap.keys()) allDims.add(dim);
// Rank by greatest(current, baseline)
const ranked = Array.from(allDims)
.map((dim) => ({
dim,
maxValue: Math.max(currentMap.get(dim) ?? 0, baselineMap.get(dim) ?? 0),
}))
.sort((a, b) => b.maxValue - a.maxValue)
.slice(0, maxDims)
.map((x) => x.dim);
return ranked;
}

2
packages/db/src/types.ts
View File

@@ -3,6 +3,7 @@ import type {
IIntegrationConfig,
INotificationRuleConfig,
IProjectFilters,
InsightPayload,
} from '@openpanel/validation';
import type {
IClickhouseBotEvent,
@@ -18,6 +19,7 @@ declare global {
type IPrismaIntegrationConfig = IIntegrationConfig;
type IPrismaNotificationPayload = INotificationPayload;
type IPrismaProjectFilters = IProjectFilters[];
type IPrismaProjectInsightPayload = InsightPayload;
type IPrismaClickhouseEvent = IClickhouseEvent;
type IPrismaClickhouseProfile = IClickhouseProfile;
type IPrismaClickhouseBotEvent = IClickhouseBotEvent;