+ {groupedByModule.map(([moduleKey, moduleInsights]) => (
+
+
+
+ {getModuleDisplayName(moduleKey)}
+
+
+ {moduleInsights.length}{' '}
+ {moduleInsights.length === 1 ? 'insight' : 'insights'}
+
+
+
+
+
+ {moduleInsights.map((insight, index) => (
+
+ {
+ const filterString = insight.payload?.dimensions
+ .map(
+ (dim) =>
+ `${dim.key},is,${encodeURIComponent(dim.value)}`,
+ )
+ .join(';');
+ if (filterString) {
+ return () => {
+ navigate({
+ to: '/$organizationId/$projectId',
+ from: Route.fullPath,
+ search: {
+ f: filterString,
+ },
+ });
+ };
+ }
+ return undefined;
+ })()}
+ />
+
+ ))}
+
+
+
+
+
+
+ ))}
+
+ )}
{filteredAndSorted.length > 0 && (
-
+
Showing {filteredAndSorted.length} of {insights?.length ?? 0} insights
)}
diff --git a/apps/start/src/utils/title.ts b/apps/start/src/utils/title.ts
index e735636e..47b43fb8 100644
--- a/apps/start/src/utils/title.ts
+++ b/apps/start/src/utils/title.ts
@@ -90,6 +90,7 @@ export const PAGE_TITLES = {
CHAT: 'AI Assistant',
REALTIME: 'Realtime',
REFERENCES: 'References',
+ INSIGHTS: 'Insights',
// Profiles
PROFILES: 'Profiles',
PROFILE_EVENTS: 'Profile events',
diff --git a/apps/worker/src/boot-cron.ts b/apps/worker/src/boot-cron.ts
index 8e5b30fd..a67c1e7f 100644
--- a/apps/worker/src/boot-cron.ts
+++ b/apps/worker/src/boot-cron.ts
@@ -34,6 +34,11 @@ export async function bootCron() {
type: 'flushSessions',
pattern: 1000 * 10,
},
+ {
+ name: 'insightsDaily',
+ type: 'insightsDaily',
+ pattern: '0 2 * * *',
+ },
];
if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {
@@ -44,12 +49,6 @@ export async function bootCron() {
});
}
- jobs.push({
- name: 'insightsDaily',
- type: 'insightsDaily',
- pattern: '0 2 * * *', // 2 AM daily
- });
-
logger.info('Updating cron jobs');
const jobSchedulers = await cronQueue.getJobSchedulers();
diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts
index 1109e6e0..274c9826 100644
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -79,11 +79,6 @@ async function start() {
await bootCron();
} else {
logger.warn('Workers are disabled');
-
- // Start insights worker
- const insightsWorker = new Worker(insightsQueue.name, insightsProjectJob, {
- connection: getRedisQueue(),
- });
}
await createInitialSalts();
diff --git a/apps/worker/src/jobs/insights.ts b/apps/worker/src/jobs/insights.ts
index b418dd69..a0156e0d 100644
--- a/apps/worker/src/jobs/insights.ts
+++ b/apps/worker/src/jobs/insights.ts
@@ -16,7 +16,7 @@ import { insightsQueue } from '@openpanel/queue';
import type { Job } from 'bullmq';
const defaultEngineConfig = {
- keepTopNPerModuleWindow: 5,
+ keepTopNPerModuleWindow: 20,
closeStaleAfterDays: 7,
dimensionBatchSize: 50,
globalThresholds: {
@@ -24,8 +24,6 @@ const defaultEngineConfig = {
minAbsDelta: 80,
minPct: 0.15,
},
- enableExplain: false,
- explainTopNPerProjectPerDay: 3,
};
export async function insightsDailyJob(job: Job
) {
@@ -63,9 +61,12 @@ export async function insightsProjectJob(
config: defaultEngineConfig,
});
+ const projectCreatedAt = await insightStore.getProjectCreatedAt(projectId);
+
await engine.runProject({
projectId,
cadence: 'daily',
now: new Date(date),
+ projectCreatedAt,
});
}
diff --git a/packages/constants/index.ts b/packages/constants/index.ts
index 17cd2d15..aff1b11d 100644
--- a/packages/constants/index.ts
+++ b/packages/constants/index.ts
@@ -245,3 +245,259 @@ export function getDefaultIntervalByDates(
return null;
}
+
+export const countries = {
+ AF: 'Afghanistan',
+ AL: 'Albania',
+ DZ: 'Algeria',
+ AS: 'American Samoa',
+ AD: 'Andorra',
+ AO: 'Angola',
+ AI: 'Anguilla',
+ AQ: 'Antarctica',
+ AG: 'Antigua and Barbuda',
+ AR: 'Argentina',
+ AM: 'Armenia',
+ AW: 'Aruba',
+ AU: 'Australia',
+ AT: 'Austria',
+ AZ: 'Azerbaijan',
+ BS: 'Bahamas',
+ BH: 'Bahrain',
+ BD: 'Bangladesh',
+ BB: 'Barbados',
+ BY: 'Belarus',
+ BE: 'Belgium',
+ BZ: 'Belize',
+ BJ: 'Benin',
+ BM: 'Bermuda',
+ BT: 'Bhutan',
+ BO: 'Bolivia',
+ BQ: 'Bonaire, Sint Eustatius and Saba',
+ BA: 'Bosnia and Herzegovina',
+ BW: 'Botswana',
+ BV: 'Bouvet Island',
+ BR: 'Brazil',
+ IO: 'British Indian Ocean Territory',
+ BN: 'Brunei Darussalam',
+ BG: 'Bulgaria',
+ BF: 'Burkina Faso',
+ BI: 'Burundi',
+ CV: 'Cabo Verde',
+ KH: 'Cambodia',
+ CM: 'Cameroon',
+ CA: 'Canada',
+ KY: 'Cayman Islands',
+ CF: 'Central African Republic',
+ TD: 'Chad',
+ CL: 'Chile',
+ CN: 'China',
+ CX: 'Christmas Island',
+ CC: 'Cocos (Keeling) Islands',
+ CO: 'Colombia',
+ KM: 'Comoros',
+ CD: 'Congo (Democratic Republic)',
+ CG: 'Congo',
+ CK: 'Cook Islands',
+ CR: 'Costa Rica',
+ HR: 'Croatia',
+ CU: 'Cuba',
+ CW: 'Curaçao',
+ CY: 'Cyprus',
+ CZ: 'Czechia',
+ CI: "Côte d'Ivoire",
+ DK: 'Denmark',
+ DJ: 'Djibouti',
+ DM: 'Dominica',
+ DO: 'Dominican Republic',
+ EC: 'Ecuador',
+ EG: 'Egypt',
+ SV: 'El Salvador',
+ GQ: 'Equatorial Guinea',
+ ER: 'Eritrea',
+ EE: 'Estonia',
+ SZ: 'Eswatina',
+ ET: 'Ethiopia',
+ FK: 'Falkland Islands',
+ FO: 'Faroe Islands',
+ FJ: 'Fiji',
+ FI: 'Finland',
+ FR: 'France',
+ GF: 'French Guiana',
+ PF: 'French Polynesia',
+ TF: 'French Southern Territories',
+ GA: 'Gabon',
+ GM: 'Gambia',
+ GE: 'Georgia',
+ DE: 'Germany',
+ GH: 'Ghana',
+ GI: 'Gibraltar',
+ GR: 'Greece',
+ GL: 'Greenland',
+ GD: 'Grenada',
+ GP: 'Guadeloupe',
+ GU: 'Guam',
+ GT: 'Guatemala',
+ GG: 'Guernsey',
+ GN: 'Guinea',
+ GW: 'Guinea-Bissau',
+ GY: 'Guyana',
+ HT: 'Haiti',
+ HM: 'Heard Island and McDonald Islands',
+ VA: 'Holy See',
+ HN: 'Honduras',
+ HK: 'Hong Kong',
+ HU: 'Hungary',
+ IS: 'Iceland',
+ IN: 'India',
+ ID: 'Indonesia',
+ IR: 'Iran',
+ IQ: 'Iraq',
+ IE: 'Ireland',
+ IM: 'Isle of Man',
+ IL: 'Israel',
+ IT: 'Italy',
+ JM: 'Jamaica',
+ JP: 'Japan',
+ JE: 'Jersey',
+ JO: 'Jordan',
+ KZ: 'Kazakhstan',
+ KE: 'Kenya',
+ KI: 'Kiribati',
+ KP: "Korea (Democratic People's Republic)",
+ KR: 'Korea (Republic)',
+ KW: 'Kuwait',
+ KG: 'Kyrgyzstan',
+ LA: "Lao People's Democratic Republic",
+ LV: 'Latvia',
+ LB: 'Lebanon',
+ LS: 'Lesotho',
+ LR: 'Liberia',
+ LY: 'Libya',
+ LI: 'Liechtenstein',
+ LT: 'Lithuania',
+ LU: 'Luxembourg',
+ MO: 'Macao',
+ MG: 'Madagascar',
+ MW: 'Malawi',
+ MY: 'Malaysia',
+ MV: 'Maldives',
+ ML: 'Mali',
+ MT: 'Malta',
+ MH: 'Marshall Islands',
+ MQ: 'Martinique',
+ MR: 'Mauritania',
+ MU: 'Mauritius',
+ YT: 'Mayotte',
+ MX: 'Mexico',
+ FM: 'Micronesia',
+ MD: 'Moldova',
+ MC: 'Monaco',
+ MN: 'Mongolia',
+ ME: 'Montenegro',
+ MS: 'Montserrat',
+ MA: 'Morocco',
+ MZ: 'Mozambique',
+ MM: 'Myanmar',
+ NA: 'Namibia',
+ NR: 'Nauru',
+ NP: 'Nepal',
+ NL: 'Netherlands',
+ NC: 'New Caledonia',
+ NZ: 'New Zealand',
+ NI: 'Nicaragua',
+ NE: 'Niger',
+ NG: 'Nigeria',
+ NU: 'Niue',
+ NF: 'Norfolk Island',
+ MP: 'Northern Mariana Islands',
+ NO: 'Norway',
+ OM: 'Oman',
+ PK: 'Pakistan',
+ PW: 'Palau',
+ PS: 'Palestine, State of',
+ PA: 'Panama',
+ PG: 'Papua New Guinea',
+ PY: 'Paraguay',
+ PE: 'Peru',
+ PH: 'Philippines',
+ PN: 'Pitcairn',
+ PL: 'Poland',
+ PT: 'Portugal',
+ PR: 'Puerto Rico',
+ QA: 'Qatar',
+ MK: 'Republic of North Macedonia',
+ RO: 'Romania',
+ RU: 'Russian Federation',
+ RW: 'Rwanda',
+ RE: 'Réunion',
+ BL: 'Saint Barthélemy',
+ SH: 'Saint Helena, Ascension and Tristan da Cunha',
+ KN: 'Saint Kitts and Nevis',
+ LC: 'Saint Lucia',
+ MF: 'Saint Martin (French part)',
+ PM: 'Saint Pierre and Miquelon',
+ VC: 'Saint Vincent and the Grenadines',
+ WS: 'Samoa',
+ SM: 'San Marino',
+ ST: 'Sao Tome and Principe',
+ SA: 'Saudi Arabia',
+ SN: 'Senegal',
+ RS: 'Serbia',
+ SC: 'Seychelles',
+ SL: 'Sierra Leone',
+ SG: 'Singapore',
+ SX: 'Sint Maarten (Dutch part)',
+ SK: 'Slovakia',
+ SI: 'Slovenia',
+ SB: 'Solomon Islands',
+ SO: 'Somalia',
+ ZA: 'South Africa',
+ GS: 'South Georgia and the South Sandwich Islands',
+ SS: 'South Sudan',
+ ES: 'Spain',
+ LK: 'Sri Lanka',
+ SD: 'Sudan',
+ SR: 'Suriname',
+ SJ: 'Svalbard and Jan Mayen',
+ SE: 'Sweden',
+ CH: 'Switzerland',
+ SY: 'Syrian Arab Republic',
+ TW: 'Taiwan',
+ TJ: 'Tajikistan',
+ TZ: 'Tanzania, United Republic of',
+ TH: 'Thailand',
+ TL: 'Timor-Leste',
+ TG: 'Togo',
+ TK: 'Tokelau',
+ TO: 'Tonga',
+ TT: 'Trinidad and Tobago',
+ TN: 'Tunisia',
+ TR: 'Turkey',
+ TM: 'Turkmenistan',
+ TC: 'Turks and Caicos Islands',
+ TV: 'Tuvalu',
+ UG: 'Uganda',
+ UA: 'Ukraine',
+ AE: 'United Arab Emirates',
+ GB: 'United Kingdom',
+ US: 'United States',
+ UM: 'United States Minor Outlying Islands',
+ UY: 'Uruguay',
+ UZ: 'Uzbekistan',
+ VU: 'Vanuatu',
+ VE: 'Venezuela',
+ VN: 'Viet Nam',
+ VG: 'Virgin Islands (British)',
+ VI: 'Virgin Islands (U.S.)',
+ WF: 'Wallis and Futuna',
+ EH: 'Western Sahara',
+ YE: 'Yemen',
+ ZM: 'Zambia',
+ ZW: 'Zimbabwe',
+ AX: 'Åland Islands',
+} as const;
+
+export function getCountry(code?: string) {
+ return countries[code as keyof typeof countries];
+}
diff --git a/packages/db/prisma/migrations/20251217204808_insight_payload_default/migration.sql b/packages/db/prisma/migrations/20251217204808_insight_payload_default/migration.sql
new file mode 100644
index 00000000..6ad792ee
--- /dev/null
+++ b/packages/db/prisma/migrations/20251217204808_insight_payload_default/migration.sql
@@ -0,0 +1,9 @@
+/*
+ Warnings:
+
+ - Made the column `payload` on table `project_insights` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "public"."project_insights" ALTER COLUMN "payload" SET NOT NULL,
+ALTER COLUMN "payload" SET DEFAULT '{}';
diff --git a/packages/db/prisma/migrations/20251217210920_insights/migration.sql b/packages/db/prisma/migrations/20251217210920_insights/migration.sql
new file mode 100644
index 00000000..56a4b038
--- /dev/null
+++ b/packages/db/prisma/migrations/20251217210920_insights/migration.sql
@@ -0,0 +1,13 @@
+/*
+ Warnings:
+
+ - You are about to drop the column `changePct` on the `project_insights` table. All the data in the column will be lost.
+ - You are about to drop the column `compareValue` on the `project_insights` table. All the data in the column will be lost.
+ - You are about to drop the column `currentValue` on the `project_insights` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "public"."project_insights" DROP COLUMN "changePct",
+DROP COLUMN "compareValue",
+DROP COLUMN "currentValue",
+ADD COLUMN "displayName" TEXT NOT NULL DEFAULT '';
diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma
index 030cc84c..3b4d5dfe 100644
--- a/packages/db/prisma/schema.prisma
+++ b/packages/db/prisma/schema.prisma
@@ -512,13 +512,12 @@ model ProjectInsight {
windowKind String // "yesterday" | "rolling_7d" | "rolling_30d"
state InsightState @default(active)
- title String
- summary String?
- payload Json? // RenderedCard blocks, extra data
+ title String
+ summary String?
+ displayName String @default("")
+ /// [IPrismaProjectInsightPayload]
+ payload Json @default("{}") // Rendered insight payload (typed)
- currentValue Float?
- compareValue Float?
- changePct Float?
direction String? // "up" | "down" | "flat"
impactScore Float @default(0)
severityBand String? // "low" | "moderate" | "severe"
diff --git a/packages/db/src/clickhouse/query-builder.ts b/packages/db/src/clickhouse/query-builder.ts
index ba2416c7..066b7b7c 100644
--- a/packages/db/src/clickhouse/query-builder.ts
+++ b/packages/db/src/clickhouse/query-builder.ts
@@ -43,7 +43,7 @@ class Expression {
}
export class Query {
- private _select: string[] = [];
+ private _select: (string | Expression)[] = [];
private _except: string[] = [];
private _from?: string | Expression;
private _where: WhereCondition[] = [];
@@ -81,17 +81,19 @@ export class Query {
// Select methods
select(
- columns: (string | null | undefined | false)[],
+ columns: (string | Expression | null | undefined | false)[],
type: 'merge' | 'replace' = 'replace',
): Query {
if (this._skipNext) return this as unknown as Query;
if (type === 'merge') {
this._select = [
...this._select,
- ...columns.filter((col): col is string => Boolean(col)),
+ ...columns.filter((col): col is string | Expression => Boolean(col)),
];
} else {
- this._select = columns.filter((col): col is string => Boolean(col));
+ this._select = columns.filter((col): col is string | Expression =>
+ Boolean(col),
+ );
}
return this as unknown as Query;
}
@@ -372,7 +374,14 @@ export class Query {
if (this._select.length > 0) {
parts.push(
'SELECT',
- this._select.map((col) => this.escapeDate(col)).join(', '),
+ this._select
+ // Important: Expressions are treated as raw SQL; do not run escapeDate()
+ // on them, otherwise any embedded date strings get double-escaped
+ // (e.g. ''2025-12-16 23:59:59'') which ClickHouse rejects.
+ .map((col) =>
+ col instanceof Expression ? col.toString() : this.escapeDate(col),
+ )
+ .join(', '),
);
} else {
parts.push('SELECT *');
diff --git a/packages/db/src/services/insights/cached-clix.ts b/packages/db/src/services/insights/cached-clix.ts
new file mode 100644
index 00000000..2dd772d6
--- /dev/null
+++ b/packages/db/src/services/insights/cached-clix.ts
@@ -0,0 +1,68 @@
+import crypto from 'node:crypto';
+import type { ClickHouseClient } from '@clickhouse/client';
+import {
+ type Query,
+ clix as originalClix,
+} from '../../clickhouse/query-builder';
+
+/**
+ * Creates a cached wrapper around clix that automatically caches query results
+ * based on query hash. This eliminates duplicate queries within the same module/window context.
+ *
+ * @param client - ClickHouse client
+ * @param cache - Optional cache Map to store query results
+ * @param timezone - Timezone for queries (defaults to UTC)
+ * @returns A function that creates cached Query instances (compatible with clix API)
+ */
+export function createCachedClix(
+ client: ClickHouseClient,
+ cache?: Map,
+ timezone?: string,
+) {
+ function clixCached(): Query {
+ const query = originalClix(client, timezone);
+ const queryTimezone = timezone ?? 'UTC';
+
+ // Override execute() method to add caching
+ const originalExecute = query.execute.bind(query);
+ query.execute = async () => {
+ // Build the query SQL string
+ const querySQL = query.toSQL();
+
+ // Create cache key from query SQL + timezone
+ const cacheKey = crypto
+ .createHash('sha256')
+ .update(`${querySQL}|${queryTimezone}`)
+ .digest('hex');
+
+ // Check cache first
+ if (cache?.has(cacheKey)) {
+ return cache.get(cacheKey);
+ }
+
+ // Execute query
+ const result = await originalExecute();
+
+ // Cache the result
+ if (cache) {
+ cache.set(cacheKey, result);
+ }
+
+ return result;
+ };
+
+ return query;
+ }
+
+ // Copy static methods from original clix
+ clixCached.exp = originalClix.exp;
+ clixCached.date = originalClix.date;
+ clixCached.datetime = originalClix.datetime;
+ clixCached.dynamicDatetime = originalClix.dynamicDatetime;
+ clixCached.toStartOf = originalClix.toStartOf;
+ clixCached.toStartOfInterval = originalClix.toStartOfInterval;
+ clixCached.toInterval = originalClix.toInterval;
+ clixCached.toDate = originalClix.toDate;
+
+ return clixCached;
+}
diff --git a/packages/db/src/services/insights/engine.ts b/packages/db/src/services/insights/engine.ts
index 7c0944ba..086cb8f5 100644
--- a/packages/db/src/services/insights/engine.ts
+++ b/packages/db/src/services/insights/engine.ts
@@ -1,17 +1,22 @@
-import crypto from 'node:crypto';
+import { createCachedClix } from './cached-clix';
import { materialDecision } from './material';
import { defaultImpactScore, severityBand } from './scoring';
import type {
Cadence,
ComputeContext,
ComputeResult,
- ExplainQueue,
InsightModule,
InsightStore,
WindowKind,
} from './types';
import { resolveWindow } from './windows';
+const DEFAULT_WINDOWS: WindowKind[] = [
+ 'yesterday',
+ 'rolling_7d',
+ 'rolling_30d',
+];
+
export interface EngineConfig {
keepTopNPerModuleWindow: number; // e.g. 5
closeStaleAfterDays: number; // e.g. 7
@@ -21,8 +26,6 @@ export interface EngineConfig {
minAbsDelta: number; // e.g. 80
minPct: number; // e.g. 0.15
};
- enableExplain: boolean;
- explainTopNPerProjectPerDay: number; // e.g. 3
}
/** Simple gating to cut noise; modules can override via thresholds. */
@@ -53,64 +56,84 @@ function chunk(arr: T[], size: number): T[][] {
return out;
}
-function sha256(x: string) {
- return crypto.createHash('sha256').update(x).digest('hex');
-}
-
-/**
- * Engine entrypoint: runs all projects for a cadence.
- * Recommended: call this from a per-project worker (fanout), but it can also run directly.
- */
export function createEngine(args: {
store: InsightStore;
modules: InsightModule[];
db: any;
logger?: Pick;
- explainQueue?: ExplainQueue;
config: EngineConfig;
}) {
- const { store, modules, db, explainQueue, config } = args;
+ const { store, modules, db, config } = args;
const logger = args.logger ?? console;
- async function runCadence(cadence: Cadence, now: Date): Promise {
- const projectIds = await store.listProjectIdsForCadence(cadence);
- for (const projectId of projectIds) {
- await runProject({ projectId, cadence, now });
- }
+ function isProjectOldEnoughForWindow(
+ projectCreatedAt: Date | null | undefined,
+ baselineStart: Date,
+ ): boolean {
+ if (!projectCreatedAt) return true; // best-effort; don't block if unknown
+ return projectCreatedAt.getTime() <= baselineStart.getTime();
}
async function runProject(opts: {
projectId: string;
cadence: Cadence;
now: Date;
+ projectCreatedAt?: Date | null;
}): Promise {
- const { projectId, cadence, now } = opts;
+ const { projectId, cadence, now, projectCreatedAt } = opts;
const projLogger = logger;
const eligible = modules.filter((m) => m.cadence.includes(cadence));
- // Track top insights (by impact) for optional explain step across all modules/windows
- const explainCandidates: Array<{
- insightId: string;
- impact: number;
- evidence: any;
- evidenceHash: string;
- }> = [];
-
for (const mod of eligible) {
- for (const windowKind of mod.windows) {
- const window = resolveWindow(windowKind as WindowKind, now);
- const ctx: ComputeContext = {
- projectId,
- window,
- db,
- now,
- logger: projLogger,
- };
+ const windows = mod.windows ?? DEFAULT_WINDOWS;
+ for (const windowKind of windows) {
+ let window: ReturnType;
+ let ctx: ComputeContext;
+ try {
+ window = resolveWindow(windowKind, now);
+ if (
+ !isProjectOldEnoughForWindow(projectCreatedAt, window.baselineStart)
+ ) {
+ continue;
+ }
+ // Initialize cache for this module+window combination.
+ // Cache is automatically garbage collected when context goes out of scope.
+ const cache = new Map();
+ ctx = {
+ projectId,
+ window,
+ db,
+ now,
+ logger: projLogger,
+ clix: createCachedClix(db, cache),
+ };
+ } catch (e) {
+ projLogger.error('[insights] failed to create compute context', {
+ projectId,
+ module: mod.key,
+ windowKind,
+ err: e,
+ });
+ continue;
+ }
// 1) enumerate dimensions
- let dims = mod.enumerateDimensions
- ? await mod.enumerateDimensions(ctx)
- : [];
+ let dims: string[] = [];
+ try {
+ dims = mod.enumerateDimensions
+ ? await mod.enumerateDimensions(ctx)
+ : [];
+ } catch (e) {
+ // Important: enumeration failures should not abort the whole project run.
+ // Also avoid lifecycle close/suppression when we didn't actually evaluate dims.
+ projLogger.error('[insights] module enumerateDimensions failed', {
+ projectId,
+ module: mod.key,
+ windowKind,
+ err: e,
+ });
+ continue;
+ }
const maxDims = mod.thresholds?.maxDims ?? 25;
if (dims.length > maxDims) dims = dims.slice(0, maxDims);
@@ -190,9 +213,6 @@ export function createEngine(args: {
window,
card,
metrics: {
- currentValue: r.currentValue,
- compareValue: r.compareValue,
- changePct: r.changePct,
direction: r.direction,
impactScore: impact,
severityBand: sev,
@@ -241,7 +261,6 @@ export function createEngine(args: {
windowKind,
eventKind,
changeFrom: {
- changePct: prev.changePct,
direction: prev.direction,
impactScore: prev.impactScore,
severityBand: prev.severityBand,
@@ -255,48 +274,6 @@ export function createEngine(args: {
now,
});
}
-
- // 9) optional AI explain candidates (only for top-impact insights)
- if (config.enableExplain && explainQueue && mod.drivers) {
- // compute evidence deterministically (drivers)
- try {
- const drivers = await mod.drivers(r, ctx);
- const evidence = {
- insight: {
- moduleKey: mod.key,
- dimensionKey: r.dimensionKey,
- windowKind,
- currentValue: r.currentValue,
- compareValue: r.compareValue,
- changePct: r.changePct,
- direction: r.direction,
- },
- drivers,
- window: {
- start: window.start.toISOString().slice(0, 10),
- end: window.end.toISOString().slice(0, 10),
- baselineStart: window.baselineStart
- .toISOString()
- .slice(0, 10),
- baselineEnd: window.baselineEnd.toISOString().slice(0, 10),
- },
- };
- const evidenceHash = sha256(JSON.stringify(evidence));
- explainCandidates.push({
- insightId: persisted.id,
- impact,
- evidence,
- evidenceHash,
- });
- } catch (e) {
- projLogger.warn('[insights] drivers() failed', {
- projectId,
- module: mod.key,
- dimensionKey: r.dimensionKey,
- err: e,
- });
- }
- }
}
}
@@ -320,27 +297,7 @@ export function createEngine(args: {
});
}
}
-
- // 12) enqueue explains for top insights across the whole project run
- if (config.enableExplain && explainQueue) {
- explainCandidates.sort((a, b) => b.impact - a.impact);
- const top = explainCandidates.slice(
- 0,
- config.explainTopNPerProjectPerDay,
- );
- for (const c of top) {
- await explainQueue.enqueueExplain({
- insightId: c.insightId,
- projectId,
- moduleKey: 'n/a', // optional; you can include it in evidence instead
- dimensionKey: 'n/a',
- windowKind: 'yesterday',
- evidence: c.evidence,
- evidenceHash: c.evidenceHash,
- });
- }
- }
}
- return { runCadence, runProject };
+ return { runProject };
}
diff --git a/packages/db/src/services/insights/index.ts b/packages/db/src/services/insights/index.ts
index 5e0740c1..2d606504 100644
--- a/packages/db/src/services/insights/index.ts
+++ b/packages/db/src/services/insights/index.ts
@@ -4,6 +4,5 @@ export * from './scoring';
export * from './material';
export * from './engine';
export * from './store';
-export * from './normalize';
export * from './utils';
export * from './modules';
diff --git a/packages/db/src/services/insights/modules/devices.module.ts b/packages/db/src/services/insights/modules/devices.module.ts
index 1a3601fb..9afea524 100644
--- a/packages/db/src/services/insights/modules/devices.module.ts
+++ b/packages/db/src/services/insights/modules/devices.module.ts
@@ -1,77 +1,42 @@
-import { TABLE_NAMES } from '../../../clickhouse/client';
-import { clix } from '../../../clickhouse/query-builder';
-import type { ComputeResult, InsightModule, RenderedCard } from '../types';
-import { computeWeekdayMedians, getEndOfDay, getWeekday } from '../utils';
+import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
+import type {
+ ComputeContext,
+ ComputeResult,
+ InsightModule,
+ RenderedCard,
+} from '../types';
+import {
+ buildLookupMap,
+ computeChangePct,
+ computeDirection,
+ computeMedian,
+ getEndOfDay,
+ getWeekday,
+ selectTopDimensions,
+} from '../utils';
-function normalizeDevice(device: string): string {
- const d = (device || '').toLowerCase().trim();
- if (d.includes('mobile') || d === 'phone') return 'mobile';
- if (d.includes('tablet')) return 'tablet';
- if (d.includes('desktop')) return 'desktop';
- return d || 'unknown';
-}
-
-export const devicesModule: InsightModule = {
- key: 'devices',
- cadence: ['daily'],
- windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
- thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 5 },
-
- async enumerateDimensions(ctx) {
- // Query devices from current window (limited set, no need for baseline merge)
- const results = await clix(ctx.db)
- .select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
- .from(TABLE_NAMES.sessions)
- .where('project_id', '=', ctx.projectId)
- .where('sign', '=', 1)
- .where('created_at', 'BETWEEN', [
- ctx.window.start,
- getEndOfDay(ctx.window.end),
- ])
- .where('device', '!=', '')
- .groupBy(['device'])
- .orderBy('cnt', 'DESC')
- .execute();
-
- // Normalize and dedupe device types
- const dims = new Set();
- for (const r of results) {
- dims.add(`device:${normalizeDevice(r.device)}`);
- }
-
- return Array.from(dims);
- },
-
- async computeMany(ctx, dimensionKeys): Promise {
- // Single query for ALL current values
- const currentResults = await clix(ctx.db)
- .select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
- .from(TABLE_NAMES.sessions)
- .where('project_id', '=', ctx.projectId)
- .where('sign', '=', 1)
- .where('created_at', 'BETWEEN', [
- ctx.window.start,
- getEndOfDay(ctx.window.end),
- ])
- .groupBy(['device'])
- .execute();
-
- // Build current lookup map (normalized) and total
- const currentMap = new Map();
- let totalCurrentValue = 0;
- for (const r of currentResults) {
- const key = normalizeDevice(r.device);
- const cnt = Number(r.cnt ?? 0);
- currentMap.set(key, (currentMap.get(key) ?? 0) + cnt);
- totalCurrentValue += cnt;
- }
-
- // Single query for baseline
- let baselineMap: Map;
- let totalBaselineValue = 0;
-
- if (ctx.window.kind === 'yesterday') {
- const baselineResults = await clix(ctx.db)
+async function fetchDeviceAggregates(ctx: ComputeContext): Promise<{
+ currentMap: Map;
+ baselineMap: Map;
+ totalCurrent: number;
+ totalBaseline: number;
+}> {
+ if (ctx.window.kind === 'yesterday') {
+ const [currentResults, baselineResults, totals] = await Promise.all([
+ ctx
+ .clix()
+ .select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.start,
+ getEndOfDay(ctx.window.end),
+ ])
+ .groupBy(['device'])
+ .execute(),
+ ctx
+ .clix()
.select<{ date: string; device: string; cnt: number }>([
'toDate(created_at) as date',
'device',
@@ -85,77 +50,144 @@ export const devicesModule: InsightModule = {
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['date', 'device'])
- .execute();
-
- const targetWeekday = getWeekday(ctx.window.start);
-
- // Group by normalized device type before computing medians
- const normalizedResults = baselineResults.map((r) => ({
- date: r.date,
- device: normalizeDevice(r.device),
- cnt: r.cnt,
- }));
-
- // Aggregate by date + normalized device first
- const aggregated = new Map();
- for (const r of normalizedResults) {
- const key = `${r.date}|${r.device}`;
- if (!aggregated.has(r.device)) {
- aggregated.set(r.device, []);
- }
- // Find existing entry for this date+device or add new
- const entries = aggregated.get(r.device)!;
- const existing = entries.find((e) => e.date === r.date);
- if (existing) {
- existing.cnt += Number(r.cnt ?? 0);
- } else {
- entries.push({ date: r.date, cnt: Number(r.cnt ?? 0) });
- }
- }
-
- // Compute weekday medians per device type
- baselineMap = new Map();
- for (const [deviceType, entries] of aggregated) {
- const sameWeekdayValues = entries
- .filter((e) => getWeekday(new Date(e.date)) === targetWeekday)
- .map((e) => e.cnt)
- .sort((a, b) => a - b);
-
- if (sameWeekdayValues.length > 0) {
- const mid = Math.floor(sameWeekdayValues.length / 2);
- const median =
- sameWeekdayValues.length % 2 === 0
- ? ((sameWeekdayValues[mid - 1] ?? 0) +
- (sameWeekdayValues[mid] ?? 0)) /
- 2
- : (sameWeekdayValues[mid] ?? 0);
- baselineMap.set(deviceType, median);
- totalBaselineValue += median;
- }
- }
- } else {
- const baselineResults = await clix(ctx.db)
- .select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
+ ),
+ ])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
+ getEndOfDay(ctx.window.end),
])
- .groupBy(['device'])
- .execute();
+ .execute(),
+ ]);
- baselineMap = new Map();
- for (const r of baselineResults) {
- const key = normalizeDevice(r.device);
- const cnt = Number(r.cnt ?? 0);
- baselineMap.set(key, (baselineMap.get(key) ?? 0) + cnt);
- totalBaselineValue += cnt;
+ const currentMap = buildLookupMap(currentResults, (r) => r.device);
+
+ const targetWeekday = getWeekday(ctx.window.start);
+ const aggregated = new Map();
+ for (const r of baselineResults) {
+ if (!aggregated.has(r.device)) {
+ aggregated.set(r.device, []);
+ }
+ const entries = aggregated.get(r.device)!;
+ const existing = entries.find((e) => e.date === r.date);
+ if (existing) {
+ existing.cnt += Number(r.cnt ?? 0);
+ } else {
+ entries.push({ date: r.date, cnt: Number(r.cnt ?? 0) });
}
}
- // Build results from maps
+ const baselineMap = new Map();
+ for (const [deviceType, entries] of aggregated) {
+ const sameWeekdayValues = entries
+ .filter((e) => getWeekday(new Date(e.date)) === targetWeekday)
+ .map((e) => e.cnt)
+ .sort((a, b) => a - b);
+
+ if (sameWeekdayValues.length > 0) {
+ baselineMap.set(deviceType, computeMedian(sameWeekdayValues));
+ }
+ }
+
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline =
+ baselineMap.size > 0
+ ? Array.from(baselineMap.values()).reduce((sum, val) => sum + val, 0)
+ : 0;
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+ }
+
+ const curStart = formatClickhouseDate(ctx.window.start);
+ const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
+ const baseStart = formatClickhouseDate(ctx.window.baselineStart);
+ const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
+
+ const [results, totals] = await Promise.all([
+ ctx
+ .clix()
+ .select<{ device: string; cur: number; base: number }>([
+ 'device',
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
+ ),
+ ])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .groupBy(['device'])
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number; base_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
+ ),
+ ])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .execute(),
+ ]);
+
+ const currentMap = buildLookupMap(
+ results,
+ (r) => r.device,
+ (r) => Number(r.cur ?? 0),
+ );
+
+ const baselineMap = buildLookupMap(
+ results,
+ (r) => r.device,
+ (r) => Number(r.base ?? 0),
+ );
+
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = totals[0]?.base_total ?? 0;
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+}
+
+export const devicesModule: InsightModule = {
+ key: 'devices',
+ cadence: ['daily'],
+ thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 5 },
+
+ async enumerateDimensions(ctx) {
+ const { currentMap, baselineMap } = await fetchDeviceAggregates(ctx);
+ const topDims = selectTopDimensions(
+ currentMap,
+ baselineMap,
+ this.thresholds?.maxDims ?? 5,
+ );
+ return topDims.map((dim) => `device:${dim}`);
+ },
+
+ async computeMany(ctx, dimensionKeys): Promise {
+ const { currentMap, baselineMap, totalCurrent, totalBaseline } =
+ await fetchDeviceAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
@@ -165,23 +197,12 @@ export const devicesModule: InsightModule = {
const currentValue = currentMap.get(deviceType) ?? 0;
const compareValue = baselineMap.get(deviceType) ?? 0;
- const currentShare =
- totalCurrentValue > 0 ? currentValue / totalCurrentValue : 0;
- const compareShare =
- totalBaselineValue > 0 ? compareValue / totalBaselineValue : 0;
+ const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
+ const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
- // Share shift in percentage points
const shareShiftPp = (currentShare - compareShare) * 100;
- const changePct =
- compareShare > 0
- ? (currentShare - compareShare) / compareShare
- : currentShare > 0
- ? 1
- : 0;
-
- // Direction should match the sign of the pp shift (so title + delta agree)
- const direction: 'up' | 'down' | 'flat' =
- shareShiftPp > 0 ? 'up' : shareShiftPp < 0 ? 'down' : 'flat';
+ const changePct = computeChangePct(currentValue, compareValue);
+ const direction = computeDirection(changePct);
results.push({
ok: true,
@@ -203,20 +224,51 @@ export const devicesModule: InsightModule = {
render(result, ctx): RenderedCard {
const device = result.dimensionKey.replace('device:', '');
- const shareShiftPp = (result.extra?.shareShiftPp as number) ?? 0;
- const isIncrease = shareShiftPp >= 0;
+ const changePct = result.changePct ?? 0;
+ const isIncrease = changePct >= 0;
+
+ const sessionsCurrent = result.currentValue ?? 0;
+ const sessionsCompare = result.compareValue ?? 0;
+ const shareCurrent = Number(result.extra?.currentShare ?? 0);
+ const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
- kind: 'insight_v1',
- title: `${device} ${isIncrease ? '↑' : '↓'} ${Math.abs(shareShiftPp).toFixed(1)}pp`,
- summary: `${ctx.window.label}. Device share shift.`,
- primaryDimension: { type: 'device', key: device, displayName: device },
- tags: ['devices', ctx.window.kind, isIncrease ? 'increase' : 'decrease'],
- metric: 'share',
- extra: {
- currentShare: result.extra?.currentShare,
- compareShare: result.extra?.compareShare,
- shareShiftPp: result.extra?.shareShiftPp,
+ title: `${device} ${isIncrease ? '↑' : '↓'} ${Math.abs(changePct * 100).toFixed(0)}%`,
+ summary: `${ctx.window.label}. Device traffic change.`,
+ displayName: device,
+ payload: {
+ kind: 'insight_v1',
+ dimensions: [{ key: 'device', value: device, displayName: device }],
+ primaryMetric: 'sessions',
+ metrics: {
+ sessions: {
+ current: sessionsCurrent,
+ compare: sessionsCompare,
+ delta: sessionsCurrent - sessionsCompare,
+ changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
+ direction: result.direction ?? 'flat',
+ unit: 'count',
+ },
+ share: {
+ current: shareCurrent,
+ compare: shareCompare,
+ delta: shareCurrent - shareCompare,
+ changePct:
+ shareCompare > 0
+ ? (shareCurrent - shareCompare) / shareCompare
+ : null,
+ direction:
+ shareCurrent - shareCompare > 0.0005
+ ? 'up'
+ : shareCurrent - shareCompare < -0.0005
+ ? 'down'
+ : 'flat',
+ unit: 'ratio',
+ },
+ },
+ extra: {
+ // keep module-specific flags/fields if needed later
+ },
},
};
},
diff --git a/packages/db/src/services/insights/modules/entry-pages.module.ts b/packages/db/src/services/insights/modules/entry-pages.module.ts
index 76742f76..ea92df93 100644
--- a/packages/db/src/services/insights/modules/entry-pages.module.ts
+++ b/packages/db/src/services/insights/modules/entry-pages.module.ts
@@ -1,26 +1,34 @@
-import { TABLE_NAMES } from '../../../clickhouse/client';
-import { clix } from '../../../clickhouse/query-builder';
-import { normalizePath } from '../normalize';
-import type { ComputeResult, InsightModule, RenderedCard } from '../types';
+import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
+import type {
+ ComputeContext,
+ ComputeResult,
+ InsightModule,
+ RenderedCard,
+} from '../types';
import {
+ buildLookupMap,
computeChangePct,
computeDirection,
computeWeekdayMedians,
getEndOfDay,
getWeekday,
+ selectTopDimensions,
} from '../utils';
-export const entryPagesModule: InsightModule = {
- key: 'entry-pages',
- cadence: ['daily'],
- windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
- thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
+const DELIMITER = '|||';
- async enumerateDimensions(ctx) {
- // Query top entry pages from BOTH current and baseline windows
- const [currentResults, baselineResults] = await Promise.all([
- clix(ctx.db)
- .select<{ entry_path: string; cnt: number }>([
+async function fetchEntryPageAggregates(ctx: ComputeContext): Promise<{
+ currentMap: Map;
+ baselineMap: Map;
+ totalCurrent: number;
+ totalBaseline: number;
+}> {
+ if (ctx.window.kind === 'yesterday') {
+ const [currentResults, baselineResults, totals] = await Promise.all([
+ ctx
+ .clix()
+ .select<{ entry_origin: string; entry_path: string; cnt: number }>([
+ 'entry_origin',
'entry_path',
'count(*) as cnt',
])
@@ -31,12 +39,18 @@ export const entryPagesModule: InsightModule = {
ctx.window.start,
getEndOfDay(ctx.window.end),
])
- .groupBy(['entry_path'])
- .orderBy('cnt', 'DESC')
- .limit(this.thresholds?.maxDims ?? 100)
+ .groupBy(['entry_origin', 'entry_path'])
.execute(),
- clix(ctx.db)
- .select<{ entry_path: string; cnt: number }>([
+ ctx
+ .clix()
+ .select<{
+ date: string;
+ entry_origin: string;
+ entry_path: string;
+ cnt: number;
+ }>([
+ 'toDate(created_at) as date',
+ 'entry_origin',
'entry_path',
'count(*) as cnt',
])
@@ -47,104 +61,147 @@ export const entryPagesModule: InsightModule = {
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
- .groupBy(['entry_path'])
- .orderBy('cnt', 'DESC')
- .limit(this.thresholds?.maxDims ?? 100)
+ .groupBy(['date', 'entry_origin', 'entry_path'])
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
+ ),
+ ])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
.execute(),
]);
- // Merge both sets
- const dims = new Set();
- for (const r of currentResults) {
- dims.add(`entry:${normalizePath(r.entry_path || '/')}`);
- }
- for (const r of baselineResults) {
- dims.add(`entry:${normalizePath(r.entry_path || '/')}`);
- }
+ const currentMap = buildLookupMap(
+ currentResults,
+ (r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
+ );
- return Array.from(dims);
- },
+ const targetWeekday = getWeekday(ctx.window.start);
+ const baselineMap = computeWeekdayMedians(
+ baselineResults,
+ targetWeekday,
+ (r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
+ );
- async computeMany(ctx, dimensionKeys): Promise {
- // Single query for ALL current values
- const currentResults = await clix(ctx.db)
- .select<{ entry_path: string; cnt: number }>([
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = Array.from(baselineMap.values()).reduce(
+ (sum, val) => sum + val,
+ 0,
+ );
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+ }
+
+ const curStart = formatClickhouseDate(ctx.window.start);
+ const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
+ const baseStart = formatClickhouseDate(ctx.window.baselineStart);
+ const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
+
+ const [results, totals] = await Promise.all([
+ ctx
+ .clix()
+ .select<{
+ entry_origin: string;
+ entry_path: string;
+ cur: number;
+ base: number;
+ }>([
+ 'entry_origin',
'entry_path',
- 'count(*) as cnt',
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
+ ),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
- ctx.window.start,
+ ctx.window.baselineStart,
getEndOfDay(ctx.window.end),
])
- .groupBy(['entry_path'])
- .execute();
+ .groupBy(['entry_origin', 'entry_path'])
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number; base_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
+ ),
+ ])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .execute(),
+ ]);
- // Build current lookup map
- const currentMap = new Map();
- for (const r of currentResults) {
- const key = normalizePath(r.entry_path || '/');
- currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
- }
+ const currentMap = buildLookupMap(
+ results,
+ (r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
+ (r) => Number(r.cur ?? 0),
+ );
- // Single query for baseline
- let baselineMap: Map;
+ const baselineMap = buildLookupMap(
+ results,
+ (r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
+ (r) => Number(r.base ?? 0),
+ );
- if (ctx.window.kind === 'yesterday') {
- const baselineResults = await clix(ctx.db)
- .select<{ date: string; entry_path: string; cnt: number }>([
- 'toDate(created_at) as date',
- 'entry_path',
- 'count(*) as cnt',
- ])
- .from(TABLE_NAMES.sessions)
- .where('project_id', '=', ctx.projectId)
- .where('sign', '=', 1)
- .where('created_at', 'BETWEEN', [
- ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
- ])
- .groupBy(['date', 'entry_path'])
- .execute();
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = totals[0]?.base_total ?? 0;
- const targetWeekday = getWeekday(ctx.window.start);
- baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
- normalizePath(r.entry_path || '/'),
- );
- } else {
- const baselineResults = await clix(ctx.db)
- .select<{ entry_path: string; cnt: number }>([
- 'entry_path',
- 'count(*) as cnt',
- ])
- .from(TABLE_NAMES.sessions)
- .where('project_id', '=', ctx.projectId)
- .where('sign', '=', 1)
- .where('created_at', 'BETWEEN', [
- ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
- ])
- .groupBy(['entry_path'])
- .execute();
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+}
- baselineMap = new Map();
- for (const r of baselineResults) {
- const key = normalizePath(r.entry_path || '/');
- baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
- }
- }
+export const entryPagesModule: InsightModule = {
+ key: 'entry-pages',
+ cadence: ['daily'],
+ thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
- // Build results from maps
+ async enumerateDimensions(ctx) {
+ const { currentMap, baselineMap } = await fetchEntryPageAggregates(ctx);
+ const topDims = selectTopDimensions(
+ currentMap,
+ baselineMap,
+ this.thresholds?.maxDims ?? 100,
+ );
+ return topDims.map((dim) => `entry:${dim}`);
+ },
+
+ async computeMany(ctx, dimensionKeys): Promise {
+ const { currentMap, baselineMap, totalCurrent, totalBaseline } =
+ await fetchEntryPageAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
if (!dimKey.startsWith('entry:')) continue;
- const entryPath = dimKey.replace('entry:', '');
+ const originPath = dimKey.replace('entry:', '');
- const currentValue = currentMap.get(entryPath) ?? 0;
- const compareValue = baselineMap.get(entryPath) ?? 0;
+ const currentValue = currentMap.get(originPath) ?? 0;
+ const compareValue = baselineMap.get(originPath) ?? 0;
+
+ const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
+ const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
+
+ const shareShiftPp = (currentShare - compareShare) * 100;
const changePct = computeChangePct(currentValue, compareValue);
const direction = computeDirection(changePct);
@@ -156,6 +213,9 @@ export const entryPagesModule: InsightModule = {
changePct,
direction,
extra: {
+ shareShiftPp,
+ currentShare,
+ compareShare,
isNew: compareValue === 0 && currentValue > 0,
},
});
@@ -165,28 +225,62 @@ export const entryPagesModule: InsightModule = {
},
render(result, ctx): RenderedCard {
- const path = result.dimensionKey.replace('entry:', '');
+ const originPath = result.dimensionKey.replace('entry:', '');
+ const [origin, path] = originPath.split(DELIMITER);
+ const displayValue = origin ? `${origin}${path}` : path || '/';
const pct = ((result.changePct ?? 0) * 100).toFixed(1);
const isIncrease = (result.changePct ?? 0) >= 0;
const isNew = result.extra?.isNew as boolean | undefined;
const title = isNew
- ? `New entry page: ${path}`
- : `Entry page ${path} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
+ ? `New entry page: ${displayValue}`
+ : `Entry page ${displayValue} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
+
+ const sessionsCurrent = result.currentValue ?? 0;
+ const sessionsCompare = result.compareValue ?? 0;
+ const shareCurrent = Number(result.extra?.currentShare ?? 0);
+ const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
- kind: 'insight_v1',
title,
- summary: `${ctx.window.label}. Sessions ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
- primaryDimension: { type: 'entry', key: path, displayName: path },
- tags: [
- 'entry-pages',
- ctx.window.kind,
- isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
- ],
- metric: 'sessions',
- extra: {
- isNew: result.extra?.isNew,
+ summary: `${ctx.window.label}. Sessions ${sessionsCurrent} vs ${sessionsCompare}.`,
+ displayName: displayValue,
+ payload: {
+ kind: 'insight_v1',
+ dimensions: [
+ { key: 'origin', value: origin ?? '', displayName: origin ?? '' },
+ { key: 'path', value: path ?? '', displayName: path ?? '' },
+ ],
+ primaryMetric: 'sessions',
+ metrics: {
+ sessions: {
+ current: sessionsCurrent,
+ compare: sessionsCompare,
+ delta: sessionsCurrent - sessionsCompare,
+ changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
+ direction: result.direction ?? 'flat',
+ unit: 'count',
+ },
+ share: {
+ current: shareCurrent,
+ compare: shareCompare,
+ delta: shareCurrent - shareCompare,
+ changePct:
+ shareCompare > 0
+ ? (shareCurrent - shareCompare) / shareCompare
+ : null,
+ direction:
+ shareCurrent - shareCompare > 0.0005
+ ? 'up'
+ : shareCurrent - shareCompare < -0.0005
+ ? 'down'
+ : 'flat',
+ unit: 'ratio',
+ },
+ },
+ extra: {
+ isNew: result.extra?.isNew,
+ },
},
};
},
diff --git a/packages/db/src/services/insights/modules/geo.module.ts b/packages/db/src/services/insights/modules/geo.module.ts
index d0ba3664..e848507f 100644
--- a/packages/db/src/services/insights/modules/geo.module.ts
+++ b/packages/db/src/services/insights/modules/geo.module.ts
@@ -1,18 +1,31 @@
-import { TABLE_NAMES } from '../../../clickhouse/client';
-import { clix } from '../../../clickhouse/query-builder';
-import type { ComputeResult, InsightModule, RenderedCard } from '../types';
-import { computeWeekdayMedians, getEndOfDay, getWeekday } from '../utils';
+import { getCountry } from '@openpanel/constants';
+import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
+import type {
+ ComputeContext,
+ ComputeResult,
+ InsightModule,
+ RenderedCard,
+} from '../types';
+import {
+ buildLookupMap,
+ computeChangePct,
+ computeDirection,
+ computeWeekdayMedians,
+ getEndOfDay,
+ getWeekday,
+ selectTopDimensions,
+} from '../utils';
-export const geoModule: InsightModule = {
- key: 'geo',
- cadence: ['daily'],
- windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
- thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 30 },
-
- async enumerateDimensions(ctx) {
- // Query top countries from BOTH current and baseline windows
- const [currentResults, baselineResults] = await Promise.all([
- clix(ctx.db)
+async function fetchGeoAggregates(ctx: ComputeContext): Promise<{
+ currentMap: Map;
+ baselineMap: Map;
+ totalCurrent: number;
+ totalBaseline: number;
+}> {
+ if (ctx.window.kind === 'yesterday') {
+ const [currentResults, baselineResults, totals] = await Promise.all([
+ ctx
+ .clix()
.select<{ country: string; cnt: number }>([
'country',
'count(*) as cnt',
@@ -24,72 +37,10 @@ export const geoModule: InsightModule = {
ctx.window.start,
getEndOfDay(ctx.window.end),
])
- .where('country', '!=', '')
.groupBy(['country'])
- .orderBy('cnt', 'DESC')
- .limit(this.thresholds?.maxDims ?? 30)
.execute(),
- clix(ctx.db)
- .select<{ country: string; cnt: number }>([
- 'country',
- 'count(*) as cnt',
- ])
- .from(TABLE_NAMES.sessions)
- .where('project_id', '=', ctx.projectId)
- .where('sign', '=', 1)
- .where('created_at', 'BETWEEN', [
- ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
- ])
- .where('country', '!=', '')
- .groupBy(['country'])
- .orderBy('cnt', 'DESC')
- .limit(this.thresholds?.maxDims ?? 30)
- .execute(),
- ]);
-
- // Merge both sets
- const dims = new Set();
- for (const r of currentResults) {
- dims.add(`country:${r.country || 'unknown'}`);
- }
- for (const r of baselineResults) {
- dims.add(`country:${r.country || 'unknown'}`);
- }
-
- return Array.from(dims);
- },
-
- async computeMany(ctx, dimensionKeys): Promise {
- // Single query for ALL current values + total
- const currentResults = await clix(ctx.db)
- .select<{ country: string; cnt: number }>(['country', 'count(*) as cnt'])
- .from(TABLE_NAMES.sessions)
- .where('project_id', '=', ctx.projectId)
- .where('sign', '=', 1)
- .where('created_at', 'BETWEEN', [
- ctx.window.start,
- getEndOfDay(ctx.window.end),
- ])
- .groupBy(['country'])
- .execute();
-
- // Build current lookup map and total
- const currentMap = new Map();
- let totalCurrentValue = 0;
- for (const r of currentResults) {
- const key = r.country || 'unknown';
- const cnt = Number(r.cnt ?? 0);
- currentMap.set(key, (currentMap.get(key) ?? 0) + cnt);
- totalCurrentValue += cnt;
- }
-
- // Single query for baseline
- let baselineMap: Map;
- let totalBaselineValue = 0;
-
- if (ctx.window.kind === 'yesterday') {
- const baselineResults = await clix(ctx.db)
+ ctx
+ .clix()
.select<{ date: string; country: string; cnt: number }>([
'toDate(created_at) as date',
'country',
@@ -103,45 +54,127 @@ export const geoModule: InsightModule = {
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['date', 'country'])
- .execute();
-
- const targetWeekday = getWeekday(ctx.window.start);
- baselineMap = computeWeekdayMedians(
- baselineResults,
- targetWeekday,
- (r) => r.country || 'unknown',
- );
-
- // Compute total baseline from medians
- for (const value of baselineMap.values()) {
- totalBaselineValue += value;
- }
- } else {
- const baselineResults = await clix(ctx.db)
- .select<{ country: string; cnt: number }>([
- 'country',
- 'count(*) as cnt',
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
+ ),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
+ getEndOfDay(ctx.window.end),
])
- .groupBy(['country'])
- .execute();
+ .execute(),
+ ]);
- baselineMap = new Map();
- for (const r of baselineResults) {
- const key = r.country || 'unknown';
- const cnt = Number(r.cnt ?? 0);
- baselineMap.set(key, (baselineMap.get(key) ?? 0) + cnt);
- totalBaselineValue += cnt;
- }
- }
+ const currentMap = buildLookupMap(
+ currentResults,
+ (r) => r.country || 'unknown',
+ );
- // Build results from maps
+ const targetWeekday = getWeekday(ctx.window.start);
+ const baselineMap = computeWeekdayMedians(
+ baselineResults,
+ targetWeekday,
+ (r) => r.country || 'unknown',
+ );
+
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = Array.from(baselineMap.values()).reduce(
+ (sum, val) => sum + val,
+ 0,
+ );
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+ }
+
+ const curStart = formatClickhouseDate(ctx.window.start);
+ const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
+ const baseStart = formatClickhouseDate(ctx.window.baselineStart);
+ const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
+
+ const [results, totals] = await Promise.all([
+ ctx
+ .clix()
+ .select<{ country: string; cur: number; base: number }>([
+ 'country',
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
+ ),
+ ])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .groupBy(['country'])
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number; base_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
+ ),
+ ])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .execute(),
+ ]);
+
+ const currentMap = buildLookupMap(
+ results,
+ (r) => r.country || 'unknown',
+ (r) => Number(r.cur ?? 0),
+ );
+
+ const baselineMap = buildLookupMap(
+ results,
+ (r) => r.country || 'unknown',
+ (r) => Number(r.base ?? 0),
+ );
+
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = totals[0]?.base_total ?? 0;
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+}
+
+export const geoModule: InsightModule = {
+ key: 'geo',
+ cadence: ['daily'],
+ thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 30 },
+
+ async enumerateDimensions(ctx) {
+ const { currentMap, baselineMap } = await fetchGeoAggregates(ctx);
+ const topDims = selectTopDimensions(
+ currentMap,
+ baselineMap,
+ this.thresholds?.maxDims ?? 30,
+ );
+ return topDims.map((dim) => `country:${dim}`);
+ },
+
+ async computeMany(ctx, dimensionKeys): Promise {
+ const { currentMap, baselineMap, totalCurrent, totalBaseline } =
+ await fetchGeoAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
@@ -151,23 +184,12 @@ export const geoModule: InsightModule = {
const currentValue = currentMap.get(country) ?? 0;
const compareValue = baselineMap.get(country) ?? 0;
- const currentShare =
- totalCurrentValue > 0 ? currentValue / totalCurrentValue : 0;
- const compareShare =
- totalBaselineValue > 0 ? compareValue / totalBaselineValue : 0;
+ const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
+ const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
- // Share shift in percentage points
const shareShiftPp = (currentShare - compareShare) * 100;
- const changePct =
- compareShare > 0
- ? (currentShare - compareShare) / compareShare
- : currentShare > 0
- ? 1
- : 0;
-
- // Direction should match the sign of the pp shift (so title + delta agree)
- const direction: 'up' | 'down' | 'flat' =
- shareShiftPp > 0 ? 'up' : shareShiftPp < 0 ? 'down' : 'flat';
+ const changePct = computeChangePct(currentValue, compareValue);
+ const direction = computeDirection(changePct);
results.push({
ok: true,
@@ -190,30 +212,59 @@ export const geoModule: InsightModule = {
render(result, ctx): RenderedCard {
const country = result.dimensionKey.replace('country:', '');
- const shareShiftPp = (result.extra?.shareShiftPp as number) ?? 0;
- const isIncrease = shareShiftPp >= 0;
+ const changePct = result.changePct ?? 0;
+ const isIncrease = changePct >= 0;
const isNew = result.extra?.isNew as boolean | undefined;
+ const displayName = getCountry(country);
const title = isNew
- ? `New traffic from: ${country}`
- : `${country} ${isIncrease ? '↑' : '↓'} ${Math.abs(shareShiftPp).toFixed(1)}pp`;
+ ? `New traffic from: ${displayName}`
+ : `${displayName} ${isIncrease ? '↑' : '↓'} ${Math.abs(changePct * 100).toFixed(0)}%`;
+
+ const sessionsCurrent = result.currentValue ?? 0;
+ const sessionsCompare = result.compareValue ?? 0;
+ const shareCurrent = Number(result.extra?.currentShare ?? 0);
+ const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
- kind: 'insight_v1',
title,
- summary: `${ctx.window.label}. Share shift from ${country}.`,
- primaryDimension: { type: 'country', key: country, displayName: country },
- tags: [
- 'geo',
- ctx.window.kind,
- isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
- ],
- metric: 'share',
- extra: {
- currentShare: result.extra?.currentShare,
- compareShare: result.extra?.compareShare,
- shareShiftPp: result.extra?.shareShiftPp,
- isNew: result.extra?.isNew,
+ summary: `${ctx.window.label}. Traffic change from ${displayName}.`,
+ displayName,
+ payload: {
+ kind: 'insight_v1',
+ dimensions: [
+ { key: 'country', value: country, displayName: displayName },
+ ],
+ primaryMetric: 'sessions',
+ metrics: {
+ sessions: {
+ current: sessionsCurrent,
+ compare: sessionsCompare,
+ delta: sessionsCurrent - sessionsCompare,
+ changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
+ direction: result.direction ?? 'flat',
+ unit: 'count',
+ },
+ share: {
+ current: shareCurrent,
+ compare: shareCompare,
+ delta: shareCurrent - shareCompare,
+ changePct:
+ shareCompare > 0
+ ? (shareCurrent - shareCompare) / shareCompare
+ : null,
+ direction:
+ shareCurrent - shareCompare > 0.0005
+ ? 'up'
+ : shareCurrent - shareCompare < -0.0005
+ ? 'down'
+ : 'flat',
+ unit: 'ratio',
+ },
+ },
+ extra: {
+ isNew: result.extra?.isNew,
+ },
},
};
},
diff --git a/packages/db/src/services/insights/modules/page-trends.module.ts b/packages/db/src/services/insights/modules/page-trends.module.ts
index a9fde665..0b8853fb 100644
--- a/packages/db/src/services/insights/modules/page-trends.module.ts
+++ b/packages/db/src/services/insights/modules/page-trends.module.ts
@@ -1,26 +1,37 @@
-import { TABLE_NAMES } from '../../../clickhouse/client';
-import { clix } from '../../../clickhouse/query-builder';
-import { normalizePath } from '../normalize';
-import type { ComputeResult, InsightModule, RenderedCard } from '../types';
+import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
+import type {
+ ComputeContext,
+ ComputeResult,
+ InsightModule,
+ RenderedCard,
+} from '../types';
import {
+ buildLookupMap,
computeChangePct,
computeDirection,
computeWeekdayMedians,
getEndOfDay,
getWeekday,
+ selectTopDimensions,
} from '../utils';
-export const pageTrendsModule: InsightModule = {
- key: 'page-trends',
- cadence: ['daily'],
- windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
- thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
+const DELIMITER = '|||';
- async enumerateDimensions(ctx) {
- // Query top pages from BOTH current and baseline windows
- const [currentResults, baselineResults] = await Promise.all([
- clix(ctx.db)
- .select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
+async function fetchPageTrendAggregates(ctx: ComputeContext): Promise<{
+ currentMap: Map;
+ baselineMap: Map;
+ totalCurrent: number;
+ totalBaseline: number;
+}> {
+ if (ctx.window.kind === 'yesterday') {
+ const [currentResults, baselineResults, totals] = await Promise.all([
+ ctx
+ .clix()
+ .select<{ origin: string; path: string; cnt: number }>([
+ 'origin',
+ 'path',
+ 'count(*) as cnt',
+ ])
.from(TABLE_NAMES.events)
.where('project_id', '=', ctx.projectId)
.where('name', '=', 'screen_view')
@@ -28,65 +39,13 @@ export const pageTrendsModule: InsightModule = {
ctx.window.start,
getEndOfDay(ctx.window.end),
])
- .groupBy(['path'])
- .orderBy('cnt', 'DESC')
- .limit(this.thresholds?.maxDims ?? 100)
+ .groupBy(['origin', 'path'])
.execute(),
- clix(ctx.db)
- .select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
- .from(TABLE_NAMES.events)
- .where('project_id', '=', ctx.projectId)
- .where('name', '=', 'screen_view')
- .where('created_at', 'BETWEEN', [
- ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
- ])
- .groupBy(['path'])
- .orderBy('cnt', 'DESC')
- .limit(this.thresholds?.maxDims ?? 100)
- .execute(),
- ]);
-
- // Merge both sets
- const dims = new Set();
- for (const r of currentResults) {
- dims.add(`page:${normalizePath(r.path || '/')}`);
- }
- for (const r of baselineResults) {
- dims.add(`page:${normalizePath(r.path || '/')}`);
- }
-
- return Array.from(dims);
- },
-
- async computeMany(ctx, dimensionKeys): Promise {
- // Single query for ALL current values
- const currentResults = await clix(ctx.db)
- .select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
- .from(TABLE_NAMES.events)
- .where('project_id', '=', ctx.projectId)
- .where('name', '=', 'screen_view')
- .where('created_at', 'BETWEEN', [
- ctx.window.start,
- getEndOfDay(ctx.window.end),
- ])
- .groupBy(['path'])
- .execute();
-
- // Build current lookup map
- const currentMap = new Map();
- for (const r of currentResults) {
- const key = normalizePath(r.path || '/');
- currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
- }
-
- // Single query for baseline
- let baselineMap: Map;
-
- if (ctx.window.kind === 'yesterday') {
- const baselineResults = await clix(ctx.db)
- .select<{ date: string; path: string; cnt: number }>([
+ ctx
+ .clix()
+ .select<{ date: string; origin: string; path: string; cnt: number }>([
'toDate(created_at) as date',
+ 'origin',
'path',
'count(*) as cnt',
])
@@ -97,42 +56,142 @@ export const pageTrendsModule: InsightModule = {
ctx.window.baselineStart,
getEndOfDay(ctx.window.baselineEnd),
])
- .groupBy(['date', 'path'])
- .execute();
-
- const targetWeekday = getWeekday(ctx.window.start);
- baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
- normalizePath(r.path || '/'),
- );
- } else {
- const baselineResults = await clix(ctx.db)
- .select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
+ .groupBy(['date', 'origin', 'path'])
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
+ ),
+ ])
.from(TABLE_NAMES.events)
.where('project_id', '=', ctx.projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
+ getEndOfDay(ctx.window.end),
])
- .groupBy(['path'])
- .execute();
+ .execute(),
+ ]);
- baselineMap = new Map();
- for (const r of baselineResults) {
- const key = normalizePath(r.path || '/');
- baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
- }
- }
+ const currentMap = buildLookupMap(
+ currentResults,
+ (r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
+ );
- // Build results from maps
+ const targetWeekday = getWeekday(ctx.window.start);
+ const baselineMap = computeWeekdayMedians(
+ baselineResults,
+ targetWeekday,
+ (r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
+ );
+
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = Array.from(baselineMap.values()).reduce(
+ (sum, val) => sum + val,
+ 0,
+ );
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+ }
+
+ const curStart = formatClickhouseDate(ctx.window.start);
+ const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
+ const baseStart = formatClickhouseDate(ctx.window.baselineStart);
+ const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
+
+ const [results, totals] = await Promise.all([
+ ctx
+ .clix()
+ .select<{ origin: string; path: string; cur: number; base: number }>([
+ 'origin',
+ 'path',
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
+ ),
+ ])
+ .from(TABLE_NAMES.events)
+ .where('project_id', '=', ctx.projectId)
+ .where('name', '=', 'screen_view')
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .groupBy(['origin', 'path'])
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number; base_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
+ ),
+ ])
+ .from(TABLE_NAMES.events)
+ .where('project_id', '=', ctx.projectId)
+ .where('name', '=', 'screen_view')
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .execute(),
+ ]);
+
+ const currentMap = buildLookupMap(
+ results,
+ (r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
+ (r) => Number(r.cur ?? 0),
+ );
+
+ const baselineMap = buildLookupMap(
+ results,
+ (r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
+ (r) => Number(r.base ?? 0),
+ );
+
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = totals[0]?.base_total ?? 0;
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+}
+
+export const pageTrendsModule: InsightModule = {
+ key: 'page-trends',
+ cadence: ['daily'],
+ thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
+
+ async enumerateDimensions(ctx) {
+ const { currentMap, baselineMap } = await fetchPageTrendAggregates(ctx);
+ const topDims = selectTopDimensions(
+ currentMap,
+ baselineMap,
+ this.thresholds?.maxDims ?? 100,
+ );
+ return topDims.map((dim) => `page:${dim}`);
+ },
+
+ async computeMany(ctx, dimensionKeys): Promise {
+ const { currentMap, baselineMap, totalCurrent, totalBaseline } =
+ await fetchPageTrendAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
if (!dimKey.startsWith('page:')) continue;
- const pagePath = dimKey.replace('page:', '');
+ const originPath = dimKey.replace('page:', '');
- const currentValue = currentMap.get(pagePath) ?? 0;
- const compareValue = baselineMap.get(pagePath) ?? 0;
+ const currentValue = currentMap.get(originPath) ?? 0;
+ const compareValue = baselineMap.get(originPath) ?? 0;
+
+ const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
+ const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
+
+ const shareShiftPp = (currentShare - compareShare) * 100;
const changePct = computeChangePct(currentValue, compareValue);
const direction = computeDirection(changePct);
@@ -144,6 +203,9 @@ export const pageTrendsModule: InsightModule = {
changePct,
direction,
extra: {
+ shareShiftPp,
+ currentShare,
+ compareShare,
isNew: compareValue === 0 && currentValue > 0,
},
});
@@ -153,28 +215,62 @@ export const pageTrendsModule: InsightModule = {
},
render(result, ctx): RenderedCard {
- const path = result.dimensionKey.replace('page:', '');
+ const originPath = result.dimensionKey.replace('page:', '');
+ const [origin, path] = originPath.split(DELIMITER);
+ const displayValue = origin ? `${origin}${path}` : path || '/';
const pct = ((result.changePct ?? 0) * 100).toFixed(1);
const isIncrease = (result.changePct ?? 0) >= 0;
const isNew = result.extra?.isNew as boolean | undefined;
const title = isNew
- ? `New page getting views: ${path}`
- : `Page ${path} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
+ ? `New page getting views: ${displayValue}`
+ : `Page ${displayValue} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
+
+ const pageviewsCurrent = result.currentValue ?? 0;
+ const pageviewsCompare = result.compareValue ?? 0;
+ const shareCurrent = Number(result.extra?.currentShare ?? 0);
+ const shareCompare = Number(result.extra?.compareShare ?? 0);
return {
- kind: 'insight_v1',
title,
- summary: `${ctx.window.label}. Pageviews ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
- primaryDimension: { type: 'page', key: path, displayName: path },
- tags: [
- 'page-trends',
- ctx.window.kind,
- isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
- ],
- metric: 'pageviews',
- extra: {
- isNew: result.extra?.isNew,
+ summary: `${ctx.window.label}. Pageviews ${pageviewsCurrent} vs ${pageviewsCompare}.`,
+ displayName: displayValue,
+ payload: {
+ kind: 'insight_v1',
+ dimensions: [
+ { key: 'origin', value: origin ?? '', displayName: origin ?? '' },
+ { key: 'path', value: path ?? '', displayName: path ?? '' },
+ ],
+ primaryMetric: 'pageviews',
+ metrics: {
+ pageviews: {
+ current: pageviewsCurrent,
+ compare: pageviewsCompare,
+ delta: pageviewsCurrent - pageviewsCompare,
+ changePct: pageviewsCompare > 0 ? (result.changePct ?? 0) : null,
+ direction: result.direction ?? 'flat',
+ unit: 'count',
+ },
+ share: {
+ current: shareCurrent,
+ compare: shareCompare,
+ delta: shareCurrent - shareCompare,
+ changePct:
+ shareCompare > 0
+ ? (shareCurrent - shareCompare) / shareCompare
+ : null,
+ direction:
+ shareCurrent - shareCompare > 0.0005
+ ? 'up'
+ : shareCurrent - shareCompare < -0.0005
+ ? 'down'
+ : 'flat',
+ unit: 'ratio',
+ },
+ },
+ extra: {
+ isNew: result.extra?.isNew,
+ },
},
};
},
diff --git a/packages/db/src/services/insights/modules/referrers.module.ts b/packages/db/src/services/insights/modules/referrers.module.ts
index c22989dc..2b9d4efd 100644
--- a/packages/db/src/services/insights/modules/referrers.module.ts
+++ b/packages/db/src/services/insights/modules/referrers.module.ts
@@ -1,26 +1,30 @@
-import { TABLE_NAMES } from '../../../clickhouse/client';
-import { clix } from '../../../clickhouse/query-builder';
-import { normalizeReferrer } from '../normalize';
-import type { ComputeResult, InsightModule, RenderedCard } from '../types';
+import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
+import type {
+ ComputeContext,
+ ComputeResult,
+ InsightModule,
+ RenderedCard,
+} from '../types';
import {
+ buildLookupMap,
computeChangePct,
computeDirection,
computeWeekdayMedians,
getEndOfDay,
getWeekday,
+ selectTopDimensions,
} from '../utils';
-export const referrersModule: InsightModule = {
- key: 'referrers',
- cadence: ['daily'],
- windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
- thresholds: { minTotal: 100, minAbsDelta: 20, minPct: 0.15, maxDims: 50 },
-
- async enumerateDimensions(ctx) {
- // Query top referrers from BOTH current and baseline windows
- // This allows detecting new sources that didn't exist in baseline
- const [currentResults, baselineResults] = await Promise.all([
- clix(ctx.db)
+async function fetchReferrerAggregates(ctx: ComputeContext): Promise<{
+ currentMap: Map;
+ baselineMap: Map;
+ totalCurrent: number;
+ totalBaseline: number;
+}> {
+ if (ctx.window.kind === 'yesterday') {
+ const [currentResults, baselineResults, totals] = await Promise.all([
+ ctx
+ .clix()
.select<{ referrer_name: string; cnt: number }>([
'referrer_name',
'count(*) as cnt',
@@ -33,69 +37,9 @@ export const referrersModule: InsightModule = {
getEndOfDay(ctx.window.end),
])
.groupBy(['referrer_name'])
- .orderBy('cnt', 'DESC')
- .limit(this.thresholds?.maxDims ?? 50)
.execute(),
- clix(ctx.db)
- .select<{ referrer_name: string; cnt: number }>([
- 'referrer_name',
- 'count(*) as cnt',
- ])
- .from(TABLE_NAMES.sessions)
- .where('project_id', '=', ctx.projectId)
- .where('sign', '=', 1)
- .where('created_at', 'BETWEEN', [
- ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
- ])
- .groupBy(['referrer_name'])
- .orderBy('cnt', 'DESC')
- .limit(this.thresholds?.maxDims ?? 50)
- .execute(),
- ]);
-
- // Merge both sets to catch new/emerging sources
- const dims = new Set();
- for (const r of currentResults) {
- dims.add(`referrer:${normalizeReferrer(r.referrer_name || 'direct')}`);
- }
- for (const r of baselineResults) {
- dims.add(`referrer:${normalizeReferrer(r.referrer_name || 'direct')}`);
- }
-
- return Array.from(dims);
- },
-
- async computeMany(ctx, dimensionKeys): Promise {
- // Single query for ALL current values (batched)
- const currentResults = await clix(ctx.db)
- .select<{ referrer_name: string; cnt: number }>([
- 'referrer_name',
- 'count(*) as cnt',
- ])
- .from(TABLE_NAMES.sessions)
- .where('project_id', '=', ctx.projectId)
- .where('sign', '=', 1)
- .where('created_at', 'BETWEEN', [
- ctx.window.start,
- getEndOfDay(ctx.window.end),
- ])
- .groupBy(['referrer_name'])
- .execute();
-
- // Build current lookup map
- const currentMap = new Map();
- for (const r of currentResults) {
- const key = normalizeReferrer(r.referrer_name || 'direct');
- currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
- }
-
- // Single query for baseline (with date breakdown for weekday median if needed)
- let baselineMap: Map;
-
- if (ctx.window.kind === 'yesterday') {
- // Need daily breakdown for weekday median calculation
- const baselineResults = await clix(ctx.db)
+ ctx
+ .clix()
.select<{ date: string; referrer_name: string; cnt: number }>([
'toDate(created_at) as date',
'referrer_name',
@@ -109,37 +53,127 @@ export const referrersModule: InsightModule = {
getEndOfDay(ctx.window.baselineEnd),
])
.groupBy(['date', 'referrer_name'])
- .execute();
-
- const targetWeekday = getWeekday(ctx.window.start);
- baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
- normalizeReferrer(r.referrer_name || 'direct'),
- );
- } else {
- // Rolling windows: simple aggregate
- const baselineResults = await clix(ctx.db)
- .select<{ referrer_name: string; cnt: number }>([
- 'referrer_name',
- 'count(*) as cnt',
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
+ ),
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', ctx.projectId)
.where('sign', '=', 1)
.where('created_at', 'BETWEEN', [
ctx.window.baselineStart,
- getEndOfDay(ctx.window.baselineEnd),
+ getEndOfDay(ctx.window.end),
])
- .groupBy(['referrer_name'])
- .execute();
+ .execute(),
+ ]);
- baselineMap = new Map();
- for (const r of baselineResults) {
- const key = normalizeReferrer(r.referrer_name || 'direct');
- baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
- }
- }
+ const currentMap = buildLookupMap(
+ currentResults,
+ (r) => r.referrer_name || 'direct',
+ );
- // Build results from maps (in memory, no more queries!)
+ const targetWeekday = getWeekday(ctx.window.start);
+ const baselineMap = computeWeekdayMedians(
+ baselineResults,
+ targetWeekday,
+ (r) => r.referrer_name || 'direct',
+ );
+
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = Array.from(baselineMap.values()).reduce(
+ (sum, val) => sum + val,
+ 0,
+ );
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+ }
+
+ const curStart = formatClickhouseDate(ctx.window.start);
+ const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
+ const baseStart = formatClickhouseDate(ctx.window.baselineStart);
+ const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
+
+ const [results, totals] = await Promise.all([
+ ctx
+ .clix()
+ .select<{ referrer_name: string; cur: number; base: number }>([
+ 'referrer_name',
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
+ ),
+ ])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .groupBy(['referrer_name'])
+ .execute(),
+ ctx
+ .clix()
+ .select<{ cur_total: number; base_total: number }>([
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
+ ),
+ ctx.clix.exp(
+ `countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
+ ),
+ ])
+ .from(TABLE_NAMES.sessions)
+ .where('project_id', '=', ctx.projectId)
+ .where('sign', '=', 1)
+ .where('created_at', 'BETWEEN', [
+ ctx.window.baselineStart,
+ getEndOfDay(ctx.window.end),
+ ])
+ .execute(),
+ ]);
+
+ const currentMap = buildLookupMap(
+ results,
+ (r) => r.referrer_name || 'direct',
+ (r) => Number(r.cur ?? 0),
+ );
+
+ const baselineMap = buildLookupMap(
+ results,
+ (r) => r.referrer_name || 'direct',
+ (r) => Number(r.base ?? 0),
+ );
+
+ const totalCurrent = totals[0]?.cur_total ?? 0;
+ const totalBaseline = totals[0]?.base_total ?? 0;
+
+ return { currentMap, baselineMap, totalCurrent, totalBaseline };
+}
+
+export const referrersModule: InsightModule = {
+ key: 'referrers',
+ cadence: ['daily'],
+ thresholds: { minTotal: 100, minAbsDelta: 20, minPct: 0.15, maxDims: 50 },
+
+ async enumerateDimensions(ctx) {
+ const { currentMap, baselineMap } = await fetchReferrerAggregates(ctx);
+ const topDims = selectTopDimensions(
+ currentMap,
+ baselineMap,
+ this.thresholds?.maxDims ?? 50,
+ );
+ return topDims.map((dim) => `referrer:${dim}`);
+ },
+
+ async computeMany(ctx, dimensionKeys): Promise {
+ const { currentMap, baselineMap, totalCurrent, totalBaseline } =
+ await fetchReferrerAggregates(ctx);
const results: ComputeResult[] = [];
for (const dimKey of dimensionKeys) {
@@ -148,6 +182,11 @@ export const referrersModule: InsightModule = {
const currentValue = currentMap.get(referrerName) ?? 0;
const compareValue = baselineMap.get(referrerName) ?? 0;
+
+ const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
+ const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
+
+ const shareShiftPp = (currentShare - compareShare) * 100;
const changePct = computeChangePct(currentValue, compareValue);
const direction = computeDirection(changePct);
@@ -159,6 +198,9 @@ export const referrersModule: InsightModule = {
changePct,
direction,
extra: {
+ shareShiftPp,
+ currentShare,
+ compareShare,
isNew: compareValue === 0 && currentValue > 0,
isGone: currentValue === 0 && compareValue > 0,
},
@@ -178,24 +220,55 @@ export const referrersModule: InsightModule = {
? `New traffic source: ${referrer}`
: `Traffic from ${referrer} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
+ const sessionsCurrent = result.currentValue ?? 0;
+ const sessionsCompare = result.compareValue ?? 0;
+ const shareCurrent = Number(result.extra?.currentShare ?? 0);
+ const shareCompare = Number(result.extra?.compareShare ?? 0);
+
return {
- kind: 'insight_v1',
title,
- summary: `${ctx.window.label}. Sessions ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
- primaryDimension: {
- type: 'referrer',
- key: referrer,
- displayName: referrer,
- },
- tags: [
- 'referrers',
- ctx.window.kind,
- isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
- ],
- metric: 'sessions',
- extra: {
- isNew: result.extra?.isNew,
- isGone: result.extra?.isGone,
+ summary: `${ctx.window.label}. Sessions ${sessionsCurrent} vs ${sessionsCompare}.`,
+ displayName: referrer,
+ payload: {
+ kind: 'insight_v1',
+ dimensions: [
+ {
+ key: 'referrer_name',
+ value: referrer,
+ displayName: referrer,
+ },
+ ],
+ primaryMetric: 'sessions',
+ metrics: {
+ sessions: {
+ current: sessionsCurrent,
+ compare: sessionsCompare,
+ delta: sessionsCurrent - sessionsCompare,
+ changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
+ direction: result.direction ?? 'flat',
+ unit: 'count',
+ },
+ share: {
+ current: shareCurrent,
+ compare: shareCompare,
+ delta: shareCurrent - shareCompare,
+ changePct:
+ shareCompare > 0
+ ? (shareCurrent - shareCompare) / shareCompare
+ : null,
+ direction:
+ shareCurrent - shareCompare > 0.0005
+ ? 'up'
+ : shareCurrent - shareCompare < -0.0005
+ ? 'down'
+ : 'flat',
+ unit: 'ratio',
+ },
+ },
+ extra: {
+ isNew: result.extra?.isNew,
+ isGone: result.extra?.isGone,
+ },
},
};
},
diff --git a/packages/db/src/services/insights/normalize.ts b/packages/db/src/services/insights/normalize.ts
deleted file mode 100644
index 65451c21..00000000
--- a/packages/db/src/services/insights/normalize.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-export function normalizeReferrer(name: string): string {
- if (!name || name === '') return 'direct';
-
- const normalized = name.toLowerCase().trim();
-
- // Normalize common referrer variations
- const map: Record = {
- 'm.instagram.com': 'instagram',
- 'l.instagram.com': 'instagram',
- 'www.instagram.com': 'instagram',
- 'instagram.com': 'instagram',
- 't.co': 'twitter',
- 'twitter.com': 'twitter',
- 'x.com': 'twitter',
- 'lm.facebook.com': 'facebook',
- 'm.facebook.com': 'facebook',
- 'facebook.com': 'facebook',
- 'l.facebook.com': 'facebook',
- 'linkedin.com': 'linkedin',
- 'www.linkedin.com': 'linkedin',
- 'youtube.com': 'youtube',
- 'www.youtube.com': 'youtube',
- 'm.youtube.com': 'youtube',
- 'reddit.com': 'reddit',
- 'www.reddit.com': 'reddit',
- 'tiktok.com': 'tiktok',
- 'www.tiktok.com': 'tiktok',
- };
-
- // Check exact match first
- if (map[normalized]) {
- return map[normalized];
- }
-
- // Check if it contains any of the mapped domains
- for (const [key, value] of Object.entries(map)) {
- if (normalized.includes(key)) {
- return value;
- }
- }
-
- // Extract domain from URL if present
- try {
- const url = normalized.startsWith('http')
- ? normalized
- : `https://${normalized}`;
- const hostname = new URL(url).hostname;
- // Remove www. prefix
- return hostname.replace(/^www\./, '');
- } catch {
- // If not a valid URL, return as-is
- return normalized || 'direct';
- }
-}
-
-export function normalizePath(path: string): string {
- if (!path || path === '') return '/';
-
- try {
- // If it's a full URL, extract pathname
- const url = path.startsWith('http')
- ? new URL(path)
- : new URL(path, 'http://x');
- const pathname = url.pathname;
- // Normalize trailing slash (remove unless it's root)
- return pathname === '/' ? '/' : pathname.replace(/\/$/, '');
- } catch {
- // If not a valid URL, treat as path
- return path === '/' ? '/' : path.replace(/\/$/, '') || '/';
- }
-}
-
-export function normalizeUtmCombo(source: string, medium: string): string {
- const s = (source || '').toLowerCase().trim();
- const m = (medium || '').toLowerCase().trim();
- if (!s && !m) return 'none';
- if (!s) return `utm:${m}`;
- if (!m) return `utm:${s}`;
- return `utm:${s}/${m}`;
-}
diff --git a/packages/db/src/services/insights/scoring.ts b/packages/db/src/services/insights/scoring.ts
index a373bc41..7aff4a61 100644
--- a/packages/db/src/services/insights/scoring.ts
+++ b/packages/db/src/services/insights/scoring.ts
@@ -11,8 +11,8 @@ export function severityBand(
changePct?: number | null,
): 'low' | 'moderate' | 'severe' | null {
const p = Math.abs(changePct ?? 0);
- if (p < 0.05) return null;
- if (p < 0.15) return 'low';
- if (p < 0.3) return 'moderate';
+ if (p < 0.1) return null;
+ if (p < 0.5) return 'low';
+ if (p < 1) return 'moderate';
return 'severe';
}
diff --git a/packages/db/src/services/insights/store.ts b/packages/db/src/services/insights/store.ts
index 95c08ec3..6753c3ce 100644
--- a/packages/db/src/services/insights/store.ts
+++ b/packages/db/src/services/insights/store.ts
@@ -13,6 +13,8 @@ export const insightStore: InsightStore = {
const projects = await db.project.findMany({
where: {
deleteAt: null,
+ eventsCount: { gt: 10_000 },
+ updatedAt: { gt: new Date(Date.now() - 1000 * 60 * 60 * 24) },
organization: {
subscriptionStatus: 'active',
},
@@ -22,6 +24,14 @@ export const insightStore: InsightStore = {
return projects.map((p) => p.id);
},
+ async getProjectCreatedAt(projectId: string): Promise {
+ const project = await db.project.findFirst({
+ where: { id: projectId, deleteAt: null },
+ select: { createdAt: true },
+ });
+ return project?.createdAt ?? null;
+ },
+
async getActiveInsightByIdentity({
projectId,
moduleKey,
@@ -52,7 +62,6 @@ export const insightStore: InsightStore = {
lastSeenAt: insight.lastSeenAt,
lastUpdatedAt: insight.lastUpdatedAt,
direction: insight.direction,
- changePct: insight.changePct,
severityBand: insight.severityBand,
};
},
@@ -68,8 +77,6 @@ export const insightStore: InsightStore = {
decision,
prev,
}): Promise {
- const payloadData = (card.payload ?? card) as Prisma.InputJsonValue;
-
const baseData = {
projectId,
moduleKey,
@@ -78,10 +85,8 @@ export const insightStore: InsightStore = {
state: prev?.state === 'closed' ? 'active' : (prev?.state ?? 'active'),
title: card.title,
summary: card.summary ?? null,
- payload: payloadData as Prisma.InputJsonValue,
- currentValue: metrics.currentValue ?? null,
- compareValue: metrics.compareValue ?? null,
- changePct: metrics.changePct ?? null,
+ displayName: card.displayName,
+ payload: card.payload,
direction: metrics.direction ?? null,
impactScore: metrics.impactScore,
severityBand: metrics.severityBand ?? null,
@@ -161,7 +166,6 @@ export const insightStore: InsightStore = {
lastSeenAt: insight.lastSeenAt,
lastUpdatedAt: insight.lastUpdatedAt,
direction: insight.direction,
- changePct: insight.changePct,
severityBand: insight.severityBand,
};
},
diff --git a/packages/db/src/services/insights/types.ts b/packages/db/src/services/insights/types.ts
index 19215457..91728fc7 100644
--- a/packages/db/src/services/insights/types.ts
+++ b/packages/db/src/services/insights/types.ts
@@ -1,4 +1,11 @@
-export type Cadence = 'hourly' | 'daily' | 'weekly';
+import type {
+ InsightDimension,
+ InsightMetricEntry,
+ InsightMetricKey,
+ InsightPayload,
+} from '@openpanel/validation';
+
+export type Cadence = 'daily';
export type WindowKind = 'yesterday' | 'rolling_7d' | 'rolling_30d';
@@ -17,6 +24,12 @@ export interface ComputeContext {
db: any; // your DB client
now: Date;
logger: Pick;
+ /**
+ * Cached clix function that automatically caches query results based on query hash.
+ * This eliminates duplicate queries within the same module+window context.
+ * Use this instead of importing clix directly to benefit from automatic caching.
+ */
+ clix: ReturnType;
}
export interface ComputeResult {
@@ -29,31 +42,22 @@ export interface ComputeResult {
extra?: Record; // share delta pp, rank, sparkline, etc.
}
+// Types imported from @openpanel/validation:
+// - InsightMetricKey
+// - InsightMetricEntry
+// - InsightDimension
+// - InsightPayload
+
/**
* Render should be deterministic and safe to call multiple times.
- * Raw computed values (currentValue, compareValue, changePct, direction)
- * are stored in top-level DB fields. The payload only contains display
- * metadata and module-specific extra data for frontend flexibility.
+ * Returns the shape that matches ProjectInsight create input.
+ * The payload contains all metric data and display metadata.
*/
export interface RenderedCard {
- kind?: 'insight_v1';
title: string;
summary?: string;
- tags?: string[];
- primaryDimension?: { type: string; key: string; displayName?: string };
-
- /**
- * What metric this insight measures - frontend uses this to format values.
- * 'sessions' | 'pageviews' for absolute counts
- * 'share' for percentage-based (geo, devices)
- */
- metric?: 'sessions' | 'pageviews' | 'share';
-
- /**
- * Module-specific extra data (e.g., share values for geo/devices).
- * Frontend can use this based on moduleKey.
- */
- extra?: Record;
+ displayName: string;
+ payload: InsightPayload; // Contains dimensions, primaryMetric, metrics, extra
}
/** Optional per-module thresholds (the engine can still apply global defaults) */
@@ -67,7 +71,8 @@ export interface ModuleThresholds {
export interface InsightModule {
key: string;
cadence: Cadence[];
- windows: WindowKind[];
+ /** Optional per-module override; engine applies a default if omitted. */
+ windows?: WindowKind[];
thresholds?: ModuleThresholds;
enumerateDimensions?(ctx: ComputeContext): Promise;
/** Preferred path: batch compute many dimensions in one go. */
@@ -99,7 +104,6 @@ export interface PersistedInsight {
lastSeenAt: Date;
lastUpdatedAt: Date;
direction?: string | null;
- changePct?: number | null;
severityBand?: string | null;
}
@@ -124,6 +128,8 @@ export interface MaterialDecision {
*/
export interface InsightStore {
listProjectIdsForCadence(cadence: Cadence): Promise;
+ /** Used by the engine/worker to decide if a window has enough baseline history. */
+ getProjectCreatedAt(projectId: string): Promise;
getActiveInsightByIdentity(args: {
projectId: string;
moduleKey: string;
@@ -137,9 +143,6 @@ export interface InsightStore {
window: WindowRange;
card: RenderedCard;
metrics: {
- currentValue?: number;
- compareValue?: number;
- changePct?: number;
direction?: 'up' | 'down' | 'flat';
impactScore: number;
severityBand?: string | null;
@@ -186,15 +189,3 @@ export interface InsightStore {
now: Date;
}): Promise<{ suppressed: number; unsuppressed: number }>;
}
-
-export interface ExplainQueue {
- enqueueExplain(job: {
- insightId: string;
- projectId: string;
- moduleKey: string;
- dimensionKey: string;
- windowKind: WindowKind;
- evidence: Record;
- evidenceHash: string;
- }): Promise;
-}
diff --git a/packages/db/src/services/insights/utils.ts b/packages/db/src/services/insights/utils.ts
index 35807ca4..3dc5eb57 100644
--- a/packages/db/src/services/insights/utils.ts
+++ b/packages/db/src/services/insights/utils.ts
@@ -29,9 +29,7 @@ export function computeMedian(sortedValues: number[]): number {
* @param getDimension - Function to extract normalized dimension from row
* @returns Map of dimension -> median value
*/
-export function computeWeekdayMedians<
- T extends { date: string; cnt: number | string },
->(
+export function computeWeekdayMedians(
data: T[],
targetWeekday: number,
getDimension: (row: T) => string,
@@ -40,12 +38,12 @@ export function computeWeekdayMedians<
const byDimension = new Map();
for (const row of data) {
- const rowWeekday = getWeekday(new Date(row.date));
+ const rowWeekday = getWeekday(new Date((row as any).date));
if (rowWeekday !== targetWeekday) continue;
const dim = getDimension(row);
const values = byDimension.get(dim) ?? [];
- values.push(Number(row.cnt ?? 0));
+ values.push(Number((row as any).cnt ?? 0));
byDimension.set(dim, values);
}
@@ -87,19 +85,6 @@ export function computeDirection(
: 'flat';
}
-/**
- * Merge dimension sets from current and baseline to detect new/gone dimensions
- */
-export function mergeDimensionSets(
- currentDims: Set,
- baselineDims: Set,
-): string[] {
- const merged = new Set();
- for (const dim of currentDims) merged.add(dim);
- for (const dim of baselineDims) merged.add(dim);
- return Array.from(merged);
-}
-
/**
* Get end of day timestamp (23:59:59.999) for a given date.
* Used to ensure BETWEEN queries include the full day.
@@ -109,3 +94,58 @@ export function getEndOfDay(date: Date): Date {
end.setUTCHours(23, 59, 59, 999);
return end;
}
+
+/**
+ * Build a lookup map from query results.
+ * Aggregates counts by key, handling duplicate keys by summing values.
+ *
+ * @param results - Array of result rows
+ * @param getKey - Function to extract the key from each row
+ * @param getCount - Function to extract the count from each row (defaults to 'cnt' field)
+ * @returns Map of key -> aggregated count
+ */
+export function buildLookupMap(
+ results: T[],
+ getKey: (row: T) => string,
+ getCount: (row: T) => number = (row) => Number((row as any).cnt ?? 0),
+): Map {
+ const map = new Map();
+ for (const row of results) {
+ const key = getKey(row);
+ const cnt = getCount(row);
+ map.set(key, (map.get(key) ?? 0) + cnt);
+ }
+ return map;
+}
+
+/**
+ * Select top-N dimensions by ranking on greatest(current, baseline).
+ * This preserves union behavior: dimensions with high values in either period are included.
+ *
+ * @param currentMap - Map of dimension -> current value
+ * @param baselineMap - Map of dimension -> baseline value
+ * @param maxDims - Maximum number of dimensions to return
+ * @returns Array of dimension keys, ranked by greatest(current, baseline)
+ */
+export function selectTopDimensions(
+ currentMap: Map,
+ baselineMap: Map,
+ maxDims: number,
+): string[] {
+ // Merge all dimensions from both maps
+ const allDims = new Set();
+ for (const dim of currentMap.keys()) allDims.add(dim);
+ for (const dim of baselineMap.keys()) allDims.add(dim);
+
+ // Rank by greatest(current, baseline)
+ const ranked = Array.from(allDims)
+ .map((dim) => ({
+ dim,
+ maxValue: Math.max(currentMap.get(dim) ?? 0, baselineMap.get(dim) ?? 0),
+ }))
+ .sort((a, b) => b.maxValue - a.maxValue)
+ .slice(0, maxDims)
+ .map((x) => x.dim);
+
+ return ranked;
+}
diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts
index 7e525f25..ee1a915d 100644
--- a/packages/db/src/types.ts
+++ b/packages/db/src/types.ts
@@ -3,6 +3,7 @@ import type {
IIntegrationConfig,
INotificationRuleConfig,
IProjectFilters,
+ InsightPayload,
} from '@openpanel/validation';
import type {
IClickhouseBotEvent,
@@ -18,6 +19,7 @@ declare global {
type IPrismaIntegrationConfig = IIntegrationConfig;
type IPrismaNotificationPayload = INotificationPayload;
type IPrismaProjectFilters = IProjectFilters[];
+ type IPrismaProjectInsightPayload = InsightPayload;
type IPrismaClickhouseEvent = IClickhouseEvent;
type IPrismaClickhouseProfile = IClickhouseProfile;
type IPrismaClickhouseBotEvent = IClickhouseBotEvent;
diff --git a/packages/trpc/src/routers/insight.ts b/packages/trpc/src/routers/insight.ts
index 6928d6c8..51d2c5a9 100644
--- a/packages/trpc/src/routers/insight.ts
+++ b/packages/trpc/src/routers/insight.ts
@@ -35,22 +35,6 @@ export const insightRouter = createTRPCRouter({
impactScore: 'desc',
},
take: limit * 3, // Fetch 3x to account for deduplication
- select: {
- id: true,
- title: true,
- summary: true,
- payload: true,
- currentValue: true,
- compareValue: true,
- changePct: true,
- direction: true,
- moduleKey: true,
- dimensionKey: true,
- windowKind: true,
- severityBand: true,
- firstDetectedAt: true,
- impactScore: true,
- },
});
// WindowKind priority: yesterday (1) > rolling_7d (2) > rolling_30d (3)
@@ -111,22 +95,6 @@ export const insightRouter = createTRPCRouter({
impactScore: 'desc',
},
take: limit,
- select: {
- id: true,
- title: true,
- summary: true,
- payload: true,
- currentValue: true,
- compareValue: true,
- changePct: true,
- direction: true,
- moduleKey: true,
- dimensionKey: true,
- windowKind: true,
- severityBand: true,
- firstDetectedAt: true,
- impactScore: true,
- },
});
return insights;
diff --git a/packages/validation/index.ts b/packages/validation/index.ts
index bc09a9e7..273d53f8 100644
--- a/packages/validation/index.ts
+++ b/packages/validation/index.ts
@@ -1,2 +1,3 @@
export * from './src/index';
export * from './src/types.validation';
+export * from './src/types.insights';
diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts
index bc8777ee..62162d71 100644
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -553,3 +553,5 @@ export const zCreateImport = z.object({
});
export type ICreateImport = z.infer;
+
+export * from './types.insights';
diff --git a/packages/validation/src/types.insights.ts b/packages/validation/src/types.insights.ts
new file mode 100644
index 00000000..7d2e13cf
--- /dev/null
+++ b/packages/validation/src/types.insights.ts
@@ -0,0 +1,43 @@
+export type InsightMetricKey = 'sessions' | 'pageviews' | 'share';
+
+export type InsightMetricUnit = 'count' | 'ratio';
+
+export interface InsightMetricEntry {
+ current: number;
+ compare: number;
+ delta: number;
+ changePct: number | null;
+ direction: 'up' | 'down' | 'flat';
+ unit: InsightMetricUnit;
+}
+
+export interface InsightDimension {
+ key: string;
+ value: string;
+ displayName?: string;
+}
+
+export interface InsightExtra {
+ [key: string]: unknown;
+ currentShare?: number;
+ compareShare?: number;
+ shareShiftPp?: number;
+ isNew?: boolean;
+ isGone?: boolean;
+}
+
+/**
+ * Shared payload shape for insights cards. This is embedded in DB rows and
+ * shipped to the frontend, so it must remain backwards compatible.
+ */
+export interface InsightPayload {
+ kind?: 'insight_v1';
+ dimensions: InsightDimension[];
+ primaryMetric: InsightMetricKey;
+ metrics: Partial>;
+
+ /**
+ * Module-specific extra data.
+ */
+ extra?: Record;
+}