wip
This commit is contained in:
@@ -44,27 +44,6 @@ const miscRouter: FastifyPluginCallback = async (fastify) => {
|
||||
url: '/geo',
|
||||
handler: controller.getGeo,
|
||||
});
|
||||
|
||||
fastify.route({
|
||||
method: 'GET',
|
||||
url: '/insights/test',
|
||||
handler: async (req, reply) => {
|
||||
const projectId = req.query.projectId as string;
|
||||
const job = await insightsQueue.add(
|
||||
'insightsProject',
|
||||
{
|
||||
type: 'insightsProject',
|
||||
payload: {
|
||||
projectId: projectId,
|
||||
date: new Date().toISOString().slice(0, 10),
|
||||
},
|
||||
},
|
||||
{ jobId: `manual:${Date.now()}:${projectId}` },
|
||||
);
|
||||
|
||||
return { jobId: job.id };
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
export default miscRouter;
|
||||
|
||||
@@ -1,41 +1,13 @@
|
||||
import { countries } from '@/translations/countries';
|
||||
import type { RouterOutputs } from '@/trpc/client';
|
||||
import { cn } from '@/utils/cn';
|
||||
import { ArrowDown, ArrowUp } from 'lucide-react';
|
||||
import type { InsightPayload } from '@openpanel/validation';
|
||||
import { ArrowDown, ArrowUp, FilterIcon, RotateCcwIcon } from 'lucide-react';
|
||||
import { last } from 'ramda';
|
||||
import { useState } from 'react';
|
||||
import { SerieIcon } from '../report-chart/common/serie-icon';
|
||||
import { Badge } from '../ui/badge';
|
||||
|
||||
type InsightPayload = {
|
||||
metric?: 'sessions' | 'pageviews' | 'share';
|
||||
primaryDimension?: {
|
||||
type: string;
|
||||
displayName: string;
|
||||
};
|
||||
extra?: {
|
||||
currentShare?: number;
|
||||
compareShare?: number;
|
||||
shareShiftPp?: number;
|
||||
isNew?: boolean;
|
||||
isGone?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
type Insight = {
|
||||
id: string;
|
||||
title: string;
|
||||
summary: string | null;
|
||||
payload: unknown;
|
||||
currentValue: number | null;
|
||||
compareValue: number | null;
|
||||
changePct: number | null;
|
||||
direction: string | null;
|
||||
moduleKey: string;
|
||||
dimensionKey: string;
|
||||
windowKind: string;
|
||||
severityBand: string | null;
|
||||
impactScore?: number | null;
|
||||
firstDetectedAt?: string | Date;
|
||||
};
|
||||
|
||||
function formatWindowKind(windowKind: string): string {
|
||||
switch (windowKind) {
|
||||
case 'yesterday':
|
||||
@@ -49,82 +21,95 @@ function formatWindowKind(windowKind: string): string {
|
||||
}
|
||||
|
||||
interface InsightCardProps {
|
||||
insight: Insight;
|
||||
insight: RouterOutputs['insight']['list'][number];
|
||||
className?: string;
|
||||
onFilter?: () => void;
|
||||
}
|
||||
|
||||
export function InsightCard({ insight, className }: InsightCardProps) {
|
||||
const payload = insight.payload as InsightPayload | null;
|
||||
const dimension = payload?.primaryDimension;
|
||||
const metric = payload?.metric ?? 'sessions';
|
||||
const extra = payload?.extra;
|
||||
export function InsightCard({
|
||||
insight,
|
||||
className,
|
||||
onFilter,
|
||||
}: InsightCardProps) {
|
||||
const payload = insight.payload;
|
||||
const dimensions = payload?.dimensions;
|
||||
const availableMetrics = Object.entries(payload?.metrics ?? {});
|
||||
|
||||
// Determine if this is a share-based insight (geo, devices)
|
||||
const isShareBased = metric === 'share';
|
||||
// Pick what to display: prefer share if available (geo/devices), else primaryMetric
|
||||
const [metricIndex, setMetricIndex] = useState(
|
||||
availableMetrics.findIndex(([key]) => key === payload?.primaryMetric),
|
||||
);
|
||||
const currentMetricKey = availableMetrics[metricIndex][0];
|
||||
const currentMetricEntry = availableMetrics[metricIndex][1];
|
||||
|
||||
// Get the values to display based on metric type
|
||||
const currentValue = isShareBased
|
||||
? (extra?.currentShare ?? null)
|
||||
: (insight.currentValue ?? null);
|
||||
const compareValue = isShareBased
|
||||
? (extra?.compareShare ?? null)
|
||||
: (insight.compareValue ?? null);
|
||||
const metricUnit = currentMetricEntry?.unit;
|
||||
const currentValue = currentMetricEntry?.current ?? null;
|
||||
const compareValue = currentMetricEntry?.compare ?? null;
|
||||
|
||||
// Get direction and change
|
||||
const direction = insight.direction ?? 'flat';
|
||||
const direction = currentMetricEntry?.direction ?? 'flat';
|
||||
const isIncrease = direction === 'up';
|
||||
const isDecrease = direction === 'down';
|
||||
|
||||
// Format the delta display
|
||||
const deltaText = isShareBased
|
||||
? `${Math.abs(extra?.shareShiftPp ?? 0).toFixed(1)}pp`
|
||||
: `${Math.abs((insight.changePct ?? 0) * 100).toFixed(1)}%`;
|
||||
const deltaText =
|
||||
metricUnit === 'ratio'
|
||||
? `${Math.abs((currentMetricEntry?.delta ?? 0) * 100).toFixed(1)}pp`
|
||||
: `${Math.abs((currentMetricEntry?.changePct ?? 0) * 100).toFixed(1)}%`;
|
||||
|
||||
// Format metric values
|
||||
const formatValue = (value: number | null): string => {
|
||||
if (value == null) return '-';
|
||||
if (isShareBased) return `${(value * 100).toFixed(1)}%`;
|
||||
if (metricUnit === 'ratio') return `${(value * 100).toFixed(1)}%`;
|
||||
return Math.round(value).toLocaleString();
|
||||
};
|
||||
|
||||
// Get the metric label
|
||||
const metricLabel = isShareBased
|
||||
? 'Share'
|
||||
: metric === 'pageviews'
|
||||
? 'Pageviews'
|
||||
: 'Sessions';
|
||||
const metricKeyToLabel = (key: string) =>
|
||||
key === 'share' ? 'Share' : key === 'pageviews' ? 'Pageviews' : 'Sessions';
|
||||
|
||||
const metricLabel = metricKeyToLabel(currentMetricKey);
|
||||
|
||||
const renderTitle = () => {
|
||||
const t = insight.title.replace(/↑.*$/, '').replace(/↓.*$/, '').trim();
|
||||
if (
|
||||
dimension &&
|
||||
(dimension.type === 'country' ||
|
||||
dimension.type === 'referrer' ||
|
||||
dimension.type === 'device')
|
||||
dimensions[0]?.key === 'country' ||
|
||||
dimensions[0]?.key === 'referrer_name' ||
|
||||
dimensions[0]?.key === 'device'
|
||||
) {
|
||||
return (
|
||||
<span className="capitalize flex items-center gap-2">
|
||||
<SerieIcon name={dimension.displayName} />{' '}
|
||||
{countries[dimension.displayName as keyof typeof countries] || t}
|
||||
<SerieIcon name={dimensions[0]?.value} /> {insight.displayName}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
return t;
|
||||
if (insight.displayName.startsWith('http')) {
|
||||
return (
|
||||
<span className="flex items-center gap-2">
|
||||
<SerieIcon
|
||||
name={dimensions[0]?.displayName ?? dimensions[0]?.value}
|
||||
/>
|
||||
<span className="line-clamp-2">{dimensions[1]?.displayName}</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
return insight.displayName;
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
'card p-4 h-full flex flex-col hover:bg-def-50 transition-colors',
|
||||
'card p-4 h-full flex flex-col hover:bg-def-50 transition-colors group/card',
|
||||
className,
|
||||
)}
|
||||
>
|
||||
<div className="row justify-between">
|
||||
<div
|
||||
className={cn(
|
||||
'row justify-between h-4 items-center',
|
||||
onFilter && 'group-hover/card:hidden',
|
||||
)}
|
||||
>
|
||||
<Badge variant="outline" className="-ml-2">
|
||||
{formatWindowKind(insight.windowKind)}
|
||||
<span className="text-muted-foreground mx-1">/</span>
|
||||
<span className="capitalize">{dimension?.type ?? 'unknown'}</span>
|
||||
</Badge>
|
||||
{/* Severity: subtle dot instead of big pill */}
|
||||
{insight.severityBand && (
|
||||
@@ -145,6 +130,36 @@ export function InsightCard({ insight, className }: InsightCardProps) {
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{onFilter && (
|
||||
<div className="row group-hover/card:flex hidden h-4 justify-between gap-2">
|
||||
{availableMetrics.length > 1 ? (
|
||||
<button
|
||||
type="button"
|
||||
className="text-[11px] text-muted-foreground capitalize flex items-center gap-1"
|
||||
onClick={() =>
|
||||
setMetricIndex((metricIndex + 1) % availableMetrics.length)
|
||||
}
|
||||
>
|
||||
<RotateCcwIcon className="size-2" />
|
||||
Show{' '}
|
||||
{metricKeyToLabel(
|
||||
availableMetrics[
|
||||
(metricIndex + 1) % availableMetrics.length
|
||||
][0],
|
||||
)}
|
||||
</button>
|
||||
) : (
|
||||
<div />
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
className="text-[11px] text-muted-foreground capitalize flex items-center gap-1"
|
||||
onClick={onFilter}
|
||||
>
|
||||
Filter <FilterIcon className="size-2" />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
<div className="font-semibold text-sm leading-snug line-clamp-2 mt-2">
|
||||
{renderTitle()}
|
||||
</div>
|
||||
@@ -157,7 +172,7 @@ export function InsightCard({ insight, className }: InsightCardProps) {
|
||||
{metricLabel}
|
||||
</div>
|
||||
|
||||
<div className="flex items-baseline gap-2">
|
||||
<div className="col gap-1">
|
||||
<div className="text-2xl font-semibold tracking-tight">
|
||||
{formatValue(currentValue)}
|
||||
</div>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { useEventQueryFilters } from '@/hooks/use-event-query-filters';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { InsightCard } from '../insights/insight-card';
|
||||
@@ -16,6 +17,7 @@ interface OverviewInsightsProps {
|
||||
|
||||
export default function OverviewInsights({ projectId }: OverviewInsightsProps) {
|
||||
const trpc = useTRPC();
|
||||
const [filters, setFilter] = useEventQueryFilters();
|
||||
const { data: insights, isLoading } = useQuery(
|
||||
trpc.insight.list.queryOptions({
|
||||
projectId,
|
||||
@@ -54,7 +56,14 @@ export default function OverviewInsights({ projectId }: OverviewInsightsProps) {
|
||||
key={insight.id}
|
||||
className="pl-4 basis-full sm:basis-1/2 lg:basis-1/4"
|
||||
>
|
||||
<InsightCard insight={insight} />
|
||||
<InsightCard
|
||||
insight={insight}
|
||||
onFilter={() => {
|
||||
insight.payload.dimensions.forEach((dim) => {
|
||||
void setFilter(dim.key, dim.value, 'is');
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</CarouselItem>
|
||||
))}
|
||||
</CarouselContent>
|
||||
|
||||
@@ -123,7 +123,7 @@ export function SidebarContainer({
|
||||
</div>
|
||||
<div
|
||||
className={cn([
|
||||
'flex flex-grow col gap-1 overflow-auto p-4',
|
||||
'flex flex-grow col gap-1 overflow-auto p-4 hide-scrollbar',
|
||||
"[&_a[data-status='active']]:bg-def-200",
|
||||
])}
|
||||
>
|
||||
|
||||
@@ -208,7 +208,7 @@ const CarouselPrevious = React.forwardRef<
|
||||
variant={variant}
|
||||
size={size}
|
||||
className={cn(
|
||||
'absolute h-10 w-10 rounded-full hover:scale-100 hover:translate-y-[-50%] transition-all duration-200',
|
||||
'absolute h-10 w-10 rounded-full hover:scale-100 hover:translate-y-[-50%] transition-all duration-200',
|
||||
orientation === 'horizontal'
|
||||
? 'left-6 top-1/2 -translate-y-1/2'
|
||||
: '-top-12 left-1/2 -translate-x-1/2 rotate-90',
|
||||
|
||||
@@ -3,6 +3,13 @@ import { InsightCard } from '@/components/insights/insight-card';
|
||||
import { PageContainer } from '@/components/page-container';
|
||||
import { PageHeader } from '@/components/page-header';
|
||||
import { Skeleton } from '@/components/skeleton';
|
||||
import {
|
||||
Carousel,
|
||||
CarouselContent,
|
||||
CarouselItem,
|
||||
CarouselNext,
|
||||
CarouselPrevious,
|
||||
} from '@/components/ui/carousel';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import {
|
||||
Select,
|
||||
@@ -13,10 +20,12 @@ import {
|
||||
} from '@/components/ui/select';
|
||||
import { TableButtons } from '@/components/ui/table';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
import { cn } from '@/utils/cn';
|
||||
import { PAGE_TITLES, createProjectTitle } from '@/utils/title';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { createFileRoute } from '@tanstack/react-router';
|
||||
import { useMemo, useState } from 'react';
|
||||
import { createFileRoute, useNavigate } from '@tanstack/react-router';
|
||||
import { parseAsString, parseAsStringEnum, useQueryState } from 'nuqs';
|
||||
import { useMemo } from 'react';
|
||||
|
||||
export const Route = createFileRoute(
|
||||
'/_app/$organizationId/$projectId/insights',
|
||||
@@ -26,7 +35,7 @@ export const Route = createFileRoute(
|
||||
return {
|
||||
meta: [
|
||||
{
|
||||
title: createProjectTitle('Insights'),
|
||||
title: createProjectTitle(PAGE_TITLES.INSIGHTS),
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -40,6 +49,19 @@ type SortOption =
|
||||
| 'severity-asc'
|
||||
| 'recent';
|
||||
|
||||
function getModuleDisplayName(moduleKey: string): string {
|
||||
const displayNames: Record<string, string> = {
|
||||
geo: 'Geographic',
|
||||
devices: 'Devices',
|
||||
referrers: 'Referrers',
|
||||
'entry-pages': 'Entry Pages',
|
||||
'page-trends': 'Page Trends',
|
||||
'exit-pages': 'Exit Pages',
|
||||
'traffic-anomalies': 'Anomalies',
|
||||
};
|
||||
return displayNames[moduleKey] || moduleKey.replace('-', ' ');
|
||||
}
|
||||
|
||||
function Component() {
|
||||
const { projectId } = Route.useParams();
|
||||
const trpc = useTRPC();
|
||||
@@ -49,13 +71,45 @@ function Component() {
|
||||
limit: 500,
|
||||
}),
|
||||
);
|
||||
const navigate = useNavigate();
|
||||
|
||||
const [search, setSearch] = useState('');
|
||||
const [moduleFilter, setModuleFilter] = useState<string>('all');
|
||||
const [windowKindFilter, setWindowKindFilter] = useState<string>('all');
|
||||
const [severityFilter, setSeverityFilter] = useState<string>('all');
|
||||
const [directionFilter, setDirectionFilter] = useState<string>('all');
|
||||
const [sortBy, setSortBy] = useState<SortOption>('impact-desc');
|
||||
const [search, setSearch] = useQueryState(
|
||||
'search',
|
||||
parseAsString.withDefault(''),
|
||||
);
|
||||
const [moduleFilter, setModuleFilter] = useQueryState(
|
||||
'module',
|
||||
parseAsString.withDefault('all'),
|
||||
);
|
||||
const [windowKindFilter, setWindowKindFilter] = useQueryState(
|
||||
'window',
|
||||
parseAsStringEnum([
|
||||
'all',
|
||||
'yesterday',
|
||||
'rolling_7d',
|
||||
'rolling_30d',
|
||||
]).withDefault('all'),
|
||||
);
|
||||
const [severityFilter, setSeverityFilter] = useQueryState(
|
||||
'severity',
|
||||
parseAsStringEnum(['all', 'severe', 'moderate', 'low', 'none']).withDefault(
|
||||
'all',
|
||||
),
|
||||
);
|
||||
const [directionFilter, setDirectionFilter] = useQueryState(
|
||||
'direction',
|
||||
parseAsStringEnum(['all', 'up', 'down', 'flat']).withDefault('all'),
|
||||
);
|
||||
const [sortBy, setSortBy] = useQueryState(
|
||||
'sort',
|
||||
parseAsStringEnum<SortOption>([
|
||||
'impact-desc',
|
||||
'impact-asc',
|
||||
'severity-desc',
|
||||
'severity-asc',
|
||||
'recent',
|
||||
]).withDefault('impact-desc'),
|
||||
);
|
||||
|
||||
const filteredAndSorted = useMemo(() => {
|
||||
if (!insights) return [];
|
||||
@@ -155,18 +209,60 @@ function Component() {
|
||||
sortBy,
|
||||
]);
|
||||
|
||||
const uniqueModules = useMemo(() => {
|
||||
if (!insights) return [];
|
||||
return Array.from(new Set(insights.map((i) => i.moduleKey))).sort();
|
||||
}, [insights]);
|
||||
// Group insights by module
|
||||
const groupedByModule = useMemo(() => {
|
||||
const groups = new Map<string, typeof filteredAndSorted>();
|
||||
|
||||
for (const insight of filteredAndSorted) {
|
||||
const existing = groups.get(insight.moduleKey) ?? [];
|
||||
existing.push(insight);
|
||||
groups.set(insight.moduleKey, existing);
|
||||
}
|
||||
|
||||
// Sort modules by impact (referrers first, then by average impact score)
|
||||
return Array.from(groups.entries()).sort(
|
||||
([keyA, insightsA], [keyB, insightsB]) => {
|
||||
// Referrers always first
|
||||
if (keyA === 'referrers') return -1;
|
||||
if (keyB === 'referrers') return 1;
|
||||
|
||||
// Calculate average impact for each module
|
||||
const avgImpactA =
|
||||
insightsA.reduce((sum, i) => sum + (i.impactScore ?? 0), 0) /
|
||||
insightsA.length;
|
||||
const avgImpactB =
|
||||
insightsB.reduce((sum, i) => sum + (i.impactScore ?? 0), 0) /
|
||||
insightsB.length;
|
||||
|
||||
// Sort by average impact (high to low)
|
||||
return avgImpactB - avgImpactA;
|
||||
},
|
||||
);
|
||||
}, [filteredAndSorted]);
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<PageContainer>
|
||||
<PageHeader title="Insights" className="mb-8" />
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4">
|
||||
{Array.from({ length: 8 }, (_, i) => `skeleton-${i}`).map((key) => (
|
||||
<Skeleton key={key} className="h-48" />
|
||||
<div className="space-y-8">
|
||||
{Array.from({ length: 3 }, (_, i) => `section-${i}`).map((key) => (
|
||||
<div key={key} className="space-y-4">
|
||||
<Skeleton className="h-8 w-32" />
|
||||
<Carousel opts={{ align: 'start' }} className="w-full">
|
||||
<CarouselContent className="-ml-4">
|
||||
{Array.from({ length: 4 }, (_, i) => `skeleton-${i}`).map(
|
||||
(cardKey) => (
|
||||
<CarouselItem
|
||||
key={cardKey}
|
||||
className="pl-4 basis-full sm:basis-1/2 lg:basis-1/3 xl:basis-1/4"
|
||||
>
|
||||
<Skeleton className="h-48 w-full" />
|
||||
</CarouselItem>
|
||||
),
|
||||
)}
|
||||
</CarouselContent>
|
||||
</Carousel>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</PageContainer>
|
||||
@@ -180,27 +276,19 @@ function Component() {
|
||||
description="Discover trends and changes in your analytics"
|
||||
className="mb-8"
|
||||
/>
|
||||
<TableButtons>
|
||||
<TableButtons className="mb-8">
|
||||
<Input
|
||||
placeholder="Search insights..."
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
value={search ?? ''}
|
||||
onChange={(e) => void setSearch(e.target.value || null)}
|
||||
className="max-w-xs"
|
||||
/>
|
||||
<Select value={moduleFilter} onValueChange={setModuleFilter}>
|
||||
<SelectTrigger className="w-[140px]">
|
||||
<SelectValue placeholder="Module" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="all">All Modules</SelectItem>
|
||||
{uniqueModules.map((module) => (
|
||||
<SelectItem key={module} value={module}>
|
||||
{module.replace('-', ' ')}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Select value={windowKindFilter} onValueChange={setWindowKindFilter}>
|
||||
<Select
|
||||
value={windowKindFilter ?? 'all'}
|
||||
onValueChange={(v) =>
|
||||
void setWindowKindFilter(v as typeof windowKindFilter)
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-[140px]">
|
||||
<SelectValue placeholder="Time Window" />
|
||||
</SelectTrigger>
|
||||
@@ -211,7 +299,12 @@ function Component() {
|
||||
<SelectItem value="rolling_30d">30 Days</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Select value={severityFilter} onValueChange={setSeverityFilter}>
|
||||
<Select
|
||||
value={severityFilter ?? 'all'}
|
||||
onValueChange={(v) =>
|
||||
void setSeverityFilter(v as typeof severityFilter)
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-[140px]">
|
||||
<SelectValue placeholder="Severity" />
|
||||
</SelectTrigger>
|
||||
@@ -223,7 +316,12 @@ function Component() {
|
||||
<SelectItem value="none">No Severity</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Select value={directionFilter} onValueChange={setDirectionFilter}>
|
||||
<Select
|
||||
value={directionFilter ?? 'all'}
|
||||
onValueChange={(v) =>
|
||||
void setDirectionFilter(v as typeof directionFilter)
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-[140px]">
|
||||
<SelectValue placeholder="Direction" />
|
||||
</SelectTrigger>
|
||||
@@ -235,8 +333,8 @@ function Component() {
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Select
|
||||
value={sortBy}
|
||||
onValueChange={(v) => setSortBy(v as SortOption)}
|
||||
value={sortBy ?? 'impact-desc'}
|
||||
onValueChange={(v) => void setSortBy(v as SortOption)}
|
||||
>
|
||||
<SelectTrigger className="w-[160px]">
|
||||
<SelectValue placeholder="Sort by" />
|
||||
@@ -262,14 +360,69 @@ function Component() {
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4">
|
||||
{filteredAndSorted.map((insight) => (
|
||||
<InsightCard key={insight.id} insight={insight} />
|
||||
))}
|
||||
</div>
|
||||
{groupedByModule.length > 0 && (
|
||||
<div className="space-y-8">
|
||||
{groupedByModule.map(([moduleKey, moduleInsights]) => (
|
||||
<div key={moduleKey} className="space-y-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="text-lg font-semibold capitalize">
|
||||
{getModuleDisplayName(moduleKey)}
|
||||
</h2>
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{moduleInsights.length}{' '}
|
||||
{moduleInsights.length === 1 ? 'insight' : 'insights'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="-mx-8">
|
||||
<Carousel
|
||||
opts={{ align: 'start', dragFree: true }}
|
||||
className="w-full group"
|
||||
>
|
||||
<CarouselContent className="mx-4 mr-8">
|
||||
{moduleInsights.map((insight, index) => (
|
||||
<CarouselItem
|
||||
key={insight.id}
|
||||
className={cn(
|
||||
'pl-4 basis-full sm:basis-1/2 lg:basis-1/3 xl:basis-1/4',
|
||||
)}
|
||||
>
|
||||
<InsightCard
|
||||
insight={insight}
|
||||
onFilter={(() => {
|
||||
const filterString = insight.payload?.dimensions
|
||||
.map(
|
||||
(dim) =>
|
||||
`${dim.key},is,${encodeURIComponent(dim.value)}`,
|
||||
)
|
||||
.join(';');
|
||||
if (filterString) {
|
||||
return () => {
|
||||
navigate({
|
||||
to: '/$organizationId/$projectId',
|
||||
from: Route.fullPath,
|
||||
search: {
|
||||
f: filterString,
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
})()}
|
||||
/>
|
||||
</CarouselItem>
|
||||
))}
|
||||
</CarouselContent>
|
||||
<CarouselPrevious className="opacity-0 [&:disabled]:opacity-0 pointer-events-none transition-opacity group-hover:opacity-100 group-hover:pointer-events-auto left-3" />
|
||||
<CarouselNext className="opacity-0 &:disabled:opacity-0 pointer-events-none transition-opacity group-hover:opacity-100 group-hover:pointer-events-auto right-3" />
|
||||
</Carousel>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{filteredAndSorted.length > 0 && (
|
||||
<div className="mt-4 text-sm text-muted-foreground text-center">
|
||||
<div className="mt-8 text-sm text-muted-foreground text-center">
|
||||
Showing {filteredAndSorted.length} of {insights?.length ?? 0} insights
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -90,6 +90,7 @@ export const PAGE_TITLES = {
|
||||
CHAT: 'AI Assistant',
|
||||
REALTIME: 'Realtime',
|
||||
REFERENCES: 'References',
|
||||
INSIGHTS: 'Insights',
|
||||
// Profiles
|
||||
PROFILES: 'Profiles',
|
||||
PROFILE_EVENTS: 'Profile events',
|
||||
|
||||
@@ -34,6 +34,11 @@ export async function bootCron() {
|
||||
type: 'flushSessions',
|
||||
pattern: 1000 * 10,
|
||||
},
|
||||
{
|
||||
name: 'insightsDaily',
|
||||
type: 'insightsDaily',
|
||||
pattern: '0 2 * * *',
|
||||
},
|
||||
];
|
||||
|
||||
if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {
|
||||
@@ -44,12 +49,6 @@ export async function bootCron() {
|
||||
});
|
||||
}
|
||||
|
||||
jobs.push({
|
||||
name: 'insightsDaily',
|
||||
type: 'insightsDaily',
|
||||
pattern: '0 2 * * *', // 2 AM daily
|
||||
});
|
||||
|
||||
logger.info('Updating cron jobs');
|
||||
|
||||
const jobSchedulers = await cronQueue.getJobSchedulers();
|
||||
|
||||
@@ -79,11 +79,6 @@ async function start() {
|
||||
await bootCron();
|
||||
} else {
|
||||
logger.warn('Workers are disabled');
|
||||
|
||||
// Start insights worker
|
||||
const insightsWorker = new Worker(insightsQueue.name, insightsProjectJob, {
|
||||
connection: getRedisQueue(),
|
||||
});
|
||||
}
|
||||
|
||||
await createInitialSalts();
|
||||
|
||||
@@ -16,7 +16,7 @@ import { insightsQueue } from '@openpanel/queue';
|
||||
import type { Job } from 'bullmq';
|
||||
|
||||
const defaultEngineConfig = {
|
||||
keepTopNPerModuleWindow: 5,
|
||||
keepTopNPerModuleWindow: 20,
|
||||
closeStaleAfterDays: 7,
|
||||
dimensionBatchSize: 50,
|
||||
globalThresholds: {
|
||||
@@ -24,8 +24,6 @@ const defaultEngineConfig = {
|
||||
minAbsDelta: 80,
|
||||
minPct: 0.15,
|
||||
},
|
||||
enableExplain: false,
|
||||
explainTopNPerProjectPerDay: 3,
|
||||
};
|
||||
|
||||
export async function insightsDailyJob(job: Job<CronQueuePayload>) {
|
||||
@@ -63,9 +61,12 @@ export async function insightsProjectJob(
|
||||
config: defaultEngineConfig,
|
||||
});
|
||||
|
||||
const projectCreatedAt = await insightStore.getProjectCreatedAt(projectId);
|
||||
|
||||
await engine.runProject({
|
||||
projectId,
|
||||
cadence: 'daily',
|
||||
now: new Date(date),
|
||||
projectCreatedAt,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -245,3 +245,259 @@ export function getDefaultIntervalByDates(
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export const countries = {
|
||||
AF: 'Afghanistan',
|
||||
AL: 'Albania',
|
||||
DZ: 'Algeria',
|
||||
AS: 'American Samoa',
|
||||
AD: 'Andorra',
|
||||
AO: 'Angola',
|
||||
AI: 'Anguilla',
|
||||
AQ: 'Antarctica',
|
||||
AG: 'Antigua and Barbuda',
|
||||
AR: 'Argentina',
|
||||
AM: 'Armenia',
|
||||
AW: 'Aruba',
|
||||
AU: 'Australia',
|
||||
AT: 'Austria',
|
||||
AZ: 'Azerbaijan',
|
||||
BS: 'Bahamas',
|
||||
BH: 'Bahrain',
|
||||
BD: 'Bangladesh',
|
||||
BB: 'Barbados',
|
||||
BY: 'Belarus',
|
||||
BE: 'Belgium',
|
||||
BZ: 'Belize',
|
||||
BJ: 'Benin',
|
||||
BM: 'Bermuda',
|
||||
BT: 'Bhutan',
|
||||
BO: 'Bolivia',
|
||||
BQ: 'Bonaire, Sint Eustatius and Saba',
|
||||
BA: 'Bosnia and Herzegovina',
|
||||
BW: 'Botswana',
|
||||
BV: 'Bouvet Island',
|
||||
BR: 'Brazil',
|
||||
IO: 'British Indian Ocean Territory',
|
||||
BN: 'Brunei Darussalam',
|
||||
BG: 'Bulgaria',
|
||||
BF: 'Burkina Faso',
|
||||
BI: 'Burundi',
|
||||
CV: 'Cabo Verde',
|
||||
KH: 'Cambodia',
|
||||
CM: 'Cameroon',
|
||||
CA: 'Canada',
|
||||
KY: 'Cayman Islands',
|
||||
CF: 'Central African Republic',
|
||||
TD: 'Chad',
|
||||
CL: 'Chile',
|
||||
CN: 'China',
|
||||
CX: 'Christmas Island',
|
||||
CC: 'Cocos (Keeling) Islands',
|
||||
CO: 'Colombia',
|
||||
KM: 'Comoros',
|
||||
CD: 'Congo (Democratic Republic)',
|
||||
CG: 'Congo',
|
||||
CK: 'Cook Islands',
|
||||
CR: 'Costa Rica',
|
||||
HR: 'Croatia',
|
||||
CU: 'Cuba',
|
||||
CW: 'Curaçao',
|
||||
CY: 'Cyprus',
|
||||
CZ: 'Czechia',
|
||||
CI: "Côte d'Ivoire",
|
||||
DK: 'Denmark',
|
||||
DJ: 'Djibouti',
|
||||
DM: 'Dominica',
|
||||
DO: 'Dominican Republic',
|
||||
EC: 'Ecuador',
|
||||
EG: 'Egypt',
|
||||
SV: 'El Salvador',
|
||||
GQ: 'Equatorial Guinea',
|
||||
ER: 'Eritrea',
|
||||
EE: 'Estonia',
|
||||
SZ: 'Eswatina',
|
||||
ET: 'Ethiopia',
|
||||
FK: 'Falkland Islands',
|
||||
FO: 'Faroe Islands',
|
||||
FJ: 'Fiji',
|
||||
FI: 'Finland',
|
||||
FR: 'France',
|
||||
GF: 'French Guiana',
|
||||
PF: 'French Polynesia',
|
||||
TF: 'French Southern Territories',
|
||||
GA: 'Gabon',
|
||||
GM: 'Gambia',
|
||||
GE: 'Georgia',
|
||||
DE: 'Germany',
|
||||
GH: 'Ghana',
|
||||
GI: 'Gibraltar',
|
||||
GR: 'Greece',
|
||||
GL: 'Greenland',
|
||||
GD: 'Grenada',
|
||||
GP: 'Guadeloupe',
|
||||
GU: 'Guam',
|
||||
GT: 'Guatemala',
|
||||
GG: 'Guernsey',
|
||||
GN: 'Guinea',
|
||||
GW: 'Guinea-Bissau',
|
||||
GY: 'Guyana',
|
||||
HT: 'Haiti',
|
||||
HM: 'Heard Island and McDonald Islands',
|
||||
VA: 'Holy See',
|
||||
HN: 'Honduras',
|
||||
HK: 'Hong Kong',
|
||||
HU: 'Hungary',
|
||||
IS: 'Iceland',
|
||||
IN: 'India',
|
||||
ID: 'Indonesia',
|
||||
IR: 'Iran',
|
||||
IQ: 'Iraq',
|
||||
IE: 'Ireland',
|
||||
IM: 'Isle of Man',
|
||||
IL: 'Israel',
|
||||
IT: 'Italy',
|
||||
JM: 'Jamaica',
|
||||
JP: 'Japan',
|
||||
JE: 'Jersey',
|
||||
JO: 'Jordan',
|
||||
KZ: 'Kazakhstan',
|
||||
KE: 'Kenya',
|
||||
KI: 'Kiribati',
|
||||
KP: "Korea (Democratic People's Republic)",
|
||||
KR: 'Korea (Republic)',
|
||||
KW: 'Kuwait',
|
||||
KG: 'Kyrgyzstan',
|
||||
LA: "Lao People's Democratic Republic",
|
||||
LV: 'Latvia',
|
||||
LB: 'Lebanon',
|
||||
LS: 'Lesotho',
|
||||
LR: 'Liberia',
|
||||
LY: 'Libya',
|
||||
LI: 'Liechtenstein',
|
||||
LT: 'Lithuania',
|
||||
LU: 'Luxembourg',
|
||||
MO: 'Macao',
|
||||
MG: 'Madagascar',
|
||||
MW: 'Malawi',
|
||||
MY: 'Malaysia',
|
||||
MV: 'Maldives',
|
||||
ML: 'Mali',
|
||||
MT: 'Malta',
|
||||
MH: 'Marshall Islands',
|
||||
MQ: 'Martinique',
|
||||
MR: 'Mauritania',
|
||||
MU: 'Mauritius',
|
||||
YT: 'Mayotte',
|
||||
MX: 'Mexico',
|
||||
FM: 'Micronesia',
|
||||
MD: 'Moldova',
|
||||
MC: 'Monaco',
|
||||
MN: 'Mongolia',
|
||||
ME: 'Montenegro',
|
||||
MS: 'Montserrat',
|
||||
MA: 'Morocco',
|
||||
MZ: 'Mozambique',
|
||||
MM: 'Myanmar',
|
||||
NA: 'Namibia',
|
||||
NR: 'Nauru',
|
||||
NP: 'Nepal',
|
||||
NL: 'Netherlands',
|
||||
NC: 'New Caledonia',
|
||||
NZ: 'New Zealand',
|
||||
NI: 'Nicaragua',
|
||||
NE: 'Niger',
|
||||
NG: 'Nigeria',
|
||||
NU: 'Niue',
|
||||
NF: 'Norfolk Island',
|
||||
MP: 'Northern Mariana Islands',
|
||||
NO: 'Norway',
|
||||
OM: 'Oman',
|
||||
PK: 'Pakistan',
|
||||
PW: 'Palau',
|
||||
PS: 'Palestine, State of',
|
||||
PA: 'Panama',
|
||||
PG: 'Papua New Guinea',
|
||||
PY: 'Paraguay',
|
||||
PE: 'Peru',
|
||||
PH: 'Philippines',
|
||||
PN: 'Pitcairn',
|
||||
PL: 'Poland',
|
||||
PT: 'Portugal',
|
||||
PR: 'Puerto Rico',
|
||||
QA: 'Qatar',
|
||||
MK: 'Republic of North Macedonia',
|
||||
RO: 'Romania',
|
||||
RU: 'Russian Federation',
|
||||
RW: 'Rwanda',
|
||||
RE: 'Réunion',
|
||||
BL: 'Saint Barthélemy',
|
||||
SH: 'Saint Helena, Ascension and Tristan da Cunha',
|
||||
KN: 'Saint Kitts and Nevis',
|
||||
LC: 'Saint Lucia',
|
||||
MF: 'Saint Martin (French part)',
|
||||
PM: 'Saint Pierre and Miquelon',
|
||||
VC: 'Saint Vincent and the Grenadines',
|
||||
WS: 'Samoa',
|
||||
SM: 'San Marino',
|
||||
ST: 'Sao Tome and Principe',
|
||||
SA: 'Saudi Arabia',
|
||||
SN: 'Senegal',
|
||||
RS: 'Serbia',
|
||||
SC: 'Seychelles',
|
||||
SL: 'Sierra Leone',
|
||||
SG: 'Singapore',
|
||||
SX: 'Sint Maarten (Dutch part)',
|
||||
SK: 'Slovakia',
|
||||
SI: 'Slovenia',
|
||||
SB: 'Solomon Islands',
|
||||
SO: 'Somalia',
|
||||
ZA: 'South Africa',
|
||||
GS: 'South Georgia and the South Sandwich Islands',
|
||||
SS: 'South Sudan',
|
||||
ES: 'Spain',
|
||||
LK: 'Sri Lanka',
|
||||
SD: 'Sudan',
|
||||
SR: 'Suriname',
|
||||
SJ: 'Svalbard and Jan Mayen',
|
||||
SE: 'Sweden',
|
||||
CH: 'Switzerland',
|
||||
SY: 'Syrian Arab Republic',
|
||||
TW: 'Taiwan',
|
||||
TJ: 'Tajikistan',
|
||||
TZ: 'Tanzania, United Republic of',
|
||||
TH: 'Thailand',
|
||||
TL: 'Timor-Leste',
|
||||
TG: 'Togo',
|
||||
TK: 'Tokelau',
|
||||
TO: 'Tonga',
|
||||
TT: 'Trinidad and Tobago',
|
||||
TN: 'Tunisia',
|
||||
TR: 'Turkey',
|
||||
TM: 'Turkmenistan',
|
||||
TC: 'Turks and Caicos Islands',
|
||||
TV: 'Tuvalu',
|
||||
UG: 'Uganda',
|
||||
UA: 'Ukraine',
|
||||
AE: 'United Arab Emirates',
|
||||
GB: 'United Kingdom',
|
||||
US: 'United States',
|
||||
UM: 'United States Minor Outlying Islands',
|
||||
UY: 'Uruguay',
|
||||
UZ: 'Uzbekistan',
|
||||
VU: 'Vanuatu',
|
||||
VE: 'Venezuela',
|
||||
VN: 'Viet Nam',
|
||||
VG: 'Virgin Islands (British)',
|
||||
VI: 'Virgin Islands (U.S.)',
|
||||
WF: 'Wallis and Futuna',
|
||||
EH: 'Western Sahara',
|
||||
YE: 'Yemen',
|
||||
ZM: 'Zambia',
|
||||
ZW: 'Zimbabwe',
|
||||
AX: 'Åland Islands',
|
||||
} as const;
|
||||
|
||||
export function getCountry(code?: string) {
|
||||
return countries[code as keyof typeof countries];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- Made the column `payload` on table `project_insights` required. This step will fail if there are existing NULL values in that column.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "public"."project_insights" ALTER COLUMN "payload" SET NOT NULL,
|
||||
ALTER COLUMN "payload" SET DEFAULT '{}';
|
||||
@@ -0,0 +1,13 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- You are about to drop the column `changePct` on the `project_insights` table. All the data in the column will be lost.
|
||||
- You are about to drop the column `compareValue` on the `project_insights` table. All the data in the column will be lost.
|
||||
- You are about to drop the column `currentValue` on the `project_insights` table. All the data in the column will be lost.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "public"."project_insights" DROP COLUMN "changePct",
|
||||
DROP COLUMN "compareValue",
|
||||
DROP COLUMN "currentValue",
|
||||
ADD COLUMN "displayName" TEXT NOT NULL DEFAULT '';
|
||||
@@ -512,13 +512,12 @@ model ProjectInsight {
|
||||
windowKind String // "yesterday" | "rolling_7d" | "rolling_30d"
|
||||
state InsightState @default(active)
|
||||
|
||||
title String
|
||||
summary String?
|
||||
payload Json? // RenderedCard blocks, extra data
|
||||
title String
|
||||
summary String?
|
||||
displayName String @default("")
|
||||
/// [IPrismaProjectInsightPayload]
|
||||
payload Json @default("{}") // Rendered insight payload (typed)
|
||||
|
||||
currentValue Float?
|
||||
compareValue Float?
|
||||
changePct Float?
|
||||
direction String? // "up" | "down" | "flat"
|
||||
impactScore Float @default(0)
|
||||
severityBand String? // "low" | "moderate" | "severe"
|
||||
|
||||
@@ -43,7 +43,7 @@ class Expression {
|
||||
}
|
||||
|
||||
export class Query<T = any> {
|
||||
private _select: string[] = [];
|
||||
private _select: (string | Expression)[] = [];
|
||||
private _except: string[] = [];
|
||||
private _from?: string | Expression;
|
||||
private _where: WhereCondition[] = [];
|
||||
@@ -81,17 +81,19 @@ export class Query<T = any> {
|
||||
|
||||
// Select methods
|
||||
select<U>(
|
||||
columns: (string | null | undefined | false)[],
|
||||
columns: (string | Expression | null | undefined | false)[],
|
||||
type: 'merge' | 'replace' = 'replace',
|
||||
): Query<U> {
|
||||
if (this._skipNext) return this as unknown as Query<U>;
|
||||
if (type === 'merge') {
|
||||
this._select = [
|
||||
...this._select,
|
||||
...columns.filter((col): col is string => Boolean(col)),
|
||||
...columns.filter((col): col is string | Expression => Boolean(col)),
|
||||
];
|
||||
} else {
|
||||
this._select = columns.filter((col): col is string => Boolean(col));
|
||||
this._select = columns.filter((col): col is string | Expression =>
|
||||
Boolean(col),
|
||||
);
|
||||
}
|
||||
return this as unknown as Query<U>;
|
||||
}
|
||||
@@ -372,7 +374,14 @@ export class Query<T = any> {
|
||||
if (this._select.length > 0) {
|
||||
parts.push(
|
||||
'SELECT',
|
||||
this._select.map((col) => this.escapeDate(col)).join(', '),
|
||||
this._select
|
||||
// Important: Expressions are treated as raw SQL; do not run escapeDate()
|
||||
// on them, otherwise any embedded date strings get double-escaped
|
||||
// (e.g. ''2025-12-16 23:59:59'') which ClickHouse rejects.
|
||||
.map((col) =>
|
||||
col instanceof Expression ? col.toString() : this.escapeDate(col),
|
||||
)
|
||||
.join(', '),
|
||||
);
|
||||
} else {
|
||||
parts.push('SELECT *');
|
||||
|
||||
68
packages/db/src/services/insights/cached-clix.ts
Normal file
68
packages/db/src/services/insights/cached-clix.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import crypto from 'node:crypto';
|
||||
import type { ClickHouseClient } from '@clickhouse/client';
|
||||
import {
|
||||
type Query,
|
||||
clix as originalClix,
|
||||
} from '../../clickhouse/query-builder';
|
||||
|
||||
/**
|
||||
* Creates a cached wrapper around clix that automatically caches query results
|
||||
* based on query hash. This eliminates duplicate queries within the same module/window context.
|
||||
*
|
||||
* @param client - ClickHouse client
|
||||
* @param cache - Optional cache Map to store query results
|
||||
* @param timezone - Timezone for queries (defaults to UTC)
|
||||
* @returns A function that creates cached Query instances (compatible with clix API)
|
||||
*/
|
||||
export function createCachedClix(
|
||||
client: ClickHouseClient,
|
||||
cache?: Map<string, any>,
|
||||
timezone?: string,
|
||||
) {
|
||||
function clixCached(): Query {
|
||||
const query = originalClix(client, timezone);
|
||||
const queryTimezone = timezone ?? 'UTC';
|
||||
|
||||
// Override execute() method to add caching
|
||||
const originalExecute = query.execute.bind(query);
|
||||
query.execute = async () => {
|
||||
// Build the query SQL string
|
||||
const querySQL = query.toSQL();
|
||||
|
||||
// Create cache key from query SQL + timezone
|
||||
const cacheKey = crypto
|
||||
.createHash('sha256')
|
||||
.update(`${querySQL}|${queryTimezone}`)
|
||||
.digest('hex');
|
||||
|
||||
// Check cache first
|
||||
if (cache?.has(cacheKey)) {
|
||||
return cache.get(cacheKey);
|
||||
}
|
||||
|
||||
// Execute query
|
||||
const result = await originalExecute();
|
||||
|
||||
// Cache the result
|
||||
if (cache) {
|
||||
cache.set(cacheKey, result);
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
// Copy static methods from original clix
|
||||
clixCached.exp = originalClix.exp;
|
||||
clixCached.date = originalClix.date;
|
||||
clixCached.datetime = originalClix.datetime;
|
||||
clixCached.dynamicDatetime = originalClix.dynamicDatetime;
|
||||
clixCached.toStartOf = originalClix.toStartOf;
|
||||
clixCached.toStartOfInterval = originalClix.toStartOfInterval;
|
||||
clixCached.toInterval = originalClix.toInterval;
|
||||
clixCached.toDate = originalClix.toDate;
|
||||
|
||||
return clixCached;
|
||||
}
|
||||
@@ -1,17 +1,22 @@
|
||||
import crypto from 'node:crypto';
|
||||
import { createCachedClix } from './cached-clix';
|
||||
import { materialDecision } from './material';
|
||||
import { defaultImpactScore, severityBand } from './scoring';
|
||||
import type {
|
||||
Cadence,
|
||||
ComputeContext,
|
||||
ComputeResult,
|
||||
ExplainQueue,
|
||||
InsightModule,
|
||||
InsightStore,
|
||||
WindowKind,
|
||||
} from './types';
|
||||
import { resolveWindow } from './windows';
|
||||
|
||||
const DEFAULT_WINDOWS: WindowKind[] = [
|
||||
'yesterday',
|
||||
'rolling_7d',
|
||||
'rolling_30d',
|
||||
];
|
||||
|
||||
export interface EngineConfig {
|
||||
keepTopNPerModuleWindow: number; // e.g. 5
|
||||
closeStaleAfterDays: number; // e.g. 7
|
||||
@@ -21,8 +26,6 @@ export interface EngineConfig {
|
||||
minAbsDelta: number; // e.g. 80
|
||||
minPct: number; // e.g. 0.15
|
||||
};
|
||||
enableExplain: boolean;
|
||||
explainTopNPerProjectPerDay: number; // e.g. 3
|
||||
}
|
||||
|
||||
/** Simple gating to cut noise; modules can override via thresholds. */
|
||||
@@ -53,64 +56,84 @@ function chunk<T>(arr: T[], size: number): T[][] {
|
||||
return out;
|
||||
}
|
||||
|
||||
function sha256(x: string) {
|
||||
return crypto.createHash('sha256').update(x).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Engine entrypoint: runs all projects for a cadence.
|
||||
* Recommended: call this from a per-project worker (fanout), but it can also run directly.
|
||||
*/
|
||||
export function createEngine(args: {
|
||||
store: InsightStore;
|
||||
modules: InsightModule[];
|
||||
db: any;
|
||||
logger?: Pick<Console, 'info' | 'warn' | 'error'>;
|
||||
explainQueue?: ExplainQueue;
|
||||
config: EngineConfig;
|
||||
}) {
|
||||
const { store, modules, db, explainQueue, config } = args;
|
||||
const { store, modules, db, config } = args;
|
||||
const logger = args.logger ?? console;
|
||||
|
||||
async function runCadence(cadence: Cadence, now: Date): Promise<void> {
|
||||
const projectIds = await store.listProjectIdsForCadence(cadence);
|
||||
for (const projectId of projectIds) {
|
||||
await runProject({ projectId, cadence, now });
|
||||
}
|
||||
function isProjectOldEnoughForWindow(
|
||||
projectCreatedAt: Date | null | undefined,
|
||||
baselineStart: Date,
|
||||
): boolean {
|
||||
if (!projectCreatedAt) return true; // best-effort; don't block if unknown
|
||||
return projectCreatedAt.getTime() <= baselineStart.getTime();
|
||||
}
|
||||
|
||||
async function runProject(opts: {
|
||||
projectId: string;
|
||||
cadence: Cadence;
|
||||
now: Date;
|
||||
projectCreatedAt?: Date | null;
|
||||
}): Promise<void> {
|
||||
const { projectId, cadence, now } = opts;
|
||||
const { projectId, cadence, now, projectCreatedAt } = opts;
|
||||
const projLogger = logger;
|
||||
const eligible = modules.filter((m) => m.cadence.includes(cadence));
|
||||
|
||||
// Track top insights (by impact) for optional explain step across all modules/windows
|
||||
const explainCandidates: Array<{
|
||||
insightId: string;
|
||||
impact: number;
|
||||
evidence: any;
|
||||
evidenceHash: string;
|
||||
}> = [];
|
||||
|
||||
for (const mod of eligible) {
|
||||
for (const windowKind of mod.windows) {
|
||||
const window = resolveWindow(windowKind as WindowKind, now);
|
||||
const ctx: ComputeContext = {
|
||||
projectId,
|
||||
window,
|
||||
db,
|
||||
now,
|
||||
logger: projLogger,
|
||||
};
|
||||
const windows = mod.windows ?? DEFAULT_WINDOWS;
|
||||
for (const windowKind of windows) {
|
||||
let window: ReturnType<typeof resolveWindow>;
|
||||
let ctx: ComputeContext;
|
||||
try {
|
||||
window = resolveWindow(windowKind, now);
|
||||
if (
|
||||
!isProjectOldEnoughForWindow(projectCreatedAt, window.baselineStart)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
// Initialize cache for this module+window combination.
|
||||
// Cache is automatically garbage collected when context goes out of scope.
|
||||
const cache = new Map<string, any>();
|
||||
ctx = {
|
||||
projectId,
|
||||
window,
|
||||
db,
|
||||
now,
|
||||
logger: projLogger,
|
||||
clix: createCachedClix(db, cache),
|
||||
};
|
||||
} catch (e) {
|
||||
projLogger.error('[insights] failed to create compute context', {
|
||||
projectId,
|
||||
module: mod.key,
|
||||
windowKind,
|
||||
err: e,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// 1) enumerate dimensions
|
||||
let dims = mod.enumerateDimensions
|
||||
? await mod.enumerateDimensions(ctx)
|
||||
: [];
|
||||
let dims: string[] = [];
|
||||
try {
|
||||
dims = mod.enumerateDimensions
|
||||
? await mod.enumerateDimensions(ctx)
|
||||
: [];
|
||||
} catch (e) {
|
||||
// Important: enumeration failures should not abort the whole project run.
|
||||
// Also avoid lifecycle close/suppression when we didn't actually evaluate dims.
|
||||
projLogger.error('[insights] module enumerateDimensions failed', {
|
||||
projectId,
|
||||
module: mod.key,
|
||||
windowKind,
|
||||
err: e,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const maxDims = mod.thresholds?.maxDims ?? 25;
|
||||
if (dims.length > maxDims) dims = dims.slice(0, maxDims);
|
||||
|
||||
@@ -190,9 +213,6 @@ export function createEngine(args: {
|
||||
window,
|
||||
card,
|
||||
metrics: {
|
||||
currentValue: r.currentValue,
|
||||
compareValue: r.compareValue,
|
||||
changePct: r.changePct,
|
||||
direction: r.direction,
|
||||
impactScore: impact,
|
||||
severityBand: sev,
|
||||
@@ -241,7 +261,6 @@ export function createEngine(args: {
|
||||
windowKind,
|
||||
eventKind,
|
||||
changeFrom: {
|
||||
changePct: prev.changePct,
|
||||
direction: prev.direction,
|
||||
impactScore: prev.impactScore,
|
||||
severityBand: prev.severityBand,
|
||||
@@ -255,48 +274,6 @@ export function createEngine(args: {
|
||||
now,
|
||||
});
|
||||
}
|
||||
|
||||
// 9) optional AI explain candidates (only for top-impact insights)
|
||||
if (config.enableExplain && explainQueue && mod.drivers) {
|
||||
// compute evidence deterministically (drivers)
|
||||
try {
|
||||
const drivers = await mod.drivers(r, ctx);
|
||||
const evidence = {
|
||||
insight: {
|
||||
moduleKey: mod.key,
|
||||
dimensionKey: r.dimensionKey,
|
||||
windowKind,
|
||||
currentValue: r.currentValue,
|
||||
compareValue: r.compareValue,
|
||||
changePct: r.changePct,
|
||||
direction: r.direction,
|
||||
},
|
||||
drivers,
|
||||
window: {
|
||||
start: window.start.toISOString().slice(0, 10),
|
||||
end: window.end.toISOString().slice(0, 10),
|
||||
baselineStart: window.baselineStart
|
||||
.toISOString()
|
||||
.slice(0, 10),
|
||||
baselineEnd: window.baselineEnd.toISOString().slice(0, 10),
|
||||
},
|
||||
};
|
||||
const evidenceHash = sha256(JSON.stringify(evidence));
|
||||
explainCandidates.push({
|
||||
insightId: persisted.id,
|
||||
impact,
|
||||
evidence,
|
||||
evidenceHash,
|
||||
});
|
||||
} catch (e) {
|
||||
projLogger.warn('[insights] drivers() failed', {
|
||||
projectId,
|
||||
module: mod.key,
|
||||
dimensionKey: r.dimensionKey,
|
||||
err: e,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -320,27 +297,7 @@ export function createEngine(args: {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 12) enqueue explains for top insights across the whole project run
|
||||
if (config.enableExplain && explainQueue) {
|
||||
explainCandidates.sort((a, b) => b.impact - a.impact);
|
||||
const top = explainCandidates.slice(
|
||||
0,
|
||||
config.explainTopNPerProjectPerDay,
|
||||
);
|
||||
for (const c of top) {
|
||||
await explainQueue.enqueueExplain({
|
||||
insightId: c.insightId,
|
||||
projectId,
|
||||
moduleKey: 'n/a', // optional; you can include it in evidence instead
|
||||
dimensionKey: 'n/a',
|
||||
windowKind: 'yesterday',
|
||||
evidence: c.evidence,
|
||||
evidenceHash: c.evidenceHash,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { runCadence, runProject };
|
||||
return { runProject };
|
||||
}
|
||||
|
||||
@@ -4,6 +4,5 @@ export * from './scoring';
|
||||
export * from './material';
|
||||
export * from './engine';
|
||||
export * from './store';
|
||||
export * from './normalize';
|
||||
export * from './utils';
|
||||
export * from './modules';
|
||||
|
||||
@@ -1,77 +1,42 @@
|
||||
import { TABLE_NAMES } from '../../../clickhouse/client';
|
||||
import { clix } from '../../../clickhouse/query-builder';
|
||||
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
|
||||
import { computeWeekdayMedians, getEndOfDay, getWeekday } from '../utils';
|
||||
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
|
||||
import type {
|
||||
ComputeContext,
|
||||
ComputeResult,
|
||||
InsightModule,
|
||||
RenderedCard,
|
||||
} from '../types';
|
||||
import {
|
||||
buildLookupMap,
|
||||
computeChangePct,
|
||||
computeDirection,
|
||||
computeMedian,
|
||||
getEndOfDay,
|
||||
getWeekday,
|
||||
selectTopDimensions,
|
||||
} from '../utils';
|
||||
|
||||
function normalizeDevice(device: string): string {
|
||||
const d = (device || '').toLowerCase().trim();
|
||||
if (d.includes('mobile') || d === 'phone') return 'mobile';
|
||||
if (d.includes('tablet')) return 'tablet';
|
||||
if (d.includes('desktop')) return 'desktop';
|
||||
return d || 'unknown';
|
||||
}
|
||||
|
||||
export const devicesModule: InsightModule = {
|
||||
key: 'devices',
|
||||
cadence: ['daily'],
|
||||
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 5 },
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
// Query devices from current window (limited set, no need for baseline merge)
|
||||
const results = await clix(ctx.db)
|
||||
.select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.where('device', '!=', '')
|
||||
.groupBy(['device'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.execute();
|
||||
|
||||
// Normalize and dedupe device types
|
||||
const dims = new Set<string>();
|
||||
for (const r of results) {
|
||||
dims.add(`device:${normalizeDevice(r.device)}`);
|
||||
}
|
||||
|
||||
return Array.from(dims);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
// Single query for ALL current values
|
||||
const currentResults = await clix(ctx.db)
|
||||
.select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['device'])
|
||||
.execute();
|
||||
|
||||
// Build current lookup map (normalized) and total
|
||||
const currentMap = new Map<string, number>();
|
||||
let totalCurrentValue = 0;
|
||||
for (const r of currentResults) {
|
||||
const key = normalizeDevice(r.device);
|
||||
const cnt = Number(r.cnt ?? 0);
|
||||
currentMap.set(key, (currentMap.get(key) ?? 0) + cnt);
|
||||
totalCurrentValue += cnt;
|
||||
}
|
||||
|
||||
// Single query for baseline
|
||||
let baselineMap: Map<string, number>;
|
||||
let totalBaselineValue = 0;
|
||||
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const baselineResults = await clix(ctx.db)
|
||||
async function fetchDeviceAggregates(ctx: ComputeContext): Promise<{
|
||||
currentMap: Map<string, number>;
|
||||
baselineMap: Map<string, number>;
|
||||
totalCurrent: number;
|
||||
totalBaseline: number;
|
||||
}> {
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const [currentResults, baselineResults, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['device'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ date: string; device: string; cnt: number }>([
|
||||
'toDate(created_at) as date',
|
||||
'device',
|
||||
@@ -85,77 +50,144 @@ export const devicesModule: InsightModule = {
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['date', 'device'])
|
||||
.execute();
|
||||
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
|
||||
// Group by normalized device type before computing medians
|
||||
const normalizedResults = baselineResults.map((r) => ({
|
||||
date: r.date,
|
||||
device: normalizeDevice(r.device),
|
||||
cnt: r.cnt,
|
||||
}));
|
||||
|
||||
// Aggregate by date + normalized device first
|
||||
const aggregated = new Map<string, { date: string; cnt: number }[]>();
|
||||
for (const r of normalizedResults) {
|
||||
const key = `${r.date}|${r.device}`;
|
||||
if (!aggregated.has(r.device)) {
|
||||
aggregated.set(r.device, []);
|
||||
}
|
||||
// Find existing entry for this date+device or add new
|
||||
const entries = aggregated.get(r.device)!;
|
||||
const existing = entries.find((e) => e.date === r.date);
|
||||
if (existing) {
|
||||
existing.cnt += Number(r.cnt ?? 0);
|
||||
} else {
|
||||
entries.push({ date: r.date, cnt: Number(r.cnt ?? 0) });
|
||||
}
|
||||
}
|
||||
|
||||
// Compute weekday medians per device type
|
||||
baselineMap = new Map<string, number>();
|
||||
for (const [deviceType, entries] of aggregated) {
|
||||
const sameWeekdayValues = entries
|
||||
.filter((e) => getWeekday(new Date(e.date)) === targetWeekday)
|
||||
.map((e) => e.cnt)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
if (sameWeekdayValues.length > 0) {
|
||||
const mid = Math.floor(sameWeekdayValues.length / 2);
|
||||
const median =
|
||||
sameWeekdayValues.length % 2 === 0
|
||||
? ((sameWeekdayValues[mid - 1] ?? 0) +
|
||||
(sameWeekdayValues[mid] ?? 0)) /
|
||||
2
|
||||
: (sameWeekdayValues[mid] ?? 0);
|
||||
baselineMap.set(deviceType, median);
|
||||
totalBaselineValue += median;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const baselineResults = await clix(ctx.db)
|
||||
.select<{ device: string; cnt: number }>(['device', 'count(*) as cnt'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['device'])
|
||||
.execute();
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
baselineMap = new Map<string, number>();
|
||||
for (const r of baselineResults) {
|
||||
const key = normalizeDevice(r.device);
|
||||
const cnt = Number(r.cnt ?? 0);
|
||||
baselineMap.set(key, (baselineMap.get(key) ?? 0) + cnt);
|
||||
totalBaselineValue += cnt;
|
||||
const currentMap = buildLookupMap(currentResults, (r) => r.device);
|
||||
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
const aggregated = new Map<string, { date: string; cnt: number }[]>();
|
||||
for (const r of baselineResults) {
|
||||
if (!aggregated.has(r.device)) {
|
||||
aggregated.set(r.device, []);
|
||||
}
|
||||
const entries = aggregated.get(r.device)!;
|
||||
const existing = entries.find((e) => e.date === r.date);
|
||||
if (existing) {
|
||||
existing.cnt += Number(r.cnt ?? 0);
|
||||
} else {
|
||||
entries.push({ date: r.date, cnt: Number(r.cnt ?? 0) });
|
||||
}
|
||||
}
|
||||
|
||||
// Build results from maps
|
||||
const baselineMap = new Map<string, number>();
|
||||
for (const [deviceType, entries] of aggregated) {
|
||||
const sameWeekdayValues = entries
|
||||
.filter((e) => getWeekday(new Date(e.date)) === targetWeekday)
|
||||
.map((e) => e.cnt)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
if (sameWeekdayValues.length > 0) {
|
||||
baselineMap.set(deviceType, computeMedian(sameWeekdayValues));
|
||||
}
|
||||
}
|
||||
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline =
|
||||
baselineMap.size > 0
|
||||
? Array.from(baselineMap.values()).reduce((sum, val) => sum + val, 0)
|
||||
: 0;
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
const curStart = formatClickhouseDate(ctx.window.start);
|
||||
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
|
||||
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
|
||||
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
|
||||
|
||||
const [results, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ device: string; cur: number; base: number }>([
|
||||
'device',
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['device'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number; base_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
const currentMap = buildLookupMap(
|
||||
results,
|
||||
(r) => r.device,
|
||||
(r) => Number(r.cur ?? 0),
|
||||
);
|
||||
|
||||
const baselineMap = buildLookupMap(
|
||||
results,
|
||||
(r) => r.device,
|
||||
(r) => Number(r.base ?? 0),
|
||||
);
|
||||
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = totals[0]?.base_total ?? 0;
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
export const devicesModule: InsightModule = {
|
||||
key: 'devices',
|
||||
cadence: ['daily'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 5 },
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
const { currentMap, baselineMap } = await fetchDeviceAggregates(ctx);
|
||||
const topDims = selectTopDimensions(
|
||||
currentMap,
|
||||
baselineMap,
|
||||
this.thresholds?.maxDims ?? 5,
|
||||
);
|
||||
return topDims.map((dim) => `device:${dim}`);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
|
||||
await fetchDeviceAggregates(ctx);
|
||||
const results: ComputeResult[] = [];
|
||||
|
||||
for (const dimKey of dimensionKeys) {
|
||||
@@ -165,23 +197,12 @@ export const devicesModule: InsightModule = {
|
||||
const currentValue = currentMap.get(deviceType) ?? 0;
|
||||
const compareValue = baselineMap.get(deviceType) ?? 0;
|
||||
|
||||
const currentShare =
|
||||
totalCurrentValue > 0 ? currentValue / totalCurrentValue : 0;
|
||||
const compareShare =
|
||||
totalBaselineValue > 0 ? compareValue / totalBaselineValue : 0;
|
||||
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
|
||||
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
|
||||
|
||||
// Share shift in percentage points
|
||||
const shareShiftPp = (currentShare - compareShare) * 100;
|
||||
const changePct =
|
||||
compareShare > 0
|
||||
? (currentShare - compareShare) / compareShare
|
||||
: currentShare > 0
|
||||
? 1
|
||||
: 0;
|
||||
|
||||
// Direction should match the sign of the pp shift (so title + delta agree)
|
||||
const direction: 'up' | 'down' | 'flat' =
|
||||
shareShiftPp > 0 ? 'up' : shareShiftPp < 0 ? 'down' : 'flat';
|
||||
const changePct = computeChangePct(currentValue, compareValue);
|
||||
const direction = computeDirection(changePct);
|
||||
|
||||
results.push({
|
||||
ok: true,
|
||||
@@ -203,20 +224,51 @@ export const devicesModule: InsightModule = {
|
||||
|
||||
render(result, ctx): RenderedCard {
|
||||
const device = result.dimensionKey.replace('device:', '');
|
||||
const shareShiftPp = (result.extra?.shareShiftPp as number) ?? 0;
|
||||
const isIncrease = shareShiftPp >= 0;
|
||||
const changePct = result.changePct ?? 0;
|
||||
const isIncrease = changePct >= 0;
|
||||
|
||||
const sessionsCurrent = result.currentValue ?? 0;
|
||||
const sessionsCompare = result.compareValue ?? 0;
|
||||
const shareCurrent = Number(result.extra?.currentShare ?? 0);
|
||||
const shareCompare = Number(result.extra?.compareShare ?? 0);
|
||||
|
||||
return {
|
||||
kind: 'insight_v1',
|
||||
title: `${device} ${isIncrease ? '↑' : '↓'} ${Math.abs(shareShiftPp).toFixed(1)}pp`,
|
||||
summary: `${ctx.window.label}. Device share shift.`,
|
||||
primaryDimension: { type: 'device', key: device, displayName: device },
|
||||
tags: ['devices', ctx.window.kind, isIncrease ? 'increase' : 'decrease'],
|
||||
metric: 'share',
|
||||
extra: {
|
||||
currentShare: result.extra?.currentShare,
|
||||
compareShare: result.extra?.compareShare,
|
||||
shareShiftPp: result.extra?.shareShiftPp,
|
||||
title: `${device} ${isIncrease ? '↑' : '↓'} ${Math.abs(changePct * 100).toFixed(0)}%`,
|
||||
summary: `${ctx.window.label}. Device traffic change.`,
|
||||
displayName: device,
|
||||
payload: {
|
||||
kind: 'insight_v1',
|
||||
dimensions: [{ key: 'device', value: device, displayName: device }],
|
||||
primaryMetric: 'sessions',
|
||||
metrics: {
|
||||
sessions: {
|
||||
current: sessionsCurrent,
|
||||
compare: sessionsCompare,
|
||||
delta: sessionsCurrent - sessionsCompare,
|
||||
changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
|
||||
direction: result.direction ?? 'flat',
|
||||
unit: 'count',
|
||||
},
|
||||
share: {
|
||||
current: shareCurrent,
|
||||
compare: shareCompare,
|
||||
delta: shareCurrent - shareCompare,
|
||||
changePct:
|
||||
shareCompare > 0
|
||||
? (shareCurrent - shareCompare) / shareCompare
|
||||
: null,
|
||||
direction:
|
||||
shareCurrent - shareCompare > 0.0005
|
||||
? 'up'
|
||||
: shareCurrent - shareCompare < -0.0005
|
||||
? 'down'
|
||||
: 'flat',
|
||||
unit: 'ratio',
|
||||
},
|
||||
},
|
||||
extra: {
|
||||
// keep module-specific flags/fields if needed later
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
@@ -1,26 +1,34 @@
|
||||
import { TABLE_NAMES } from '../../../clickhouse/client';
|
||||
import { clix } from '../../../clickhouse/query-builder';
|
||||
import { normalizePath } from '../normalize';
|
||||
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
|
||||
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
|
||||
import type {
|
||||
ComputeContext,
|
||||
ComputeResult,
|
||||
InsightModule,
|
||||
RenderedCard,
|
||||
} from '../types';
|
||||
import {
|
||||
buildLookupMap,
|
||||
computeChangePct,
|
||||
computeDirection,
|
||||
computeWeekdayMedians,
|
||||
getEndOfDay,
|
||||
getWeekday,
|
||||
selectTopDimensions,
|
||||
} from '../utils';
|
||||
|
||||
export const entryPagesModule: InsightModule = {
|
||||
key: 'entry-pages',
|
||||
cadence: ['daily'],
|
||||
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
|
||||
const DELIMITER = '|||';
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
// Query top entry pages from BOTH current and baseline windows
|
||||
const [currentResults, baselineResults] = await Promise.all([
|
||||
clix(ctx.db)
|
||||
.select<{ entry_path: string; cnt: number }>([
|
||||
async function fetchEntryPageAggregates(ctx: ComputeContext): Promise<{
|
||||
currentMap: Map<string, number>;
|
||||
baselineMap: Map<string, number>;
|
||||
totalCurrent: number;
|
||||
totalBaseline: number;
|
||||
}> {
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const [currentResults, baselineResults, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ entry_origin: string; entry_path: string; cnt: number }>([
|
||||
'entry_origin',
|
||||
'entry_path',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
@@ -31,12 +39,18 @@ export const entryPagesModule: InsightModule = {
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['entry_path'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.limit(this.thresholds?.maxDims ?? 100)
|
||||
.groupBy(['entry_origin', 'entry_path'])
|
||||
.execute(),
|
||||
clix(ctx.db)
|
||||
.select<{ entry_path: string; cnt: number }>([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{
|
||||
date: string;
|
||||
entry_origin: string;
|
||||
entry_path: string;
|
||||
cnt: number;
|
||||
}>([
|
||||
'toDate(created_at) as date',
|
||||
'entry_origin',
|
||||
'entry_path',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
@@ -47,104 +61,147 @@ export const entryPagesModule: InsightModule = {
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['entry_path'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.limit(this.thresholds?.maxDims ?? 100)
|
||||
.groupBy(['date', 'entry_origin', 'entry_path'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
// Merge both sets
|
||||
const dims = new Set<string>();
|
||||
for (const r of currentResults) {
|
||||
dims.add(`entry:${normalizePath(r.entry_path || '/')}`);
|
||||
}
|
||||
for (const r of baselineResults) {
|
||||
dims.add(`entry:${normalizePath(r.entry_path || '/')}`);
|
||||
}
|
||||
const currentMap = buildLookupMap(
|
||||
currentResults,
|
||||
(r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
|
||||
);
|
||||
|
||||
return Array.from(dims);
|
||||
},
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
const baselineMap = computeWeekdayMedians(
|
||||
baselineResults,
|
||||
targetWeekday,
|
||||
(r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
|
||||
);
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
// Single query for ALL current values
|
||||
const currentResults = await clix(ctx.db)
|
||||
.select<{ entry_path: string; cnt: number }>([
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = Array.from(baselineMap.values()).reduce(
|
||||
(sum, val) => sum + val,
|
||||
0,
|
||||
);
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
const curStart = formatClickhouseDate(ctx.window.start);
|
||||
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
|
||||
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
|
||||
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
|
||||
|
||||
const [results, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{
|
||||
entry_origin: string;
|
||||
entry_path: string;
|
||||
cur: number;
|
||||
base: number;
|
||||
}>([
|
||||
'entry_origin',
|
||||
'entry_path',
|
||||
'count(*) as cnt',
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.start,
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['entry_path'])
|
||||
.execute();
|
||||
.groupBy(['entry_origin', 'entry_path'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number; base_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
// Build current lookup map
|
||||
const currentMap = new Map<string, number>();
|
||||
for (const r of currentResults) {
|
||||
const key = normalizePath(r.entry_path || '/');
|
||||
currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
|
||||
}
|
||||
const currentMap = buildLookupMap(
|
||||
results,
|
||||
(r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
|
||||
(r) => Number(r.cur ?? 0),
|
||||
);
|
||||
|
||||
// Single query for baseline
|
||||
let baselineMap: Map<string, number>;
|
||||
const baselineMap = buildLookupMap(
|
||||
results,
|
||||
(r) => `${r.entry_origin || ''}${DELIMITER}${r.entry_path || '/'}`,
|
||||
(r) => Number(r.base ?? 0),
|
||||
);
|
||||
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const baselineResults = await clix(ctx.db)
|
||||
.select<{ date: string; entry_path: string; cnt: number }>([
|
||||
'toDate(created_at) as date',
|
||||
'entry_path',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['date', 'entry_path'])
|
||||
.execute();
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = totals[0]?.base_total ?? 0;
|
||||
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
|
||||
normalizePath(r.entry_path || '/'),
|
||||
);
|
||||
} else {
|
||||
const baselineResults = await clix(ctx.db)
|
||||
.select<{ entry_path: string; cnt: number }>([
|
||||
'entry_path',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['entry_path'])
|
||||
.execute();
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
baselineMap = new Map<string, number>();
|
||||
for (const r of baselineResults) {
|
||||
const key = normalizePath(r.entry_path || '/');
|
||||
baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
|
||||
}
|
||||
}
|
||||
export const entryPagesModule: InsightModule = {
|
||||
key: 'entry-pages',
|
||||
cadence: ['daily'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
|
||||
|
||||
// Build results from maps
|
||||
async enumerateDimensions(ctx) {
|
||||
const { currentMap, baselineMap } = await fetchEntryPageAggregates(ctx);
|
||||
const topDims = selectTopDimensions(
|
||||
currentMap,
|
||||
baselineMap,
|
||||
this.thresholds?.maxDims ?? 100,
|
||||
);
|
||||
return topDims.map((dim) => `entry:${dim}`);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
|
||||
await fetchEntryPageAggregates(ctx);
|
||||
const results: ComputeResult[] = [];
|
||||
|
||||
for (const dimKey of dimensionKeys) {
|
||||
if (!dimKey.startsWith('entry:')) continue;
|
||||
const entryPath = dimKey.replace('entry:', '');
|
||||
const originPath = dimKey.replace('entry:', '');
|
||||
|
||||
const currentValue = currentMap.get(entryPath) ?? 0;
|
||||
const compareValue = baselineMap.get(entryPath) ?? 0;
|
||||
const currentValue = currentMap.get(originPath) ?? 0;
|
||||
const compareValue = baselineMap.get(originPath) ?? 0;
|
||||
|
||||
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
|
||||
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
|
||||
|
||||
const shareShiftPp = (currentShare - compareShare) * 100;
|
||||
const changePct = computeChangePct(currentValue, compareValue);
|
||||
const direction = computeDirection(changePct);
|
||||
|
||||
@@ -156,6 +213,9 @@ export const entryPagesModule: InsightModule = {
|
||||
changePct,
|
||||
direction,
|
||||
extra: {
|
||||
shareShiftPp,
|
||||
currentShare,
|
||||
compareShare,
|
||||
isNew: compareValue === 0 && currentValue > 0,
|
||||
},
|
||||
});
|
||||
@@ -165,28 +225,62 @@ export const entryPagesModule: InsightModule = {
|
||||
},
|
||||
|
||||
render(result, ctx): RenderedCard {
|
||||
const path = result.dimensionKey.replace('entry:', '');
|
||||
const originPath = result.dimensionKey.replace('entry:', '');
|
||||
const [origin, path] = originPath.split(DELIMITER);
|
||||
const displayValue = origin ? `${origin}${path}` : path || '/';
|
||||
const pct = ((result.changePct ?? 0) * 100).toFixed(1);
|
||||
const isIncrease = (result.changePct ?? 0) >= 0;
|
||||
const isNew = result.extra?.isNew as boolean | undefined;
|
||||
|
||||
const title = isNew
|
||||
? `New entry page: ${path}`
|
||||
: `Entry page ${path} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
|
||||
? `New entry page: ${displayValue}`
|
||||
: `Entry page ${displayValue} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
|
||||
|
||||
const sessionsCurrent = result.currentValue ?? 0;
|
||||
const sessionsCompare = result.compareValue ?? 0;
|
||||
const shareCurrent = Number(result.extra?.currentShare ?? 0);
|
||||
const shareCompare = Number(result.extra?.compareShare ?? 0);
|
||||
|
||||
return {
|
||||
kind: 'insight_v1',
|
||||
title,
|
||||
summary: `${ctx.window.label}. Sessions ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
|
||||
primaryDimension: { type: 'entry', key: path, displayName: path },
|
||||
tags: [
|
||||
'entry-pages',
|
||||
ctx.window.kind,
|
||||
isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
|
||||
],
|
||||
metric: 'sessions',
|
||||
extra: {
|
||||
isNew: result.extra?.isNew,
|
||||
summary: `${ctx.window.label}. Sessions ${sessionsCurrent} vs ${sessionsCompare}.`,
|
||||
displayName: displayValue,
|
||||
payload: {
|
||||
kind: 'insight_v1',
|
||||
dimensions: [
|
||||
{ key: 'origin', value: origin ?? '', displayName: origin ?? '' },
|
||||
{ key: 'path', value: path ?? '', displayName: path ?? '' },
|
||||
],
|
||||
primaryMetric: 'sessions',
|
||||
metrics: {
|
||||
sessions: {
|
||||
current: sessionsCurrent,
|
||||
compare: sessionsCompare,
|
||||
delta: sessionsCurrent - sessionsCompare,
|
||||
changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
|
||||
direction: result.direction ?? 'flat',
|
||||
unit: 'count',
|
||||
},
|
||||
share: {
|
||||
current: shareCurrent,
|
||||
compare: shareCompare,
|
||||
delta: shareCurrent - shareCompare,
|
||||
changePct:
|
||||
shareCompare > 0
|
||||
? (shareCurrent - shareCompare) / shareCompare
|
||||
: null,
|
||||
direction:
|
||||
shareCurrent - shareCompare > 0.0005
|
||||
? 'up'
|
||||
: shareCurrent - shareCompare < -0.0005
|
||||
? 'down'
|
||||
: 'flat',
|
||||
unit: 'ratio',
|
||||
},
|
||||
},
|
||||
extra: {
|
||||
isNew: result.extra?.isNew,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
@@ -1,18 +1,31 @@
|
||||
import { TABLE_NAMES } from '../../../clickhouse/client';
|
||||
import { clix } from '../../../clickhouse/query-builder';
|
||||
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
|
||||
import { computeWeekdayMedians, getEndOfDay, getWeekday } from '../utils';
|
||||
import { getCountry } from '@openpanel/constants';
|
||||
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
|
||||
import type {
|
||||
ComputeContext,
|
||||
ComputeResult,
|
||||
InsightModule,
|
||||
RenderedCard,
|
||||
} from '../types';
|
||||
import {
|
||||
buildLookupMap,
|
||||
computeChangePct,
|
||||
computeDirection,
|
||||
computeWeekdayMedians,
|
||||
getEndOfDay,
|
||||
getWeekday,
|
||||
selectTopDimensions,
|
||||
} from '../utils';
|
||||
|
||||
export const geoModule: InsightModule = {
|
||||
key: 'geo',
|
||||
cadence: ['daily'],
|
||||
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 30 },
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
// Query top countries from BOTH current and baseline windows
|
||||
const [currentResults, baselineResults] = await Promise.all([
|
||||
clix(ctx.db)
|
||||
async function fetchGeoAggregates(ctx: ComputeContext): Promise<{
|
||||
currentMap: Map<string, number>;
|
||||
baselineMap: Map<string, number>;
|
||||
totalCurrent: number;
|
||||
totalBaseline: number;
|
||||
}> {
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const [currentResults, baselineResults, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ country: string; cnt: number }>([
|
||||
'country',
|
||||
'count(*) as cnt',
|
||||
@@ -24,72 +37,10 @@ export const geoModule: InsightModule = {
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.where('country', '!=', '')
|
||||
.groupBy(['country'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.limit(this.thresholds?.maxDims ?? 30)
|
||||
.execute(),
|
||||
clix(ctx.db)
|
||||
.select<{ country: string; cnt: number }>([
|
||||
'country',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.where('country', '!=', '')
|
||||
.groupBy(['country'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.limit(this.thresholds?.maxDims ?? 30)
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
// Merge both sets
|
||||
const dims = new Set<string>();
|
||||
for (const r of currentResults) {
|
||||
dims.add(`country:${r.country || 'unknown'}`);
|
||||
}
|
||||
for (const r of baselineResults) {
|
||||
dims.add(`country:${r.country || 'unknown'}`);
|
||||
}
|
||||
|
||||
return Array.from(dims);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
// Single query for ALL current values + total
|
||||
const currentResults = await clix(ctx.db)
|
||||
.select<{ country: string; cnt: number }>(['country', 'count(*) as cnt'])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['country'])
|
||||
.execute();
|
||||
|
||||
// Build current lookup map and total
|
||||
const currentMap = new Map<string, number>();
|
||||
let totalCurrentValue = 0;
|
||||
for (const r of currentResults) {
|
||||
const key = r.country || 'unknown';
|
||||
const cnt = Number(r.cnt ?? 0);
|
||||
currentMap.set(key, (currentMap.get(key) ?? 0) + cnt);
|
||||
totalCurrentValue += cnt;
|
||||
}
|
||||
|
||||
// Single query for baseline
|
||||
let baselineMap: Map<string, number>;
|
||||
let totalBaselineValue = 0;
|
||||
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const baselineResults = await clix(ctx.db)
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ date: string; country: string; cnt: number }>([
|
||||
'toDate(created_at) as date',
|
||||
'country',
|
||||
@@ -103,45 +54,127 @@ export const geoModule: InsightModule = {
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['date', 'country'])
|
||||
.execute();
|
||||
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
baselineMap = computeWeekdayMedians(
|
||||
baselineResults,
|
||||
targetWeekday,
|
||||
(r) => r.country || 'unknown',
|
||||
);
|
||||
|
||||
// Compute total baseline from medians
|
||||
for (const value of baselineMap.values()) {
|
||||
totalBaselineValue += value;
|
||||
}
|
||||
} else {
|
||||
const baselineResults = await clix(ctx.db)
|
||||
.select<{ country: string; cnt: number }>([
|
||||
'country',
|
||||
'count(*) as cnt',
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['country'])
|
||||
.execute();
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
baselineMap = new Map<string, number>();
|
||||
for (const r of baselineResults) {
|
||||
const key = r.country || 'unknown';
|
||||
const cnt = Number(r.cnt ?? 0);
|
||||
baselineMap.set(key, (baselineMap.get(key) ?? 0) + cnt);
|
||||
totalBaselineValue += cnt;
|
||||
}
|
||||
}
|
||||
const currentMap = buildLookupMap(
|
||||
currentResults,
|
||||
(r) => r.country || 'unknown',
|
||||
);
|
||||
|
||||
// Build results from maps
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
const baselineMap = computeWeekdayMedians(
|
||||
baselineResults,
|
||||
targetWeekday,
|
||||
(r) => r.country || 'unknown',
|
||||
);
|
||||
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = Array.from(baselineMap.values()).reduce(
|
||||
(sum, val) => sum + val,
|
||||
0,
|
||||
);
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
const curStart = formatClickhouseDate(ctx.window.start);
|
||||
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
|
||||
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
|
||||
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
|
||||
|
||||
const [results, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ country: string; cur: number; base: number }>([
|
||||
'country',
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['country'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number; base_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
const currentMap = buildLookupMap(
|
||||
results,
|
||||
(r) => r.country || 'unknown',
|
||||
(r) => Number(r.cur ?? 0),
|
||||
);
|
||||
|
||||
const baselineMap = buildLookupMap(
|
||||
results,
|
||||
(r) => r.country || 'unknown',
|
||||
(r) => Number(r.base ?? 0),
|
||||
);
|
||||
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = totals[0]?.base_total ?? 0;
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
export const geoModule: InsightModule = {
|
||||
key: 'geo',
|
||||
cadence: ['daily'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 0, minPct: 0.08, maxDims: 30 },
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
const { currentMap, baselineMap } = await fetchGeoAggregates(ctx);
|
||||
const topDims = selectTopDimensions(
|
||||
currentMap,
|
||||
baselineMap,
|
||||
this.thresholds?.maxDims ?? 30,
|
||||
);
|
||||
return topDims.map((dim) => `country:${dim}`);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
|
||||
await fetchGeoAggregates(ctx);
|
||||
const results: ComputeResult[] = [];
|
||||
|
||||
for (const dimKey of dimensionKeys) {
|
||||
@@ -151,23 +184,12 @@ export const geoModule: InsightModule = {
|
||||
const currentValue = currentMap.get(country) ?? 0;
|
||||
const compareValue = baselineMap.get(country) ?? 0;
|
||||
|
||||
const currentShare =
|
||||
totalCurrentValue > 0 ? currentValue / totalCurrentValue : 0;
|
||||
const compareShare =
|
||||
totalBaselineValue > 0 ? compareValue / totalBaselineValue : 0;
|
||||
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
|
||||
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
|
||||
|
||||
// Share shift in percentage points
|
||||
const shareShiftPp = (currentShare - compareShare) * 100;
|
||||
const changePct =
|
||||
compareShare > 0
|
||||
? (currentShare - compareShare) / compareShare
|
||||
: currentShare > 0
|
||||
? 1
|
||||
: 0;
|
||||
|
||||
// Direction should match the sign of the pp shift (so title + delta agree)
|
||||
const direction: 'up' | 'down' | 'flat' =
|
||||
shareShiftPp > 0 ? 'up' : shareShiftPp < 0 ? 'down' : 'flat';
|
||||
const changePct = computeChangePct(currentValue, compareValue);
|
||||
const direction = computeDirection(changePct);
|
||||
|
||||
results.push({
|
||||
ok: true,
|
||||
@@ -190,30 +212,59 @@ export const geoModule: InsightModule = {
|
||||
|
||||
render(result, ctx): RenderedCard {
|
||||
const country = result.dimensionKey.replace('country:', '');
|
||||
const shareShiftPp = (result.extra?.shareShiftPp as number) ?? 0;
|
||||
const isIncrease = shareShiftPp >= 0;
|
||||
const changePct = result.changePct ?? 0;
|
||||
const isIncrease = changePct >= 0;
|
||||
const isNew = result.extra?.isNew as boolean | undefined;
|
||||
const displayName = getCountry(country);
|
||||
|
||||
const title = isNew
|
||||
? `New traffic from: ${country}`
|
||||
: `${country} ${isIncrease ? '↑' : '↓'} ${Math.abs(shareShiftPp).toFixed(1)}pp`;
|
||||
? `New traffic from: ${displayName}`
|
||||
: `${displayName} ${isIncrease ? '↑' : '↓'} ${Math.abs(changePct * 100).toFixed(0)}%`;
|
||||
|
||||
const sessionsCurrent = result.currentValue ?? 0;
|
||||
const sessionsCompare = result.compareValue ?? 0;
|
||||
const shareCurrent = Number(result.extra?.currentShare ?? 0);
|
||||
const shareCompare = Number(result.extra?.compareShare ?? 0);
|
||||
|
||||
return {
|
||||
kind: 'insight_v1',
|
||||
title,
|
||||
summary: `${ctx.window.label}. Share shift from ${country}.`,
|
||||
primaryDimension: { type: 'country', key: country, displayName: country },
|
||||
tags: [
|
||||
'geo',
|
||||
ctx.window.kind,
|
||||
isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
|
||||
],
|
||||
metric: 'share',
|
||||
extra: {
|
||||
currentShare: result.extra?.currentShare,
|
||||
compareShare: result.extra?.compareShare,
|
||||
shareShiftPp: result.extra?.shareShiftPp,
|
||||
isNew: result.extra?.isNew,
|
||||
summary: `${ctx.window.label}. Traffic change from ${displayName}.`,
|
||||
displayName,
|
||||
payload: {
|
||||
kind: 'insight_v1',
|
||||
dimensions: [
|
||||
{ key: 'country', value: country, displayName: displayName },
|
||||
],
|
||||
primaryMetric: 'sessions',
|
||||
metrics: {
|
||||
sessions: {
|
||||
current: sessionsCurrent,
|
||||
compare: sessionsCompare,
|
||||
delta: sessionsCurrent - sessionsCompare,
|
||||
changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
|
||||
direction: result.direction ?? 'flat',
|
||||
unit: 'count',
|
||||
},
|
||||
share: {
|
||||
current: shareCurrent,
|
||||
compare: shareCompare,
|
||||
delta: shareCurrent - shareCompare,
|
||||
changePct:
|
||||
shareCompare > 0
|
||||
? (shareCurrent - shareCompare) / shareCompare
|
||||
: null,
|
||||
direction:
|
||||
shareCurrent - shareCompare > 0.0005
|
||||
? 'up'
|
||||
: shareCurrent - shareCompare < -0.0005
|
||||
? 'down'
|
||||
: 'flat',
|
||||
unit: 'ratio',
|
||||
},
|
||||
},
|
||||
extra: {
|
||||
isNew: result.extra?.isNew,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
@@ -1,26 +1,37 @@
|
||||
import { TABLE_NAMES } from '../../../clickhouse/client';
|
||||
import { clix } from '../../../clickhouse/query-builder';
|
||||
import { normalizePath } from '../normalize';
|
||||
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
|
||||
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
|
||||
import type {
|
||||
ComputeContext,
|
||||
ComputeResult,
|
||||
InsightModule,
|
||||
RenderedCard,
|
||||
} from '../types';
|
||||
import {
|
||||
buildLookupMap,
|
||||
computeChangePct,
|
||||
computeDirection,
|
||||
computeWeekdayMedians,
|
||||
getEndOfDay,
|
||||
getWeekday,
|
||||
selectTopDimensions,
|
||||
} from '../utils';
|
||||
|
||||
export const pageTrendsModule: InsightModule = {
|
||||
key: 'page-trends',
|
||||
cadence: ['daily'],
|
||||
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
|
||||
const DELIMITER = '|||';
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
// Query top pages from BOTH current and baseline windows
|
||||
const [currentResults, baselineResults] = await Promise.all([
|
||||
clix(ctx.db)
|
||||
.select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
|
||||
async function fetchPageTrendAggregates(ctx: ComputeContext): Promise<{
|
||||
currentMap: Map<string, number>;
|
||||
baselineMap: Map<string, number>;
|
||||
totalCurrent: number;
|
||||
totalBaseline: number;
|
||||
}> {
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const [currentResults, baselineResults, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ origin: string; path: string; cnt: number }>([
|
||||
'origin',
|
||||
'path',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
.from(TABLE_NAMES.events)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('name', '=', 'screen_view')
|
||||
@@ -28,65 +39,13 @@ export const pageTrendsModule: InsightModule = {
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['path'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.limit(this.thresholds?.maxDims ?? 100)
|
||||
.groupBy(['origin', 'path'])
|
||||
.execute(),
|
||||
clix(ctx.db)
|
||||
.select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
|
||||
.from(TABLE_NAMES.events)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('name', '=', 'screen_view')
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['path'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.limit(this.thresholds?.maxDims ?? 100)
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
// Merge both sets
|
||||
const dims = new Set<string>();
|
||||
for (const r of currentResults) {
|
||||
dims.add(`page:${normalizePath(r.path || '/')}`);
|
||||
}
|
||||
for (const r of baselineResults) {
|
||||
dims.add(`page:${normalizePath(r.path || '/')}`);
|
||||
}
|
||||
|
||||
return Array.from(dims);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
// Single query for ALL current values
|
||||
const currentResults = await clix(ctx.db)
|
||||
.select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
|
||||
.from(TABLE_NAMES.events)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('name', '=', 'screen_view')
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['path'])
|
||||
.execute();
|
||||
|
||||
// Build current lookup map
|
||||
const currentMap = new Map<string, number>();
|
||||
for (const r of currentResults) {
|
||||
const key = normalizePath(r.path || '/');
|
||||
currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
|
||||
}
|
||||
|
||||
// Single query for baseline
|
||||
let baselineMap: Map<string, number>;
|
||||
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const baselineResults = await clix(ctx.db)
|
||||
.select<{ date: string; path: string; cnt: number }>([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ date: string; origin: string; path: string; cnt: number }>([
|
||||
'toDate(created_at) as date',
|
||||
'origin',
|
||||
'path',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
@@ -97,42 +56,142 @@ export const pageTrendsModule: InsightModule = {
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['date', 'path'])
|
||||
.execute();
|
||||
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
|
||||
normalizePath(r.path || '/'),
|
||||
);
|
||||
} else {
|
||||
const baselineResults = await clix(ctx.db)
|
||||
.select<{ path: string; cnt: number }>(['path', 'count(*) as cnt'])
|
||||
.groupBy(['date', 'origin', 'path'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.events)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('name', '=', 'screen_view')
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['path'])
|
||||
.execute();
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
baselineMap = new Map<string, number>();
|
||||
for (const r of baselineResults) {
|
||||
const key = normalizePath(r.path || '/');
|
||||
baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
|
||||
}
|
||||
}
|
||||
const currentMap = buildLookupMap(
|
||||
currentResults,
|
||||
(r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
|
||||
);
|
||||
|
||||
// Build results from maps
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
const baselineMap = computeWeekdayMedians(
|
||||
baselineResults,
|
||||
targetWeekday,
|
||||
(r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
|
||||
);
|
||||
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = Array.from(baselineMap.values()).reduce(
|
||||
(sum, val) => sum + val,
|
||||
0,
|
||||
);
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
const curStart = formatClickhouseDate(ctx.window.start);
|
||||
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
|
||||
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
|
||||
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
|
||||
|
||||
const [results, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ origin: string; path: string; cur: number; base: number }>([
|
||||
'origin',
|
||||
'path',
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.events)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('name', '=', 'screen_view')
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['origin', 'path'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number; base_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.events)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('name', '=', 'screen_view')
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
const currentMap = buildLookupMap(
|
||||
results,
|
||||
(r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
|
||||
(r) => Number(r.cur ?? 0),
|
||||
);
|
||||
|
||||
const baselineMap = buildLookupMap(
|
||||
results,
|
||||
(r) => `${r.origin || ''}${DELIMITER}${r.path || '/'}`,
|
||||
(r) => Number(r.base ?? 0),
|
||||
);
|
||||
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = totals[0]?.base_total ?? 0;
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
export const pageTrendsModule: InsightModule = {
|
||||
key: 'page-trends',
|
||||
cadence: ['daily'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
const { currentMap, baselineMap } = await fetchPageTrendAggregates(ctx);
|
||||
const topDims = selectTopDimensions(
|
||||
currentMap,
|
||||
baselineMap,
|
||||
this.thresholds?.maxDims ?? 100,
|
||||
);
|
||||
return topDims.map((dim) => `page:${dim}`);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
|
||||
await fetchPageTrendAggregates(ctx);
|
||||
const results: ComputeResult[] = [];
|
||||
|
||||
for (const dimKey of dimensionKeys) {
|
||||
if (!dimKey.startsWith('page:')) continue;
|
||||
const pagePath = dimKey.replace('page:', '');
|
||||
const originPath = dimKey.replace('page:', '');
|
||||
|
||||
const currentValue = currentMap.get(pagePath) ?? 0;
|
||||
const compareValue = baselineMap.get(pagePath) ?? 0;
|
||||
const currentValue = currentMap.get(originPath) ?? 0;
|
||||
const compareValue = baselineMap.get(originPath) ?? 0;
|
||||
|
||||
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
|
||||
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
|
||||
|
||||
const shareShiftPp = (currentShare - compareShare) * 100;
|
||||
const changePct = computeChangePct(currentValue, compareValue);
|
||||
const direction = computeDirection(changePct);
|
||||
|
||||
@@ -144,6 +203,9 @@ export const pageTrendsModule: InsightModule = {
|
||||
changePct,
|
||||
direction,
|
||||
extra: {
|
||||
shareShiftPp,
|
||||
currentShare,
|
||||
compareShare,
|
||||
isNew: compareValue === 0 && currentValue > 0,
|
||||
},
|
||||
});
|
||||
@@ -153,28 +215,62 @@ export const pageTrendsModule: InsightModule = {
|
||||
},
|
||||
|
||||
render(result, ctx): RenderedCard {
|
||||
const path = result.dimensionKey.replace('page:', '');
|
||||
const originPath = result.dimensionKey.replace('page:', '');
|
||||
const [origin, path] = originPath.split(DELIMITER);
|
||||
const displayValue = origin ? `${origin}${path}` : path || '/';
|
||||
const pct = ((result.changePct ?? 0) * 100).toFixed(1);
|
||||
const isIncrease = (result.changePct ?? 0) >= 0;
|
||||
const isNew = result.extra?.isNew as boolean | undefined;
|
||||
|
||||
const title = isNew
|
||||
? `New page getting views: ${path}`
|
||||
: `Page ${path} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
|
||||
? `New page getting views: ${displayValue}`
|
||||
: `Page ${displayValue} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
|
||||
|
||||
const pageviewsCurrent = result.currentValue ?? 0;
|
||||
const pageviewsCompare = result.compareValue ?? 0;
|
||||
const shareCurrent = Number(result.extra?.currentShare ?? 0);
|
||||
const shareCompare = Number(result.extra?.compareShare ?? 0);
|
||||
|
||||
return {
|
||||
kind: 'insight_v1',
|
||||
title,
|
||||
summary: `${ctx.window.label}. Pageviews ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
|
||||
primaryDimension: { type: 'page', key: path, displayName: path },
|
||||
tags: [
|
||||
'page-trends',
|
||||
ctx.window.kind,
|
||||
isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
|
||||
],
|
||||
metric: 'pageviews',
|
||||
extra: {
|
||||
isNew: result.extra?.isNew,
|
||||
summary: `${ctx.window.label}. Pageviews ${pageviewsCurrent} vs ${pageviewsCompare}.`,
|
||||
displayName: displayValue,
|
||||
payload: {
|
||||
kind: 'insight_v1',
|
||||
dimensions: [
|
||||
{ key: 'origin', value: origin ?? '', displayName: origin ?? '' },
|
||||
{ key: 'path', value: path ?? '', displayName: path ?? '' },
|
||||
],
|
||||
primaryMetric: 'pageviews',
|
||||
metrics: {
|
||||
pageviews: {
|
||||
current: pageviewsCurrent,
|
||||
compare: pageviewsCompare,
|
||||
delta: pageviewsCurrent - pageviewsCompare,
|
||||
changePct: pageviewsCompare > 0 ? (result.changePct ?? 0) : null,
|
||||
direction: result.direction ?? 'flat',
|
||||
unit: 'count',
|
||||
},
|
||||
share: {
|
||||
current: shareCurrent,
|
||||
compare: shareCompare,
|
||||
delta: shareCurrent - shareCompare,
|
||||
changePct:
|
||||
shareCompare > 0
|
||||
? (shareCurrent - shareCompare) / shareCompare
|
||||
: null,
|
||||
direction:
|
||||
shareCurrent - shareCompare > 0.0005
|
||||
? 'up'
|
||||
: shareCurrent - shareCompare < -0.0005
|
||||
? 'down'
|
||||
: 'flat',
|
||||
unit: 'ratio',
|
||||
},
|
||||
},
|
||||
extra: {
|
||||
isNew: result.extra?.isNew,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
@@ -1,26 +1,30 @@
|
||||
import { TABLE_NAMES } from '../../../clickhouse/client';
|
||||
import { clix } from '../../../clickhouse/query-builder';
|
||||
import { normalizeReferrer } from '../normalize';
|
||||
import type { ComputeResult, InsightModule, RenderedCard } from '../types';
|
||||
import { TABLE_NAMES, formatClickhouseDate } from '../../../clickhouse/client';
|
||||
import type {
|
||||
ComputeContext,
|
||||
ComputeResult,
|
||||
InsightModule,
|
||||
RenderedCard,
|
||||
} from '../types';
|
||||
import {
|
||||
buildLookupMap,
|
||||
computeChangePct,
|
||||
computeDirection,
|
||||
computeWeekdayMedians,
|
||||
getEndOfDay,
|
||||
getWeekday,
|
||||
selectTopDimensions,
|
||||
} from '../utils';
|
||||
|
||||
export const referrersModule: InsightModule = {
|
||||
key: 'referrers',
|
||||
cadence: ['daily'],
|
||||
windows: ['yesterday', 'rolling_7d', 'rolling_30d'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 20, minPct: 0.15, maxDims: 50 },
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
// Query top referrers from BOTH current and baseline windows
|
||||
// This allows detecting new sources that didn't exist in baseline
|
||||
const [currentResults, baselineResults] = await Promise.all([
|
||||
clix(ctx.db)
|
||||
async function fetchReferrerAggregates(ctx: ComputeContext): Promise<{
|
||||
currentMap: Map<string, number>;
|
||||
baselineMap: Map<string, number>;
|
||||
totalCurrent: number;
|
||||
totalBaseline: number;
|
||||
}> {
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
const [currentResults, baselineResults, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ referrer_name: string; cnt: number }>([
|
||||
'referrer_name',
|
||||
'count(*) as cnt',
|
||||
@@ -33,69 +37,9 @@ export const referrersModule: InsightModule = {
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['referrer_name'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.limit(this.thresholds?.maxDims ?? 50)
|
||||
.execute(),
|
||||
clix(ctx.db)
|
||||
.select<{ referrer_name: string; cnt: number }>([
|
||||
'referrer_name',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['referrer_name'])
|
||||
.orderBy('cnt', 'DESC')
|
||||
.limit(this.thresholds?.maxDims ?? 50)
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
// Merge both sets to catch new/emerging sources
|
||||
const dims = new Set<string>();
|
||||
for (const r of currentResults) {
|
||||
dims.add(`referrer:${normalizeReferrer(r.referrer_name || 'direct')}`);
|
||||
}
|
||||
for (const r of baselineResults) {
|
||||
dims.add(`referrer:${normalizeReferrer(r.referrer_name || 'direct')}`);
|
||||
}
|
||||
|
||||
return Array.from(dims);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
// Single query for ALL current values (batched)
|
||||
const currentResults = await clix(ctx.db)
|
||||
.select<{ referrer_name: string; cnt: number }>([
|
||||
'referrer_name',
|
||||
'count(*) as cnt',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.start,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['referrer_name'])
|
||||
.execute();
|
||||
|
||||
// Build current lookup map
|
||||
const currentMap = new Map<string, number>();
|
||||
for (const r of currentResults) {
|
||||
const key = normalizeReferrer(r.referrer_name || 'direct');
|
||||
currentMap.set(key, (currentMap.get(key) ?? 0) + Number(r.cnt ?? 0));
|
||||
}
|
||||
|
||||
// Single query for baseline (with date breakdown for weekday median if needed)
|
||||
let baselineMap: Map<string, number>;
|
||||
|
||||
if (ctx.window.kind === 'yesterday') {
|
||||
// Need daily breakdown for weekday median calculation
|
||||
const baselineResults = await clix(ctx.db)
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ date: string; referrer_name: string; cnt: number }>([
|
||||
'toDate(created_at) as date',
|
||||
'referrer_name',
|
||||
@@ -109,37 +53,127 @@ export const referrersModule: InsightModule = {
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
])
|
||||
.groupBy(['date', 'referrer_name'])
|
||||
.execute();
|
||||
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
baselineMap = computeWeekdayMedians(baselineResults, targetWeekday, (r) =>
|
||||
normalizeReferrer(r.referrer_name || 'direct'),
|
||||
);
|
||||
} else {
|
||||
// Rolling windows: simple aggregate
|
||||
const baselineResults = await clix(ctx.db)
|
||||
.select<{ referrer_name: string; cnt: number }>([
|
||||
'referrer_name',
|
||||
'count(*) as cnt',
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${formatClickhouseDate(ctx.window.start)}' AND '${formatClickhouseDate(getEndOfDay(ctx.window.end))}') as cur_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.baselineEnd),
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['referrer_name'])
|
||||
.execute();
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
baselineMap = new Map<string, number>();
|
||||
for (const r of baselineResults) {
|
||||
const key = normalizeReferrer(r.referrer_name || 'direct');
|
||||
baselineMap.set(key, (baselineMap.get(key) ?? 0) + Number(r.cnt ?? 0));
|
||||
}
|
||||
}
|
||||
const currentMap = buildLookupMap(
|
||||
currentResults,
|
||||
(r) => r.referrer_name || 'direct',
|
||||
);
|
||||
|
||||
// Build results from maps (in memory, no more queries!)
|
||||
const targetWeekday = getWeekday(ctx.window.start);
|
||||
const baselineMap = computeWeekdayMedians(
|
||||
baselineResults,
|
||||
targetWeekday,
|
||||
(r) => r.referrer_name || 'direct',
|
||||
);
|
||||
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = Array.from(baselineMap.values()).reduce(
|
||||
(sum, val) => sum + val,
|
||||
0,
|
||||
);
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
const curStart = formatClickhouseDate(ctx.window.start);
|
||||
const curEnd = formatClickhouseDate(getEndOfDay(ctx.window.end));
|
||||
const baseStart = formatClickhouseDate(ctx.window.baselineStart);
|
||||
const baseEnd = formatClickhouseDate(getEndOfDay(ctx.window.baselineEnd));
|
||||
|
||||
const [results, totals] = await Promise.all([
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ referrer_name: string; cur: number; base: number }>([
|
||||
'referrer_name',
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.groupBy(['referrer_name'])
|
||||
.execute(),
|
||||
ctx
|
||||
.clix()
|
||||
.select<{ cur_total: number; base_total: number }>([
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${curStart}' AND '${curEnd}') as cur_total`,
|
||||
),
|
||||
ctx.clix.exp(
|
||||
`countIf(created_at BETWEEN '${baseStart}' AND '${baseEnd}') as base_total`,
|
||||
),
|
||||
])
|
||||
.from(TABLE_NAMES.sessions)
|
||||
.where('project_id', '=', ctx.projectId)
|
||||
.where('sign', '=', 1)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
ctx.window.baselineStart,
|
||||
getEndOfDay(ctx.window.end),
|
||||
])
|
||||
.execute(),
|
||||
]);
|
||||
|
||||
const currentMap = buildLookupMap(
|
||||
results,
|
||||
(r) => r.referrer_name || 'direct',
|
||||
(r) => Number(r.cur ?? 0),
|
||||
);
|
||||
|
||||
const baselineMap = buildLookupMap(
|
||||
results,
|
||||
(r) => r.referrer_name || 'direct',
|
||||
(r) => Number(r.base ?? 0),
|
||||
);
|
||||
|
||||
const totalCurrent = totals[0]?.cur_total ?? 0;
|
||||
const totalBaseline = totals[0]?.base_total ?? 0;
|
||||
|
||||
return { currentMap, baselineMap, totalCurrent, totalBaseline };
|
||||
}
|
||||
|
||||
export const referrersModule: InsightModule = {
|
||||
key: 'referrers',
|
||||
cadence: ['daily'],
|
||||
thresholds: { minTotal: 100, minAbsDelta: 20, minPct: 0.15, maxDims: 50 },
|
||||
|
||||
async enumerateDimensions(ctx) {
|
||||
const { currentMap, baselineMap } = await fetchReferrerAggregates(ctx);
|
||||
const topDims = selectTopDimensions(
|
||||
currentMap,
|
||||
baselineMap,
|
||||
this.thresholds?.maxDims ?? 50,
|
||||
);
|
||||
return topDims.map((dim) => `referrer:${dim}`);
|
||||
},
|
||||
|
||||
async computeMany(ctx, dimensionKeys): Promise<ComputeResult[]> {
|
||||
const { currentMap, baselineMap, totalCurrent, totalBaseline } =
|
||||
await fetchReferrerAggregates(ctx);
|
||||
const results: ComputeResult[] = [];
|
||||
|
||||
for (const dimKey of dimensionKeys) {
|
||||
@@ -148,6 +182,11 @@ export const referrersModule: InsightModule = {
|
||||
|
||||
const currentValue = currentMap.get(referrerName) ?? 0;
|
||||
const compareValue = baselineMap.get(referrerName) ?? 0;
|
||||
|
||||
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
|
||||
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
|
||||
|
||||
const shareShiftPp = (currentShare - compareShare) * 100;
|
||||
const changePct = computeChangePct(currentValue, compareValue);
|
||||
const direction = computeDirection(changePct);
|
||||
|
||||
@@ -159,6 +198,9 @@ export const referrersModule: InsightModule = {
|
||||
changePct,
|
||||
direction,
|
||||
extra: {
|
||||
shareShiftPp,
|
||||
currentShare,
|
||||
compareShare,
|
||||
isNew: compareValue === 0 && currentValue > 0,
|
||||
isGone: currentValue === 0 && compareValue > 0,
|
||||
},
|
||||
@@ -178,24 +220,55 @@ export const referrersModule: InsightModule = {
|
||||
? `New traffic source: ${referrer}`
|
||||
: `Traffic from ${referrer} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
|
||||
|
||||
const sessionsCurrent = result.currentValue ?? 0;
|
||||
const sessionsCompare = result.compareValue ?? 0;
|
||||
const shareCurrent = Number(result.extra?.currentShare ?? 0);
|
||||
const shareCompare = Number(result.extra?.compareShare ?? 0);
|
||||
|
||||
return {
|
||||
kind: 'insight_v1',
|
||||
title,
|
||||
summary: `${ctx.window.label}. Sessions ${result.currentValue ?? 0} vs ${result.compareValue ?? 0}.`,
|
||||
primaryDimension: {
|
||||
type: 'referrer',
|
||||
key: referrer,
|
||||
displayName: referrer,
|
||||
},
|
||||
tags: [
|
||||
'referrers',
|
||||
ctx.window.kind,
|
||||
isNew ? 'new' : isIncrease ? 'increase' : 'decrease',
|
||||
],
|
||||
metric: 'sessions',
|
||||
extra: {
|
||||
isNew: result.extra?.isNew,
|
||||
isGone: result.extra?.isGone,
|
||||
summary: `${ctx.window.label}. Sessions ${sessionsCurrent} vs ${sessionsCompare}.`,
|
||||
displayName: referrer,
|
||||
payload: {
|
||||
kind: 'insight_v1',
|
||||
dimensions: [
|
||||
{
|
||||
key: 'referrer_name',
|
||||
value: referrer,
|
||||
displayName: referrer,
|
||||
},
|
||||
],
|
||||
primaryMetric: 'sessions',
|
||||
metrics: {
|
||||
sessions: {
|
||||
current: sessionsCurrent,
|
||||
compare: sessionsCompare,
|
||||
delta: sessionsCurrent - sessionsCompare,
|
||||
changePct: sessionsCompare > 0 ? (result.changePct ?? 0) : null,
|
||||
direction: result.direction ?? 'flat',
|
||||
unit: 'count',
|
||||
},
|
||||
share: {
|
||||
current: shareCurrent,
|
||||
compare: shareCompare,
|
||||
delta: shareCurrent - shareCompare,
|
||||
changePct:
|
||||
shareCompare > 0
|
||||
? (shareCurrent - shareCompare) / shareCompare
|
||||
: null,
|
||||
direction:
|
||||
shareCurrent - shareCompare > 0.0005
|
||||
? 'up'
|
||||
: shareCurrent - shareCompare < -0.0005
|
||||
? 'down'
|
||||
: 'flat',
|
||||
unit: 'ratio',
|
||||
},
|
||||
},
|
||||
extra: {
|
||||
isNew: result.extra?.isNew,
|
||||
isGone: result.extra?.isGone,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
export function normalizeReferrer(name: string): string {
|
||||
if (!name || name === '') return 'direct';
|
||||
|
||||
const normalized = name.toLowerCase().trim();
|
||||
|
||||
// Normalize common referrer variations
|
||||
const map: Record<string, string> = {
|
||||
'm.instagram.com': 'instagram',
|
||||
'l.instagram.com': 'instagram',
|
||||
'www.instagram.com': 'instagram',
|
||||
'instagram.com': 'instagram',
|
||||
't.co': 'twitter',
|
||||
'twitter.com': 'twitter',
|
||||
'x.com': 'twitter',
|
||||
'lm.facebook.com': 'facebook',
|
||||
'm.facebook.com': 'facebook',
|
||||
'facebook.com': 'facebook',
|
||||
'l.facebook.com': 'facebook',
|
||||
'linkedin.com': 'linkedin',
|
||||
'www.linkedin.com': 'linkedin',
|
||||
'youtube.com': 'youtube',
|
||||
'www.youtube.com': 'youtube',
|
||||
'm.youtube.com': 'youtube',
|
||||
'reddit.com': 'reddit',
|
||||
'www.reddit.com': 'reddit',
|
||||
'tiktok.com': 'tiktok',
|
||||
'www.tiktok.com': 'tiktok',
|
||||
};
|
||||
|
||||
// Check exact match first
|
||||
if (map[normalized]) {
|
||||
return map[normalized];
|
||||
}
|
||||
|
||||
// Check if it contains any of the mapped domains
|
||||
for (const [key, value] of Object.entries(map)) {
|
||||
if (normalized.includes(key)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
// Extract domain from URL if present
|
||||
try {
|
||||
const url = normalized.startsWith('http')
|
||||
? normalized
|
||||
: `https://${normalized}`;
|
||||
const hostname = new URL(url).hostname;
|
||||
// Remove www. prefix
|
||||
return hostname.replace(/^www\./, '');
|
||||
} catch {
|
||||
// If not a valid URL, return as-is
|
||||
return normalized || 'direct';
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizePath(path: string): string {
|
||||
if (!path || path === '') return '/';
|
||||
|
||||
try {
|
||||
// If it's a full URL, extract pathname
|
||||
const url = path.startsWith('http')
|
||||
? new URL(path)
|
||||
: new URL(path, 'http://x');
|
||||
const pathname = url.pathname;
|
||||
// Normalize trailing slash (remove unless it's root)
|
||||
return pathname === '/' ? '/' : pathname.replace(/\/$/, '');
|
||||
} catch {
|
||||
// If not a valid URL, treat as path
|
||||
return path === '/' ? '/' : path.replace(/\/$/, '') || '/';
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeUtmCombo(source: string, medium: string): string {
|
||||
const s = (source || '').toLowerCase().trim();
|
||||
const m = (medium || '').toLowerCase().trim();
|
||||
if (!s && !m) return 'none';
|
||||
if (!s) return `utm:${m}`;
|
||||
if (!m) return `utm:${s}`;
|
||||
return `utm:${s}/${m}`;
|
||||
}
|
||||
@@ -11,8 +11,8 @@ export function severityBand(
|
||||
changePct?: number | null,
|
||||
): 'low' | 'moderate' | 'severe' | null {
|
||||
const p = Math.abs(changePct ?? 0);
|
||||
if (p < 0.05) return null;
|
||||
if (p < 0.15) return 'low';
|
||||
if (p < 0.3) return 'moderate';
|
||||
if (p < 0.1) return null;
|
||||
if (p < 0.5) return 'low';
|
||||
if (p < 1) return 'moderate';
|
||||
return 'severe';
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ export const insightStore: InsightStore = {
|
||||
const projects = await db.project.findMany({
|
||||
where: {
|
||||
deleteAt: null,
|
||||
eventsCount: { gt: 10_000 },
|
||||
updatedAt: { gt: new Date(Date.now() - 1000 * 60 * 60 * 24) },
|
||||
organization: {
|
||||
subscriptionStatus: 'active',
|
||||
},
|
||||
@@ -22,6 +24,14 @@ export const insightStore: InsightStore = {
|
||||
return projects.map((p) => p.id);
|
||||
},
|
||||
|
||||
async getProjectCreatedAt(projectId: string): Promise<Date | null> {
|
||||
const project = await db.project.findFirst({
|
||||
where: { id: projectId, deleteAt: null },
|
||||
select: { createdAt: true },
|
||||
});
|
||||
return project?.createdAt ?? null;
|
||||
},
|
||||
|
||||
async getActiveInsightByIdentity({
|
||||
projectId,
|
||||
moduleKey,
|
||||
@@ -52,7 +62,6 @@ export const insightStore: InsightStore = {
|
||||
lastSeenAt: insight.lastSeenAt,
|
||||
lastUpdatedAt: insight.lastUpdatedAt,
|
||||
direction: insight.direction,
|
||||
changePct: insight.changePct,
|
||||
severityBand: insight.severityBand,
|
||||
};
|
||||
},
|
||||
@@ -68,8 +77,6 @@ export const insightStore: InsightStore = {
|
||||
decision,
|
||||
prev,
|
||||
}): Promise<PersistedInsight> {
|
||||
const payloadData = (card.payload ?? card) as Prisma.InputJsonValue;
|
||||
|
||||
const baseData = {
|
||||
projectId,
|
||||
moduleKey,
|
||||
@@ -78,10 +85,8 @@ export const insightStore: InsightStore = {
|
||||
state: prev?.state === 'closed' ? 'active' : (prev?.state ?? 'active'),
|
||||
title: card.title,
|
||||
summary: card.summary ?? null,
|
||||
payload: payloadData as Prisma.InputJsonValue,
|
||||
currentValue: metrics.currentValue ?? null,
|
||||
compareValue: metrics.compareValue ?? null,
|
||||
changePct: metrics.changePct ?? null,
|
||||
displayName: card.displayName,
|
||||
payload: card.payload,
|
||||
direction: metrics.direction ?? null,
|
||||
impactScore: metrics.impactScore,
|
||||
severityBand: metrics.severityBand ?? null,
|
||||
@@ -161,7 +166,6 @@ export const insightStore: InsightStore = {
|
||||
lastSeenAt: insight.lastSeenAt,
|
||||
lastUpdatedAt: insight.lastUpdatedAt,
|
||||
direction: insight.direction,
|
||||
changePct: insight.changePct,
|
||||
severityBand: insight.severityBand,
|
||||
};
|
||||
},
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
export type Cadence = 'hourly' | 'daily' | 'weekly';
|
||||
import type {
|
||||
InsightDimension,
|
||||
InsightMetricEntry,
|
||||
InsightMetricKey,
|
||||
InsightPayload,
|
||||
} from '@openpanel/validation';
|
||||
|
||||
export type Cadence = 'daily';
|
||||
|
||||
export type WindowKind = 'yesterday' | 'rolling_7d' | 'rolling_30d';
|
||||
|
||||
@@ -17,6 +24,12 @@ export interface ComputeContext {
|
||||
db: any; // your DB client
|
||||
now: Date;
|
||||
logger: Pick<Console, 'info' | 'warn' | 'error'>;
|
||||
/**
|
||||
* Cached clix function that automatically caches query results based on query hash.
|
||||
* This eliminates duplicate queries within the same module+window context.
|
||||
* Use this instead of importing clix directly to benefit from automatic caching.
|
||||
*/
|
||||
clix: ReturnType<typeof import('./cached-clix').createCachedClix>;
|
||||
}
|
||||
|
||||
export interface ComputeResult {
|
||||
@@ -29,31 +42,22 @@ export interface ComputeResult {
|
||||
extra?: Record<string, unknown>; // share delta pp, rank, sparkline, etc.
|
||||
}
|
||||
|
||||
// Types imported from @openpanel/validation:
|
||||
// - InsightMetricKey
|
||||
// - InsightMetricEntry
|
||||
// - InsightDimension
|
||||
// - InsightPayload
|
||||
|
||||
/**
|
||||
* Render should be deterministic and safe to call multiple times.
|
||||
* Raw computed values (currentValue, compareValue, changePct, direction)
|
||||
* are stored in top-level DB fields. The payload only contains display
|
||||
* metadata and module-specific extra data for frontend flexibility.
|
||||
* Returns the shape that matches ProjectInsight create input.
|
||||
* The payload contains all metric data and display metadata.
|
||||
*/
|
||||
export interface RenderedCard {
|
||||
kind?: 'insight_v1';
|
||||
title: string;
|
||||
summary?: string;
|
||||
tags?: string[];
|
||||
primaryDimension?: { type: string; key: string; displayName?: string };
|
||||
|
||||
/**
|
||||
* What metric this insight measures - frontend uses this to format values.
|
||||
* 'sessions' | 'pageviews' for absolute counts
|
||||
* 'share' for percentage-based (geo, devices)
|
||||
*/
|
||||
metric?: 'sessions' | 'pageviews' | 'share';
|
||||
|
||||
/**
|
||||
* Module-specific extra data (e.g., share values for geo/devices).
|
||||
* Frontend can use this based on moduleKey.
|
||||
*/
|
||||
extra?: Record<string, unknown>;
|
||||
displayName: string;
|
||||
payload: InsightPayload; // Contains dimensions, primaryMetric, metrics, extra
|
||||
}
|
||||
|
||||
/** Optional per-module thresholds (the engine can still apply global defaults) */
|
||||
@@ -67,7 +71,8 @@ export interface ModuleThresholds {
|
||||
export interface InsightModule {
|
||||
key: string;
|
||||
cadence: Cadence[];
|
||||
windows: WindowKind[];
|
||||
/** Optional per-module override; engine applies a default if omitted. */
|
||||
windows?: WindowKind[];
|
||||
thresholds?: ModuleThresholds;
|
||||
enumerateDimensions?(ctx: ComputeContext): Promise<string[]>;
|
||||
/** Preferred path: batch compute many dimensions in one go. */
|
||||
@@ -99,7 +104,6 @@ export interface PersistedInsight {
|
||||
lastSeenAt: Date;
|
||||
lastUpdatedAt: Date;
|
||||
direction?: string | null;
|
||||
changePct?: number | null;
|
||||
severityBand?: string | null;
|
||||
}
|
||||
|
||||
@@ -124,6 +128,8 @@ export interface MaterialDecision {
|
||||
*/
|
||||
export interface InsightStore {
|
||||
listProjectIdsForCadence(cadence: Cadence): Promise<string[]>;
|
||||
/** Used by the engine/worker to decide if a window has enough baseline history. */
|
||||
getProjectCreatedAt(projectId: string): Promise<Date | null>;
|
||||
getActiveInsightByIdentity(args: {
|
||||
projectId: string;
|
||||
moduleKey: string;
|
||||
@@ -137,9 +143,6 @@ export interface InsightStore {
|
||||
window: WindowRange;
|
||||
card: RenderedCard;
|
||||
metrics: {
|
||||
currentValue?: number;
|
||||
compareValue?: number;
|
||||
changePct?: number;
|
||||
direction?: 'up' | 'down' | 'flat';
|
||||
impactScore: number;
|
||||
severityBand?: string | null;
|
||||
@@ -186,15 +189,3 @@ export interface InsightStore {
|
||||
now: Date;
|
||||
}): Promise<{ suppressed: number; unsuppressed: number }>;
|
||||
}
|
||||
|
||||
export interface ExplainQueue {
|
||||
enqueueExplain(job: {
|
||||
insightId: string;
|
||||
projectId: string;
|
||||
moduleKey: string;
|
||||
dimensionKey: string;
|
||||
windowKind: WindowKind;
|
||||
evidence: Record<string, unknown>;
|
||||
evidenceHash: string;
|
||||
}): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -29,9 +29,7 @@ export function computeMedian(sortedValues: number[]): number {
|
||||
* @param getDimension - Function to extract normalized dimension from row
|
||||
* @returns Map of dimension -> median value
|
||||
*/
|
||||
export function computeWeekdayMedians<
|
||||
T extends { date: string; cnt: number | string },
|
||||
>(
|
||||
export function computeWeekdayMedians<T>(
|
||||
data: T[],
|
||||
targetWeekday: number,
|
||||
getDimension: (row: T) => string,
|
||||
@@ -40,12 +38,12 @@ export function computeWeekdayMedians<
|
||||
const byDimension = new Map<string, number[]>();
|
||||
|
||||
for (const row of data) {
|
||||
const rowWeekday = getWeekday(new Date(row.date));
|
||||
const rowWeekday = getWeekday(new Date((row as any).date));
|
||||
if (rowWeekday !== targetWeekday) continue;
|
||||
|
||||
const dim = getDimension(row);
|
||||
const values = byDimension.get(dim) ?? [];
|
||||
values.push(Number(row.cnt ?? 0));
|
||||
values.push(Number((row as any).cnt ?? 0));
|
||||
byDimension.set(dim, values);
|
||||
}
|
||||
|
||||
@@ -87,19 +85,6 @@ export function computeDirection(
|
||||
: 'flat';
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge dimension sets from current and baseline to detect new/gone dimensions
|
||||
*/
|
||||
export function mergeDimensionSets(
|
||||
currentDims: Set<string>,
|
||||
baselineDims: Set<string>,
|
||||
): string[] {
|
||||
const merged = new Set<string>();
|
||||
for (const dim of currentDims) merged.add(dim);
|
||||
for (const dim of baselineDims) merged.add(dim);
|
||||
return Array.from(merged);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get end of day timestamp (23:59:59.999) for a given date.
|
||||
* Used to ensure BETWEEN queries include the full day.
|
||||
@@ -109,3 +94,58 @@ export function getEndOfDay(date: Date): Date {
|
||||
end.setUTCHours(23, 59, 59, 999);
|
||||
return end;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a lookup map from query results.
|
||||
* Aggregates counts by key, handling duplicate keys by summing values.
|
||||
*
|
||||
* @param results - Array of result rows
|
||||
* @param getKey - Function to extract the key from each row
|
||||
* @param getCount - Function to extract the count from each row (defaults to 'cnt' field)
|
||||
* @returns Map of key -> aggregated count
|
||||
*/
|
||||
export function buildLookupMap<T>(
|
||||
results: T[],
|
||||
getKey: (row: T) => string,
|
||||
getCount: (row: T) => number = (row) => Number((row as any).cnt ?? 0),
|
||||
): Map<string, number> {
|
||||
const map = new Map<string, number>();
|
||||
for (const row of results) {
|
||||
const key = getKey(row);
|
||||
const cnt = getCount(row);
|
||||
map.set(key, (map.get(key) ?? 0) + cnt);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Select top-N dimensions by ranking on greatest(current, baseline).
|
||||
* This preserves union behavior: dimensions with high values in either period are included.
|
||||
*
|
||||
* @param currentMap - Map of dimension -> current value
|
||||
* @param baselineMap - Map of dimension -> baseline value
|
||||
* @param maxDims - Maximum number of dimensions to return
|
||||
* @returns Array of dimension keys, ranked by greatest(current, baseline)
|
||||
*/
|
||||
export function selectTopDimensions(
|
||||
currentMap: Map<string, number>,
|
||||
baselineMap: Map<string, number>,
|
||||
maxDims: number,
|
||||
): string[] {
|
||||
// Merge all dimensions from both maps
|
||||
const allDims = new Set<string>();
|
||||
for (const dim of currentMap.keys()) allDims.add(dim);
|
||||
for (const dim of baselineMap.keys()) allDims.add(dim);
|
||||
|
||||
// Rank by greatest(current, baseline)
|
||||
const ranked = Array.from(allDims)
|
||||
.map((dim) => ({
|
||||
dim,
|
||||
maxValue: Math.max(currentMap.get(dim) ?? 0, baselineMap.get(dim) ?? 0),
|
||||
}))
|
||||
.sort((a, b) => b.maxValue - a.maxValue)
|
||||
.slice(0, maxDims)
|
||||
.map((x) => x.dim);
|
||||
|
||||
return ranked;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import type {
|
||||
IIntegrationConfig,
|
||||
INotificationRuleConfig,
|
||||
IProjectFilters,
|
||||
InsightPayload,
|
||||
} from '@openpanel/validation';
|
||||
import type {
|
||||
IClickhouseBotEvent,
|
||||
@@ -18,6 +19,7 @@ declare global {
|
||||
type IPrismaIntegrationConfig = IIntegrationConfig;
|
||||
type IPrismaNotificationPayload = INotificationPayload;
|
||||
type IPrismaProjectFilters = IProjectFilters[];
|
||||
type IPrismaProjectInsightPayload = InsightPayload;
|
||||
type IPrismaClickhouseEvent = IClickhouseEvent;
|
||||
type IPrismaClickhouseProfile = IClickhouseProfile;
|
||||
type IPrismaClickhouseBotEvent = IClickhouseBotEvent;
|
||||
|
||||
@@ -35,22 +35,6 @@ export const insightRouter = createTRPCRouter({
|
||||
impactScore: 'desc',
|
||||
},
|
||||
take: limit * 3, // Fetch 3x to account for deduplication
|
||||
select: {
|
||||
id: true,
|
||||
title: true,
|
||||
summary: true,
|
||||
payload: true,
|
||||
currentValue: true,
|
||||
compareValue: true,
|
||||
changePct: true,
|
||||
direction: true,
|
||||
moduleKey: true,
|
||||
dimensionKey: true,
|
||||
windowKind: true,
|
||||
severityBand: true,
|
||||
firstDetectedAt: true,
|
||||
impactScore: true,
|
||||
},
|
||||
});
|
||||
|
||||
// WindowKind priority: yesterday (1) > rolling_7d (2) > rolling_30d (3)
|
||||
@@ -111,22 +95,6 @@ export const insightRouter = createTRPCRouter({
|
||||
impactScore: 'desc',
|
||||
},
|
||||
take: limit,
|
||||
select: {
|
||||
id: true,
|
||||
title: true,
|
||||
summary: true,
|
||||
payload: true,
|
||||
currentValue: true,
|
||||
compareValue: true,
|
||||
changePct: true,
|
||||
direction: true,
|
||||
moduleKey: true,
|
||||
dimensionKey: true,
|
||||
windowKind: true,
|
||||
severityBand: true,
|
||||
firstDetectedAt: true,
|
||||
impactScore: true,
|
||||
},
|
||||
});
|
||||
|
||||
return insights;
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
export * from './src/index';
|
||||
export * from './src/types.validation';
|
||||
export * from './src/types.insights';
|
||||
|
||||
@@ -553,3 +553,5 @@ export const zCreateImport = z.object({
|
||||
});
|
||||
|
||||
export type ICreateImport = z.infer<typeof zCreateImport>;
|
||||
|
||||
export * from './types.insights';
|
||||
|
||||
43
packages/validation/src/types.insights.ts
Normal file
43
packages/validation/src/types.insights.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
export type InsightMetricKey = 'sessions' | 'pageviews' | 'share';
|
||||
|
||||
export type InsightMetricUnit = 'count' | 'ratio';
|
||||
|
||||
export interface InsightMetricEntry {
|
||||
current: number;
|
||||
compare: number;
|
||||
delta: number;
|
||||
changePct: number | null;
|
||||
direction: 'up' | 'down' | 'flat';
|
||||
unit: InsightMetricUnit;
|
||||
}
|
||||
|
||||
export interface InsightDimension {
|
||||
key: string;
|
||||
value: string;
|
||||
displayName?: string;
|
||||
}
|
||||
|
||||
export interface InsightExtra {
|
||||
[key: string]: unknown;
|
||||
currentShare?: number;
|
||||
compareShare?: number;
|
||||
shareShiftPp?: number;
|
||||
isNew?: boolean;
|
||||
isGone?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared payload shape for insights cards. This is embedded in DB rows and
|
||||
* shipped to the frontend, so it must remain backwards compatible.
|
||||
*/
|
||||
export interface InsightPayload {
|
||||
kind?: 'insight_v1';
|
||||
dimensions: InsightDimension[];
|
||||
primaryMetric: InsightMetricKey;
|
||||
metrics: Partial<Record<InsightMetricKey, InsightMetricEntry>>;
|
||||
|
||||
/**
|
||||
* Module-specific extra data.
|
||||
*/
|
||||
extra?: Record<string, unknown>;
|
||||
}
|
||||
Reference in New Issue
Block a user