From 15388882be5e12b42d01d5b3f401687f6a1925ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl-Gerhard=20Lindesva=CC=88rd?= Date: Sat, 24 Feb 2024 07:22:39 +0100 Subject: [PATCH] add funnels --- apps/sdk-api/package.json | 5 +- .../src/controllers/event.controller.ts | 26 +- apps/test/package.json | 1 + apps/test/src/pages/test.tsx | 15 + apps/web/package.json | 4 +- .../[projectId]/overview-metrics.tsx | 6 +- .../[projectId]/profiles/[profileId]/page.tsx | 4 +- .../[projectId]/reports/report-editor.tsx | 6 +- .../[projectId]/test/page.tsx | 21 ++ .../overview/overview-live-histogram.tsx | 4 +- .../overview/overview-top-devices.tsx | 4 +- .../overview/overview-top-events.tsx | 4 +- .../components/overview/overview-top-geo.tsx | 4 +- .../overview/overview-top-pages.tsx | 4 +- .../overview/overview-top-sources.tsx | 4 +- .../web/src/components/report/chart/Chart.tsx | 92 +++++++ .../src/components/report/chart/LazyChart.tsx | 6 +- .../components/report/chart/MetricCard.tsx | 2 +- .../web/src/components/report/chart/index.tsx | 97 +------ .../src/components/report/funnel/Funnel.tsx | 169 ++++++++++++ .../src/components/report/funnel/index.tsx | 53 ++++ .../report/sidebar/ReportSidebar.tsx | 8 +- apps/web/src/components/ui/carousel.tsx | 258 ++++++++++++++++++ apps/web/src/server/api/routers/chart.ts | 116 +++++++- apps/worker/package.json | 1 + packages/constants/index.ts | 1 + packages/db/clickhouse_tables.sql | 7 +- .../migration.sql | 2 + packages/db/prisma/schema.prisma | 1 + packages/db/scripts/test-funnel.ts | 101 +++++++ packages/db/src/services/event.service.ts | 6 +- packages/db/src/services/profile.service.ts | 4 + packages/db/src/sql-builder.ts | 2 +- pnpm-lock.yaml | 9 + 34 files changed, 916 insertions(+), 131 deletions(-) create mode 100644 apps/web/src/app/(app)/[organizationId]/[projectId]/test/page.tsx create mode 100644 apps/web/src/components/report/chart/Chart.tsx create mode 100644 apps/web/src/components/report/funnel/Funnel.tsx create mode 100644 apps/web/src/components/report/funnel/index.tsx create mode 100644 apps/web/src/components/ui/carousel.tsx create mode 100644 packages/db/prisma/migrations/20240223193217_add_funnel_chart_type/migration.sql create mode 100644 packages/db/scripts/test-funnel.ts diff --git a/apps/sdk-api/package.json b/apps/sdk-api/package.json index 81ac4af9..21a5ef53 100644 --- a/apps/sdk-api/package.json +++ b/apps/sdk-api/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "scripts": { "dev": "dotenv -e ../../.env -c -v WATCH=1 tsup", + "testing": "API_PORT=3333 pnpm dev", "start": "node dist/index.js", "build": "rm -rf dist && tsup", "lint": "eslint .", @@ -21,7 +22,8 @@ "pino": "^8.17.2", "ramda": "^0.29.1", "sharp": "^0.33.2", - "ua-parser-js": "^1.0.37" + "ua-parser-js": "^1.0.37", + "uuid": "^9.0.1" }, "devDependencies": { "@mixan/eslint-config": "workspace:*", @@ -30,6 +32,7 @@ "@mixan/types": "workspace:*", "@types/ramda": "^0.29.6", "@types/ua-parser-js": "^0.7.39", + "@types/uuid": "^9.0.8", "@types/ws": "^8.5.10", "eslint": "^8.48.0", "prettier": "^3.0.3", diff --git a/apps/sdk-api/src/controllers/event.controller.ts b/apps/sdk-api/src/controllers/event.controller.ts index 3dbcb8da..df11541a 100644 --- a/apps/sdk-api/src/controllers/event.controller.ts +++ b/apps/sdk-api/src/controllers/event.controller.ts @@ -4,10 +4,11 @@ import { getReferrerWithQuery, parseReferrer } from '@/utils/parseReferrer'; import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent'; import type { FastifyReply, FastifyRequest } from 'fastify'; import { omit } from 'ramda'; +import { v4 as uuid } from 'uuid'; import { generateDeviceId, getTime, toISOString } from '@mixan/common'; import type { IServiceCreateEventPayload } from '@mixan/db'; -import { createBotEvent, getEvents, getSalts } from '@mixan/db'; +import { createBotEvent, createEvent, getEvents, getSalts } from '@mixan/db'; import type { JobsOptions } from '@mixan/queue'; import { eventsQueue, findJobByPrefix } from '@mixan/queue'; import type { PostEventPayload } from '@mixan/types'; @@ -108,6 +109,7 @@ export async function postEvent( payload: { name: body.name, deviceId: event?.deviceId || '', + sessionId: event?.sessionId || '', profileId, projectId, properties: body.properties ?? {}, @@ -145,11 +147,16 @@ export async function postEvent( return reply.status(200).send(''); } - const [geo, eventsJobs] = await Promise.all([ + const [geo, eventsJobs, events] = await Promise.all([ parseIp(ip), eventsQueue.getJobs(['delayed']), + getEvents( + `SELECT * FROM events WHERE name = 'session_start' AND profile_id = '${profileId}' AND project_id = '${projectId}' ORDER BY created_at DESC LIMIT 1` + ), ]); + const sessionStartEvent = events[0]; + // find session_end job const sessionEndJobCurrentDeviceId = findJobByPrefix( eventsJobs, @@ -197,6 +204,7 @@ export async function postEvent( deviceId, profileId, projectId, + sessionId: createSessionStart ? uuid() : sessionStartEvent?.sessionId ?? '', properties: Object.assign({}, omit(['path', 'referrer'], body.properties), { hash, query, @@ -246,14 +254,12 @@ export async function postEvent( } if (createSessionStart) { - eventsQueue.add('event', { - type: 'createEvent', - payload: { - ...payload, - name: 'session_start', - // @ts-expect-error - createdAt: toISOString(getTime(payload.createdAt) - 10), - }, + // We do not need to queue session_start + await createEvent({ + ...payload, + name: 'session_start', + // @ts-expect-error + createdAt: toISOString(getTime(payload.createdAt) - 10), }); } diff --git a/apps/test/package.json b/apps/test/package.json index 6b965b66..40c19fad 100644 --- a/apps/test/package.json +++ b/apps/test/package.json @@ -4,6 +4,7 @@ "private": true, "scripts": { "dev": "next dev -p 3002", + "testing": "pnpm dev", "build": "next build", "start": "next start", "lint": "eslint .", diff --git a/apps/test/src/pages/test.tsx b/apps/test/src/pages/test.tsx index 62d67829..4fe7f5cf 100644 --- a/apps/test/src/pages/test.tsx +++ b/apps/test/src/pages/test.tsx @@ -101,6 +101,21 @@ export default function Test() { Trigger event + +
+ {['a', 'b', 'c', 'd', 'f'].map((letter) => ( + + ))} +
); } diff --git a/apps/web/package.json b/apps/web/package.json index 17fac2bf..62c50474 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -4,6 +4,7 @@ "private": true, "scripts": { "dev": "rm -rf .next && pnpm with-env next dev", + "testing": "pnpm dev", "build": "next build", "start": "next start", "lint": "eslint .", @@ -17,10 +18,10 @@ "@hookform/resolvers": "^3.3.4", "@mixan/common": "workspace:^", "@mixan/constants": "workspace:^", - "@mixan/validation": "workspace:^", "@mixan/db": "workspace:^", "@mixan/queue": "workspace:^", "@mixan/types": "workspace:*", + "@mixan/validation": "workspace:^", "@radix-ui/react-alert-dialog": "^1.0.5", "@radix-ui/react-aspect-ratio": "^1.0.3", "@radix-ui/react-avatar": "^1.0.4", @@ -46,6 +47,7 @@ "class-variance-authority": "^0.7.0", "clsx": "^2.1.0", "cmdk": "^0.2.1", + "embla-carousel-react": "8.0.0-rc22", "hamburger-react": "^2.5.0", "lodash.debounce": "^4.0.8", "lodash.throttle": "^4.1.1", diff --git a/apps/web/src/app/(app)/[organizationId]/[projectId]/overview-metrics.tsx b/apps/web/src/app/(app)/[organizationId]/[projectId]/overview-metrics.tsx index 8b95d107..2b087c7b 100644 --- a/apps/web/src/app/(app)/[organizationId]/[projectId]/overview-metrics.tsx +++ b/apps/web/src/app/(app)/[organizationId]/[projectId]/overview-metrics.tsx @@ -2,7 +2,7 @@ import { WidgetHead } from '@/components/overview/overview-widget'; import { useOverviewOptions } from '@/components/overview/useOverviewOptions'; -import { Chart } from '@/components/report/chart'; +import { ChartSwitch } from '@/components/report/chart'; import { Widget, WidgetBody } from '@/components/Widget'; import { useEventQueryFilters } from '@/hooks/useEventQueryFilters'; import { cn } from '@/utils/cn'; @@ -186,7 +186,7 @@ export default function OverviewMetrics({ projectId }: OverviewMetricsProps) { setMetric(index); }} > - +
{selectedMetric.events[0]?.displayName}
- Events per day - + diff --git a/apps/web/src/app/(app)/[organizationId]/[projectId]/reports/report-editor.tsx b/apps/web/src/app/(app)/[organizationId]/[projectId]/reports/report-editor.tsx index 41b04289..3a8d5c2e 100644 --- a/apps/web/src/app/(app)/[organizationId]/[projectId]/reports/report-editor.tsx +++ b/apps/web/src/app/(app)/[organizationId]/[projectId]/reports/report-editor.tsx @@ -2,7 +2,7 @@ import { useEffect } from 'react'; import { StickyBelowHeader } from '@/app/(app)/[organizationId]/[projectId]/layout-sticky-below-header'; -import { Chart } from '@/components/report/chart'; +import { ChartSwitch } from '@/components/report/chart'; import { ReportChartType } from '@/components/report/ReportChartType'; import { ReportInterval } from '@/components/report/ReportInterval'; import { ReportLineType } from '@/components/report/ReportLineType'; @@ -74,7 +74,9 @@ export default function ReportEditor({
- {report.ready && } + {report.ready && ( + + )}
diff --git a/apps/web/src/app/(app)/[organizationId]/[projectId]/test/page.tsx b/apps/web/src/app/(app)/[organizationId]/[projectId]/test/page.tsx new file mode 100644 index 00000000..d03b06c1 --- /dev/null +++ b/apps/web/src/app/(app)/[organizationId]/[projectId]/test/page.tsx @@ -0,0 +1,21 @@ +import { Funnel } from '@/components/report/funnel'; + +import PageLayout from '../page-layout'; + +export const metadata = { + title: 'Funnel - Openpanel.dev', +}; + +interface PageProps { + params: { + organizationId: string; + }; +} + +export default function Page({ params: { organizationId } }: PageProps) { + return ( + + + + ); +} diff --git a/apps/web/src/components/overview/overview-live-histogram.tsx b/apps/web/src/components/overview/overview-live-histogram.tsx index cf39f154..78d3741b 100644 --- a/apps/web/src/components/overview/overview-live-histogram.tsx +++ b/apps/web/src/components/overview/overview-live-histogram.tsx @@ -6,7 +6,7 @@ import AnimateHeight from 'react-animate-height'; import type { IChartInput } from '@mixan/validation'; -import { Chart } from '../report/chart'; +import { ChartSwitch } from '../report/chart'; import { Widget, WidgetBody, WidgetHead } from '../Widget'; import { useOverviewOptions } from './useOverviewOptions'; @@ -61,7 +61,7 @@ export function OverviewLiveHistogram({ - + diff --git a/apps/web/src/components/overview/overview-top-devices.tsx b/apps/web/src/components/overview/overview-top-devices.tsx index 246f3c64..3dad209f 100644 --- a/apps/web/src/components/overview/overview-top-devices.tsx +++ b/apps/web/src/components/overview/overview-top-devices.tsx @@ -1,6 +1,6 @@ 'use client'; -import { Chart } from '@/components/report/chart'; +import { ChartSwitch } from '@/components/report/chart'; import { useEventQueryFilters } from '@/hooks/useEventQueryFilters'; import { cn } from '@/utils/cn'; @@ -178,7 +178,7 @@ export default function OverviewTopDevices({ - - + diff --git a/apps/web/src/components/overview/overview-top-geo.tsx b/apps/web/src/components/overview/overview-top-geo.tsx index b058bdf3..1ef83184 100644 --- a/apps/web/src/components/overview/overview-top-geo.tsx +++ b/apps/web/src/components/overview/overview-top-geo.tsx @@ -1,6 +1,6 @@ 'use client'; -import { Chart } from '@/components/report/chart'; +import { ChartSwitch } from '@/components/report/chart'; import { useEventQueryFilters } from '@/hooks/useEventQueryFilters'; import { cn } from '@/utils/cn'; @@ -148,7 +148,7 @@ export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) { - - - ; + } + + if (chartType === 'map') { + return ; + } + + if (chartType === 'histogram') { + return ; + } + + if (chartType === 'bar') { + return ; + } + + if (chartType === 'metric') { + return ; + } + + if (chartType === 'pie') { + return ; + } + + if (chartType === 'linear') { + return ( + + ); + } + + if (chartType === 'area') { + return ( + + ); + } + + return

Unknown chart type

; +} diff --git a/apps/web/src/components/report/chart/LazyChart.tsx b/apps/web/src/components/report/chart/LazyChart.tsx index 2fd3cb89..32dc42bc 100644 --- a/apps/web/src/components/report/chart/LazyChart.tsx +++ b/apps/web/src/components/report/chart/LazyChart.tsx @@ -1,10 +1,10 @@ 'use client'; -import React, { Suspense, useEffect, useRef } from 'react'; +import React, { useEffect, useRef } from 'react'; import { useInViewport } from 'react-in-viewport'; import type { ReportChartProps } from '.'; -import { Chart } from '.'; +import { ChartSwitch } from '.'; import { ChartLoading } from './ChartLoading'; import type { ChartContextType } from './ChartProvider'; @@ -24,7 +24,7 @@ export function LazyChart(props: ReportChartProps & ChartContextType) { return (
{once.current || inViewport ? ( - + ) : ( )} diff --git a/apps/web/src/components/report/chart/MetricCard.tsx b/apps/web/src/components/report/chart/MetricCard.tsx index dc1034bb..de57b954 100644 --- a/apps/web/src/components/report/chart/MetricCard.tsx +++ b/apps/web/src/components/report/chart/MetricCard.tsx @@ -62,7 +62,7 @@ export function MetricCard({
{serie.event.id} - {serie.name ?? serie.event.displayName ?? serie.event.name} + {serie.name || serie.event.displayName || serie.event.name}
diff --git a/apps/web/src/components/report/chart/index.tsx b/apps/web/src/components/report/chart/index.tsx index 2923cf46..8dcecfcf 100644 --- a/apps/web/src/components/report/chart/index.tsx +++ b/apps/web/src/components/report/chart/index.tsx @@ -1,98 +1,19 @@ 'use client'; -import { memo, useEffect, useState } from 'react'; -import type { RouterOutputs } from '@/app/_trpc/client'; -import { api } from '@/app/_trpc/client'; - import type { IChartInput } from '@mixan/validation'; -import { ChartEmpty } from './ChartEmpty'; -import { ChartLoading } from './ChartLoading'; +import { Funnel } from '../funnel'; +import { Chart } from './Chart'; import { withChartProivder } from './ChartProvider'; -import { ReportAreaChart } from './ReportAreaChart'; -import { ReportBarChart } from './ReportBarChart'; -import { ReportHistogramChart } from './ReportHistogramChart'; -import { ReportLineChart } from './ReportLineChart'; -import { ReportMapChart } from './ReportMapChart'; -import { ReportMetricChart } from './ReportMetricChart'; -import { ReportPieChart } from './ReportPieChart'; -export type ReportChartProps = IChartInput & { - initialData?: RouterOutputs['chart']['chart']; -}; +export type ReportChartProps = IChartInput; -export const Chart = withChartProivder(function Chart({ - interval, - events, - breakdowns, - chartType, - name, - range, - lineType, - previous, - formula, - unit, - metric, - projectId, -}: ReportChartProps) { - const [data] = api.chart.chart.useSuspenseQuery( - { - // dont send lineType since it does not need to be sent - lineType: 'monotone', - interval, - chartType, - events, - breakdowns, - name, - range, - startDate: null, - endDate: null, - projectId, - previous, - formula, - unit, - metric, - }, - { - keepPreviousData: true, - } - ); - - if (data.series.length === 0) { - return ; +export const ChartSwitch = withChartProivder(function ChartSwitch( + props: ReportChartProps +) { + if (props.chartType === 'funnel') { + return ; } - if (chartType === 'map') { - return ; - } - - if (chartType === 'histogram') { - return ; - } - - if (chartType === 'bar') { - return ; - } - - if (chartType === 'metric') { - return ; - } - - if (chartType === 'pie') { - return ; - } - - if (chartType === 'linear') { - return ( - - ); - } - - if (chartType === 'area') { - return ( - - ); - } - - return

Unknown chart type

; + return ; }); diff --git a/apps/web/src/components/report/funnel/Funnel.tsx b/apps/web/src/components/report/funnel/Funnel.tsx new file mode 100644 index 00000000..7ff99eec --- /dev/null +++ b/apps/web/src/components/report/funnel/Funnel.tsx @@ -0,0 +1,169 @@ +'use client'; + +import type { RouterOutputs } from '@/app/_trpc/client'; +import { + Carousel, + CarouselContent, + CarouselItem, + CarouselNext, + CarouselPrevious, +} from '@/components/ui/carousel'; +import { cn } from '@/utils/cn'; +import { round } from '@/utils/math'; +import { ArrowRight, ArrowRightIcon } from 'lucide-react'; + +function FunnelChart({ from, to }: { from: number; to: number }) { + const fromY = 100 - from; + const toY = 100 - to; + const steps = [ + `M0,${fromY}`, + 'L0,100', + 'L100,100', + `L100,${toY}`, + `L0,${fromY}`, + ]; + return ( + + + + {/* bottom */} + + {/* top */} + + + + {/* bottom */} + + {/* top */} + + + + + + + ); +} + +function getDropoffColor(value: number) { + if (value > 80) { + return 'text-red-600'; + } + if (value > 50) { + return 'text-orange-600'; + } + if (value > 30) { + return 'text-yellow-600'; + } + return 'text-green-600'; +} + +export function FunnelSteps({ + steps, + totalSessions, +}: RouterOutputs['chart']['funnel']) { + return ( + + + + {steps.map((step, index, list) => { + const finalStep = index === list.length - 1; + return ( + +
+
+

Step {index + 1}

+

+ {step.event.displayName || step.event.name} +

+
+
+ +
+
+ Sessions +
+
+ + {step.before} + + + {step.current} +
+ {index !== 0 && ( + <> +
+ {step.current} of {totalSessions} ( + {round(step.percent, 1)}%) +
+ + )} +
+
+ {finalStep ? ( +
+
+ Conversion +
+
+ {round(step.percent, 1)}% +
+
+ Converted {step.current} of {totalSessions} sessions +
+
+ ) : ( +
+
Dropoff
+
+ {round(step.dropoff.percent, 1)}% +
+
+ Lost {step.dropoff.count} sessions +
+
+ )} +
+ + ); + })} + + + + + + ); +} diff --git a/apps/web/src/components/report/funnel/index.tsx b/apps/web/src/components/report/funnel/index.tsx new file mode 100644 index 00000000..0b6c9a21 --- /dev/null +++ b/apps/web/src/components/report/funnel/index.tsx @@ -0,0 +1,53 @@ +'use client'; + +import type { RouterOutputs } from '@/app/_trpc/client'; +import { api } from '@/app/_trpc/client'; + +import type { IChartInput } from '@mixan/validation'; + +import { ChartEmpty } from '../chart/ChartEmpty'; +import { withChartProivder } from '../chart/ChartProvider'; +import { FunnelSteps } from './Funnel'; + +export type ReportChartProps = IChartInput & { + initialData?: RouterOutputs['chart']['funnel']; +}; + +export const Funnel = withChartProivder(function Chart({ + events, + name, + range, + projectId, +}: ReportChartProps) { + const [data] = api.chart.funnel.useSuspenseQuery( + { + events, + name, + range, + projectId, + lineType: 'monotone', + interval: 'day', + chartType: 'funnel', + breakdowns: [], + startDate: null, + endDate: null, + previous: false, + formula: undefined, + unit: undefined, + metric: 'sum', + }, + { + keepPreviousData: true, + } + ); + + if (data.steps.length === 0) { + return ; + } + + return ( +
+ +
+ ); +}); diff --git a/apps/web/src/components/report/sidebar/ReportSidebar.tsx b/apps/web/src/components/report/sidebar/ReportSidebar.tsx index 7fc31b2d..5e64107e 100644 --- a/apps/web/src/components/report/sidebar/ReportSidebar.tsx +++ b/apps/web/src/components/report/sidebar/ReportSidebar.tsx @@ -1,16 +1,20 @@ import { Button } from '@/components/ui/button'; import { SheetClose } from '@/components/ui/sheet'; +import { useSelector } from '@/redux'; import { ReportBreakdowns } from './ReportBreakdowns'; import { ReportEvents } from './ReportEvents'; import { ReportForumula } from './ReportForumula'; export function ReportSidebar() { + const { chartType } = useSelector((state) => state.report); + const showForumula = chartType !== 'funnel'; + const showBreakdown = chartType !== 'funnel'; return (
- - + {showForumula && } + {showBreakdown && }
diff --git a/apps/web/src/components/ui/carousel.tsx b/apps/web/src/components/ui/carousel.tsx new file mode 100644 index 00000000..cc9bb130 --- /dev/null +++ b/apps/web/src/components/ui/carousel.tsx @@ -0,0 +1,258 @@ +import * as React from 'react'; +import { Button } from '@/components/ui/button'; +import { cn } from '@/utils/cn'; +import useEmblaCarousel from 'embla-carousel-react'; +import type { UseEmblaCarouselType } from 'embla-carousel-react'; +import { ArrowLeft, ArrowRight } from 'lucide-react'; + +type CarouselApi = UseEmblaCarouselType[1]; +type UseCarouselParameters = Parameters; +type CarouselOptions = UseCarouselParameters[0]; +type CarouselPlugin = UseCarouselParameters[1]; + +interface CarouselProps { + opts?: CarouselOptions; + plugins?: CarouselPlugin; + orientation?: 'horizontal' | 'vertical'; + setApi?: (api: CarouselApi) => void; +} + +type CarouselContextProps = { + carouselRef: ReturnType[0]; + api: ReturnType[1]; + scrollPrev: () => void; + scrollNext: () => void; + canScrollPrev: boolean; + canScrollNext: boolean; +} & CarouselProps; + +const CarouselContext = React.createContext(null); + +function useCarousel() { + const context = React.useContext(CarouselContext); + + if (!context) { + throw new Error('useCarousel must be used within a '); + } + + return context; +} + +const Carousel = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes & CarouselProps +>( + ( + { + orientation = 'horizontal', + opts, + setApi, + plugins, + className, + children, + ...props + }, + ref + ) => { + const [carouselRef, api] = useEmblaCarousel( + { + ...opts, + axis: orientation === 'horizontal' ? 'x' : 'y', + }, + plugins + ); + const [canScrollPrev, setCanScrollPrev] = React.useState(false); + const [canScrollNext, setCanScrollNext] = React.useState(false); + + const onSelect = React.useCallback((api: CarouselApi) => { + if (!api) { + return; + } + + setCanScrollPrev(api.canScrollPrev()); + setCanScrollNext(api.canScrollNext()); + }, []); + + const scrollPrev = React.useCallback(() => { + api?.scrollPrev(); + }, [api]); + + const scrollNext = React.useCallback(() => { + api?.scrollNext(); + }, [api]); + + const handleKeyDown = React.useCallback( + (event: React.KeyboardEvent) => { + if (event.key === 'ArrowLeft') { + event.preventDefault(); + scrollPrev(); + } else if (event.key === 'ArrowRight') { + event.preventDefault(); + scrollNext(); + } + }, + [scrollPrev, scrollNext] + ); + + React.useEffect(() => { + if (!api || !setApi) { + return; + } + + setApi(api); + }, [api, setApi]); + + React.useEffect(() => { + if (!api) { + return; + } + + onSelect(api); + api.on('reInit', onSelect); + api.on('select', onSelect); + + return () => { + api?.off('select', onSelect); + }; + }, [api, onSelect]); + + return ( + +
+ {children} +
+ + ); + } +); +Carousel.displayName = 'Carousel'; + +const CarouselContent = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => { + const { carouselRef, orientation } = useCarousel(); + + return ( +
+
+
+ ); +}); +CarouselContent.displayName = 'CarouselContent'; + +const CarouselItem = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => { + const { orientation } = useCarousel(); + + return ( +
+ ); +}); +CarouselItem.displayName = 'CarouselItem'; + +const CarouselPrevious = React.forwardRef< + HTMLButtonElement, + React.ComponentProps +>(({ className, variant = 'outline', size = 'icon', ...props }, ref) => { + const { orientation, scrollPrev, canScrollPrev } = useCarousel(); + + return ( + + ); +}); +CarouselPrevious.displayName = 'CarouselPrevious'; + +const CarouselNext = React.forwardRef< + HTMLButtonElement, + React.ComponentProps +>(({ className, variant = 'outline', size = 'icon', ...props }, ref) => { + const { orientation, scrollNext, canScrollNext } = useCarousel(); + + return ( + + ); +}); +CarouselNext.displayName = 'CarouselNext'; + +export { + type CarouselApi, + Carousel, + CarouselContent, + CarouselItem, + CarouselPrevious, + CarouselNext, +}; diff --git a/apps/web/src/server/api/routers/chart.ts b/apps/web/src/server/api/routers/chart.ts index 8bd1e4e3..e2ae525c 100644 --- a/apps/web/src/server/api/routers/chart.ts +++ b/apps/web/src/server/api/routers/chart.ts @@ -5,7 +5,7 @@ import { } from '@/server/api/trpc'; import { getDaysOldDate } from '@/utils/date'; import { average, max, min, round, sum } from '@/utils/math'; -import { flatten, map, pipe, prop, sort, uniq } from 'ramda'; +import { flatten, map, pipe, prop, repeat, reverse, sort, uniq } from 'ramda'; import { z } from 'zod'; import { chQuery, createSqlBuilder } from '@mixan/db'; @@ -14,6 +14,116 @@ import type { IChartEvent, IChartInput, IChartRange } from '@mixan/validation'; import { getChartData, withFormula } from './chart.helpers'; +async function getFunnelData(payload: IChartInput) { + if (payload.events.length === 0) { + return { + totalSessions: 0, + steps: [], + }; + } + const sql = `SELECT + level, + count() AS count + FROM + ( + SELECT + session_id, + windowFunnel(6048000000000000,'strict_increase')(toUnixTimestamp(created_at), ${payload.events.map((event) => `name = '${event.name}'`).join(', ')}) AS level + FROM events + WHERE (created_at >= '2024-02-24') AND (created_at <= '2024-02-25') + GROUP BY session_id + ) + GROUP BY level + ORDER BY level DESC; + `; + + const [funnelRes, sessionRes] = await Promise.all([ + chQuery<{ level: number; count: number }>(sql), + chQuery<{ count: number }>( + `SELECT count(name) as count FROM events WHERE name = 'session_start' AND (created_at >= '2024-02-24') AND (created_at <= '2024-02-25')` + ), + ]); + + console.log('Funnel SQL: ', sql); + + if (funnelRes[0]?.level !== payload.events.length) { + funnelRes.unshift({ + level: payload.events.length, + count: 0, + }); + } + + const totalSessions = sessionRes[0]?.count ?? 0; + const filledFunnelRes = funnelRes.reduce( + (acc, item, index) => { + const diff = + index !== 0 ? (acc[acc.length - 1]?.level ?? 0) - item.level : 1; + + if (diff > 1) { + acc.push( + ...reverse( + repeat({}, diff - 1).map((_, index) => ({ + count: acc[acc.length - 1]?.count ?? 0, + level: item.level + index + 1, + })) + ) + ); + } + + return [ + ...acc, + { + count: item.count + (acc[acc.length - 1]?.count ?? 0), + level: item.level, + }, + ]; + }, + [] as typeof funnelRes + ); + + const steps = reverse(filledFunnelRes) + .filter((item) => item.level !== 0) + .reduce( + (acc, item, index, list) => { + const prev = list[index - 1] ?? { count: totalSessions }; + return [ + ...acc, + { + event: payload.events[item.level - 1]!, + before: prev.count, + current: item.count, + dropoff: { + bajs: { + prev, + item, + }, + count: prev.count - item.count, + percent: 100 - (item.count / prev.count) * 100, + }, + percent: (item.count / totalSessions) * 100, + prevPercent: (prev.count / totalSessions) * 100, + }, + ]; + }, + [] as { + event: IChartEvent; + before: number; + current: number; + dropoff: { + count: number; + percent: number; + }; + percent: number; + prevPercent: number; + }[] + ); + + return { + totalSessions, + steps, + }; +} + type PreviousValue = { value: number; diff: number | null; @@ -144,6 +254,10 @@ export const chartRouter = createTRPCRouter({ }; }), + funnel: publicProcedure.input(zChartInput).query(async ({ input }) => { + return getFunnelData(input); + }), + // TODO: Make this private chart: publicProcedure.input(zChartInput).query(async ({ input }) => { const current = getDatesFromRange(input.range); diff --git a/apps/worker/package.json b/apps/worker/package.json index eda22e82..02054a8d 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "scripts": { "dev": "dotenv -e ../../.env -c -v WATCH=1 tsup", + "testing": "WORKER_PORT=9999 pnpm dev", "start": "node dist/index.js", "build": "rm -rf dist && tsup", "lint": "eslint .", diff --git a/packages/constants/index.ts b/packages/constants/index.ts index aa420396..b5751948 100644 --- a/packages/constants/index.ts +++ b/packages/constants/index.ts @@ -15,6 +15,7 @@ export const chartTypes = { metric: 'Metric', area: 'Area', map: 'Map', + funnel: 'Funnel', } as const; export const lineTypes = { diff --git a/packages/db/clickhouse_tables.sql b/packages/db/clickhouse_tables.sql index 6ecde1e8..bd19af32 100644 --- a/packages/db/clickhouse_tables.sql +++ b/packages/db/clickhouse_tables.sql @@ -4,6 +4,7 @@ CREATE TABLE openpanel.events ( `device_id` String, `profile_id` String, `project_id` String, + `session_id` String, `path` String, `referrer` String, `referrer_name` String, @@ -56,9 +57,9 @@ ORDER BY ALTER TABLE events ADD - COLUMN device_id String + COLUMN session_id String AFTER - name; + project_id; ALTER TABLE events DROP COLUMN id; @@ -69,4 +70,4 @@ CREATE TABLE ba ( `b` String ) ENGINE MergeTree ORDER BY - (a, b) SETTINGS index_granularity = 8192; + (a, b) SETTINGS index_granularity = 8192; \ No newline at end of file diff --git a/packages/db/prisma/migrations/20240223193217_add_funnel_chart_type/migration.sql b/packages/db/prisma/migrations/20240223193217_add_funnel_chart_type/migration.sql new file mode 100644 index 00000000..d6440c99 --- /dev/null +++ b/packages/db/prisma/migrations/20240223193217_add_funnel_chart_type/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "ChartType" ADD VALUE 'funnel'; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 791a4990..92799467 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -98,6 +98,7 @@ enum ChartType { metric area map + funnel } model Dashboard { diff --git a/packages/db/scripts/test-funnel.ts b/packages/db/scripts/test-funnel.ts new file mode 100644 index 00000000..9bc50ba8 --- /dev/null +++ b/packages/db/scripts/test-funnel.ts @@ -0,0 +1,101 @@ +import { createEvent } from '../src/services/event.service'; + +function c(name: string, createdAt: Date, session_id: string) { + return createEvent({ + name, + deviceId: '', + profileId: '', + projectId: '', + sessionId: session_id, + properties: {}, + createdAt, + country: '', + city: '', + region: '', + continent: '', + os: '', + osVersion: '', + browser: '', + browserVersion: '', + device: '', + brand: '', + model: '', + duration: 0, + path: '/', + referrer: '', + referrerName: '', + referrerType: '', + profile: undefined, + meta: undefined, + }); +} + +async function main() { + // Level 5 + const s = Math.random().toString(36).substring(7); + await c('session_start', new Date('2024-02-24 10:10:00'), s); + + // // Level 2 + // s = 's2'; + // await c('session_start', new Date('2024-02-24 10:10:00'), ''); + // await c('a', new Date('2024-02-24 10:10:00'), s); + // await c('b', new Date('2024-02-24 10:10:02'), s); + + // // Level 5 + // s = 's3'; + // await c('session_start', new Date('2024-02-24 10:10:00'), ''); + // await c('a', new Date('2024-02-24 10:10:00'), s); + // await c('b', new Date('2024-02-24 10:10:02'), s); + // await c('c', new Date('2024-02-24 10:10:03'), s); + // await c('d', new Date('2024-02-24 10:10:04'), s); + // await c('f', new Date('2024-02-24 10:10:10'), s); + + // // Level 4 + // s = 's4'; + // await c('session_start', new Date('2024-02-24 10:10:00'), ''); + // await c('a', new Date('2024-02-24 10:10:00'), s); + // await c('b', new Date('2024-02-24 10:10:02'), s); + // await c('c', new Date('2024-02-24 10:10:03'), s); + // await c('d', new Date('2024-02-24 10:10:04'), s); + + // // Level 3 + // s = 's5'; + // await c('session_start', new Date('2024-02-24 10:10:00'), ''); + // await c('a', new Date('2024-02-24 10:10:00'), s); + // await c('b', new Date('2024-02-24 10:10:02'), s); + // await c('c', new Date('2024-02-24 10:10:03'), s); + + // // Level 3 + // s = 's6'; + // await c('session_start', new Date('2024-02-24 10:10:00'), ''); + // await c('a', new Date('2024-02-24 10:10:00'), s); + // await c('b', new Date('2024-02-24 10:10:02'), s); + // await c('c', new Date('2024-02-24 10:10:03'), s); + + // // Level 2 + // s = 's7'; + // await c('session_start', new Date('2024-02-24 10:10:00'), ''); + // await c('a', new Date('2024-02-24 10:10:00'), s); + // await c('b', new Date('2024-02-24 10:10:02'), s); + + // // Level 5 + // s = 's8'; + // await c('session_start', new Date('2024-02-24 10:10:00'), ''); + // await c('a', new Date('2024-02-24 10:10:00'), s); + // await c('b', new Date('2024-02-24 10:10:02'), s); + // await c('c', new Date('2024-02-24 10:10:03'), s); + // await c('d', new Date('2024-02-24 10:10:04'), s); + // await c('f', new Date('2024-02-24 10:10:10'), s); + + // // Level 4 + // s = 's9'; + // await c('session_start', new Date('2024-02-24 10:10:00'), ''); + // await c('a', new Date('2024-02-24 10:10:00'), s); + // await c('b', new Date('2024-02-24 10:10:02'), s); + // await c('c', new Date('2024-02-24 10:10:03'), s); + // await c('d', new Date('2024-02-24 10:10:04'), s); + + process.exit(); +} + +main(); diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts index 23d1a555..a8f9ca3f 100644 --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -24,6 +24,7 @@ export interface IClickhouseEvent { device_id: string; profile_id: string; project_id: string; + session_id: string; path: string; referrer: string; referrer_name: string; @@ -56,6 +57,7 @@ export function transformEvent( deviceId: event.device_id, profileId: event.profile_id, projectId: event.project_id, + sessionId: event.session_id, properties: event.properties, createdAt: convertClickhouseDateToJs(event.created_at), country: event.country, @@ -84,6 +86,7 @@ export interface IServiceCreateEventPayload { deviceId: string; profileId: string; projectId: string; + sessionId: string; properties: Record & { hash?: string; query?: Record; @@ -162,7 +165,7 @@ export async function createEvent( ); const exists = await getProfileById(payload.profileId); - if (!exists) { + if (!exists && payload.profileId !== '') { const { firstName, lastName } = randomSplitName(); await upsertProfile({ id: payload.profileId, @@ -198,6 +201,7 @@ export async function createEvent( device_id: payload.deviceId, profile_id: payload.profileId, project_id: payload.projectId, + session_id: payload.sessionId, properties: toDots(omit(['_path'], payload.properties)), path: payload.path ?? '', created_at: formatClickhouseDate(payload.createdAt), diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.ts index ffdfcdac..4f680c9e 100644 --- a/packages/db/src/services/profile.service.ts +++ b/packages/db/src/services/profile.service.ts @@ -6,6 +6,10 @@ import { createSqlBuilder } from '../sql-builder'; import { getEventFiltersWhereClause } from './chart.service'; export async function getProfileById(id: string) { + if (id === '') { + return null; + } + const [profile] = await chQuery( `SELECT * FROM profiles WHERE id = '${id}' ORDER BY created_at DESC LIMIT 1` ); diff --git a/packages/db/src/sql-builder.ts b/packages/db/src/sql-builder.ts index e9611b09..d17352b2 100644 --- a/packages/db/src/sql-builder.ts +++ b/packages/db/src/sql-builder.ts @@ -14,7 +14,7 @@ export function createSqlBuilder() { const sb: SqlBuilderObject = { where: {}, - from: 'openpanel.events', + from: 'events', select: {}, groupBy: {}, orderBy: {}, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b278c30d..e66aff18 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -201,6 +201,9 @@ importers: ua-parser-js: specifier: ^1.0.37 version: 1.0.37 + uuid: + specifier: ^9.0.1 + version: 9.0.1 devDependencies: '@mixan/eslint-config': specifier: workspace:* @@ -220,6 +223,9 @@ importers: '@types/ua-parser-js': specifier: ^0.7.39 version: 0.7.39 + '@types/uuid': + specifier: ^9.0.8 + version: 9.0.8 '@types/ws': specifier: ^8.5.10 version: 8.5.10 @@ -407,6 +413,9 @@ importers: cmdk: specifier: ^0.2.1 version: 0.2.1(@types/react@18.2.56)(react-dom@18.2.0)(react@18.2.0) + embla-carousel-react: + specifier: 8.0.0-rc22 + version: 8.0.0-rc22(react@18.2.0) hamburger-react: specifier: ^2.5.0 version: 2.5.0(react@18.2.0)