From df32bb04a01196eaadc0fd647fef3a2f8cb50b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl-Gerhard=20Lindesva=CC=88rd?= Date: Mon, 8 Sep 2025 22:07:09 +0200 Subject: [PATCH] feat(api): add insights endpoints --- .../src/controllers/insights.controller.ts | 178 +++++++ apps/api/src/index.ts | 2 + apps/api/src/routes/insights.router.ts | 89 ++++ .../overview/overview-top-devices.tsx | 1 - .../overview/overview-top-generic-modal.tsx | 3 - .../components/overview/overview-top-geo.tsx | 5 - .../overview/overview-top-pages-modal.tsx | 1 - .../overview/overview-top-pages.tsx | 5 - .../overview/overview-top-sources.tsx | 1 - .../content/docs/api/authentication.mdx | 63 +++ apps/public/content/docs/api/export.mdx | 470 +++++++++++++----- apps/public/content/docs/api/insights.mdx | 405 +++++++++++++++ apps/public/content/pages/contact.mdx | 4 +- packages/constants/index.ts | 5 +- packages/db/src/services/chart.service.ts | 241 ++++++++- packages/db/src/services/overview.service.ts | 13 +- packages/trpc/src/routers/chart.helpers.ts | 243 +-------- packages/trpc/src/routers/chart.ts | 10 +- packages/trpc/src/routers/event.ts | 3 +- packages/trpc/src/routers/overview.ts | 6 +- packages/trpc/src/routers/reference.ts | 8 +- 21 files changed, 1340 insertions(+), 416 deletions(-) create mode 100644 apps/api/src/controllers/insights.controller.ts create mode 100644 apps/api/src/routes/insights.router.ts create mode 100644 apps/public/content/docs/api/authentication.mdx create mode 100644 apps/public/content/docs/api/insights.mdx diff --git a/apps/api/src/controllers/insights.controller.ts b/apps/api/src/controllers/insights.controller.ts new file mode 100644 index 00000000..e1f0a913 --- /dev/null +++ b/apps/api/src/controllers/insights.controller.ts @@ -0,0 +1,178 @@ +import { parseQueryString } from '@/utils/parse-zod-query-string'; +import { getDefaultIntervalByDates } from '@openpanel/constants'; +import { + eventBuffer, + getChartStartEndDate, + getSettingsForProject, + overviewService, +} from '@openpanel/db'; +import { zChartEventFilter, zRange } from '@openpanel/validation'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import { z } from 'zod'; + +const zGetMetricsQuery = z.object({ + startDate: z.string().nullish(), + endDate: z.string().nullish(), + range: zRange.default('7d'), + filters: z.array(zChartEventFilter).default([]), +}); +// Website stats - main metrics overview +export async function getMetrics( + request: FastifyRequest<{ + Params: { projectId: string }; + Querystring: z.infer; + }>, + reply: FastifyReply, +) { + const { timezone } = await getSettingsForProject(request.params.projectId); + const parsed = zGetMetricsQuery.safeParse(parseQueryString(request.query)); + + if (parsed.success === false) { + return reply.status(400).send({ + error: 'Bad Request', + message: 'Invalid query parameters', + details: parsed.error, + }); + } + + const { startDate, endDate } = getChartStartEndDate(parsed.data, timezone); + + reply.send( + await overviewService.getMetrics({ + projectId: request.params.projectId, + filters: parsed.data.filters, + startDate: startDate, + endDate: endDate, + interval: getDefaultIntervalByDates(startDate, endDate) ?? 'day', + timezone, + }), + ); +} + +// Live visitors (real-time) +export async function getLiveVisitors( + request: FastifyRequest<{ + Params: { projectId: string }; + }>, + reply: FastifyReply, +) { + reply.send({ + visitors: await eventBuffer.getActiveVisitorCount(request.params.projectId), + }); +} + +export const zGetTopPagesQuery = z.object({ + filters: z.array(zChartEventFilter).default([]), + startDate: z.string().nullish(), + endDate: z.string().nullish(), + range: zRange.default('7d'), + cursor: z.number().optional(), + limit: z.number().default(10), +}); + +// Page views with top pages +export async function getPages( + request: FastifyRequest<{ + Params: { projectId: string }; + Querystring: z.infer; + }>, + reply: FastifyReply, +) { + const { timezone } = await getSettingsForProject(request.params.projectId); + const { startDate, endDate } = getChartStartEndDate(request.query, timezone); + const parsed = zGetTopPagesQuery.safeParse(parseQueryString(request.query)); + + if (parsed.success === false) { + return reply.status(400).send({ + error: 'Bad Request', + message: 'Invalid query parameters', + details: parsed.error, + }); + } + + return overviewService.getTopPages({ + projectId: request.params.projectId, + filters: parsed.data.filters, + startDate: startDate, + endDate: endDate, + timezone, + cursor: parsed.data.cursor, + limit: Math.min(parsed.data.limit, 50), + }); +} + +const zGetOverviewGenericQuery = z.object({ + filters: z.array(zChartEventFilter).default([]), + startDate: z.string().nullish(), + endDate: z.string().nullish(), + range: zRange.default('7d'), + column: z.enum([ + // Referrers + 'referrer', + 'referrer_name', + 'referrer_type', + 'utm_source', + 'utm_medium', + 'utm_campaign', + 'utm_term', + 'utm_content', + // Geo + 'region', + 'country', + 'city', + // Device + 'device', + 'brand', + 'model', + 'browser', + 'browser_version', + 'os', + 'os_version', + ]), + cursor: z.number().optional(), + limit: z.number().default(10), +}); + +export function getOverviewGeneric( + column: z.infer['column'], +) { + return async ( + request: FastifyRequest<{ + Params: { projectId: string; key: string }; + Querystring: z.infer; + }>, + reply: FastifyReply, + ) => { + const { timezone } = await getSettingsForProject(request.params.projectId); + const { startDate, endDate } = getChartStartEndDate( + request.query, + timezone, + ); + const parsed = zGetOverviewGenericQuery.safeParse({ + ...parseQueryString(request.query), + column, + }); + + if (parsed.success === false) { + return reply.status(400).send({ + error: 'Bad Request', + message: 'Invalid query parameters', + details: parsed.error, + }); + } + + // TODO: Implement overview generic endpoint + reply.send( + await overviewService.getTopGeneric({ + column, + projectId: request.params.projectId, + filters: parsed.data.filters, + startDate: startDate, + endDate: endDate, + timezone, + cursor: parsed.data.cursor, + limit: Math.min(parsed.data.limit, 50), + }), + ); + }; +} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 9be940a4..5226a8a6 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -32,6 +32,7 @@ import aiRouter from './routes/ai.router'; import eventRouter from './routes/event.router'; import exportRouter from './routes/export.router'; import importRouter from './routes/import.router'; +import insightsRouter from './routes/insights.router'; import liveRouter from './routes/live.router'; import miscRouter from './routes/misc.router'; import oauthRouter from './routes/oauth-callback.router'; @@ -169,6 +170,7 @@ const startServer = async () => { instance.register(profileRouter, { prefix: '/profile' }); instance.register(exportRouter, { prefix: '/export' }); instance.register(importRouter, { prefix: '/import' }); + instance.register(insightsRouter, { prefix: '/insights' }); instance.register(trackRouter, { prefix: '/track' }); instance.get('/healthcheck', healthcheck); instance.get('/healthcheck/queue', healthcheckQueue); diff --git a/apps/api/src/routes/insights.router.ts b/apps/api/src/routes/insights.router.ts new file mode 100644 index 00000000..ccd2dd39 --- /dev/null +++ b/apps/api/src/routes/insights.router.ts @@ -0,0 +1,89 @@ +import * as controller from '@/controllers/insights.controller'; +import { validateExportRequest } from '@/utils/auth'; +import { activateRateLimiter } from '@/utils/rate-limiter'; +import { Prisma } from '@openpanel/db'; +import type { FastifyPluginCallback, FastifyRequest } from 'fastify'; + +const insightsRouter: FastifyPluginCallback = async (fastify) => { + await activateRateLimiter({ + fastify, + max: 100, + timeWindow: '10 seconds', + }); + + fastify.addHook('preHandler', async (req: FastifyRequest, reply) => { + try { + const client = await validateExportRequest(req.headers); + req.client = client; + } catch (e) { + if (e instanceof Prisma.PrismaClientKnownRequestError) { + return reply.status(401).send({ + error: 'Unauthorized', + message: 'Client ID seems to be malformed', + }); + } + + if (e instanceof Error) { + return reply + .status(401) + .send({ error: 'Unauthorized', message: e.message }); + } + + return reply + .status(401) + .send({ error: 'Unauthorized', message: 'Unexpected error' }); + } + }); + + // Website stats - main metrics overview + fastify.route({ + method: 'GET', + url: '/:projectId/metrics', + handler: controller.getMetrics, + }); + + // Live visitors (real-time) + fastify.route({ + method: 'GET', + url: '/:projectId/live', + handler: controller.getLiveVisitors, + }); + + // Page views with top pages + fastify.route({ + method: 'GET', + url: '/:projectId/pages', + handler: controller.getPages, + }); + + const overviewMetrics = [ + 'referrer_name', + 'referrer', + 'referrer_type', + 'utm_source', + 'utm_medium', + 'utm_campaign', + 'utm_term', + 'utm_content', + 'device', + 'browser', + 'browser_version', + 'os', + 'os_version', + 'brand', + 'model', + 'country', + 'region', + 'city', + ] as const; + + overviewMetrics.forEach((key) => { + fastify.route({ + method: 'GET', + url: `/:projectId/${key}`, + handler: controller.getOverviewGeneric(key), + }); + }); +}; + +export default insightsRouter; diff --git a/apps/dashboard/src/components/overview/overview-top-devices.tsx b/apps/dashboard/src/components/overview/overview-top-devices.tsx index c28adc9b..b4081088 100644 --- a/apps/dashboard/src/components/overview/overview-top-devices.tsx +++ b/apps/dashboard/src/components/overview/overview-top-devices.tsx @@ -313,7 +313,6 @@ export default function OverviewTopDevices({ const query = api.overview.topGeneric.useQuery({ projectId, - interval, range, filters, column: widget.key, diff --git a/apps/dashboard/src/components/overview/overview-top-generic-modal.tsx b/apps/dashboard/src/components/overview/overview-top-generic-modal.tsx index 87fa49d4..1f4c4443 100644 --- a/apps/dashboard/src/components/overview/overview-top-generic-modal.tsx +++ b/apps/dashboard/src/components/overview/overview-top-generic-modal.tsx @@ -6,8 +6,6 @@ import { ModalContent, ModalHeader } from '@/modals/Modal/Container'; import { api } from '@/trpc/client'; import type { IGetTopGenericInput } from '@openpanel/db'; import { ChevronRightIcon } from 'lucide-react'; -import { useEffect } from 'react'; -import { toast } from 'sonner'; import { SerieIcon } from '../report-chart/common/serie-icon'; import { Button } from '../ui/button'; import { ScrollArea } from '../ui/scroll-area'; @@ -36,7 +34,6 @@ export default function OverviewTopGenericModal({ startDate, endDate, range, - interval, limit: 50, column, }, diff --git a/apps/dashboard/src/components/overview/overview-top-geo.tsx b/apps/dashboard/src/components/overview/overview-top-geo.tsx index d0ef3cdc..40a240ed 100644 --- a/apps/dashboard/src/components/overview/overview-top-geo.tsx +++ b/apps/dashboard/src/components/overview/overview-top-geo.tsx @@ -2,9 +2,6 @@ import { useEventQueryFilters } from '@/hooks/useEventQueryFilters'; import { cn } from '@/utils/cn'; -import { useState } from 'react'; - -import type { IChartType } from '@openpanel/validation'; import { useNumber } from '@/hooks/useNumerFormatter'; import { pushModal } from '@/modals'; @@ -30,7 +27,6 @@ interface OverviewTopGeoProps { export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) { const { interval, range, previous, startDate, endDate } = useOverviewOptions(); - const [chartType, setChartType] = useState('bar'); const [filters, setFilter] = useEventQueryFilters(); const isPageFilter = filters.find((filter) => filter.name === 'path'); const [widget, setWidget, widgets] = useOverviewWidgetV2('geo', { @@ -52,7 +48,6 @@ export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) { const query = api.overview.topGeneric.useQuery({ projectId, - interval, range, filters, column: widget.key, diff --git a/apps/dashboard/src/components/overview/overview-top-pages-modal.tsx b/apps/dashboard/src/components/overview/overview-top-pages-modal.tsx index ea3da7f5..2c7aeb35 100644 --- a/apps/dashboard/src/components/overview/overview-top-pages-modal.tsx +++ b/apps/dashboard/src/components/overview/overview-top-pages-modal.tsx @@ -34,7 +34,6 @@ export default function OverviewTopPagesModal({ endDate, mode: 'page', range, - interval: 'day', limit: 50, }, { diff --git a/apps/dashboard/src/components/overview/overview-top-pages.tsx b/apps/dashboard/src/components/overview/overview-top-pages.tsx index c6f3b126..4061d120 100644 --- a/apps/dashboard/src/components/overview/overview-top-pages.tsx +++ b/apps/dashboard/src/components/overview/overview-top-pages.tsx @@ -4,9 +4,6 @@ import { useEventQueryFilters } from '@/hooks/useEventQueryFilters'; import { cn } from '@/utils/cn'; import { Globe2Icon } from 'lucide-react'; import { parseAsBoolean, useQueryState } from 'nuqs'; -import { useState } from 'react'; - -import type { IChartType } from '@openpanel/validation'; import { pushModal } from '@/modals'; import { api } from '@/trpc/client'; @@ -15,7 +12,6 @@ import { Widget, WidgetBody } from '../widget'; import OverviewDetailsButton from './overview-details-button'; import { WidgetButtons, WidgetFooter, WidgetHead } from './overview-widget'; import { - OverviewWidgetTableBots, OverviewWidgetTableLoading, OverviewWidgetTablePages, } from './overview-widget-table'; @@ -72,7 +68,6 @@ export default function OverviewTopPages({ projectId }: OverviewTopPagesProps) { endDate, mode: widget.key, range, - interval, }); const data = query.data; diff --git a/apps/dashboard/src/components/overview/overview-top-sources.tsx b/apps/dashboard/src/components/overview/overview-top-sources.tsx index 400fb857..cddbfe40 100644 --- a/apps/dashboard/src/components/overview/overview-top-sources.tsx +++ b/apps/dashboard/src/components/overview/overview-top-sources.tsx @@ -63,7 +63,6 @@ export default function OverviewTopSources({ const query = api.overview.topGeneric.useQuery({ projectId, - interval, range, filters, column: widget.key, diff --git a/apps/public/content/docs/api/authentication.mdx b/apps/public/content/docs/api/authentication.mdx new file mode 100644 index 00000000..f2d6a054 --- /dev/null +++ b/apps/public/content/docs/api/authentication.mdx @@ -0,0 +1,63 @@ +--- +title: Authentication +description: Learn how to authenticate with the OpenPanel API using client credentials. +--- + +## Authentication + +To authenticate with the OpenPanel API, you need to use your `clientId` and `clientSecret`. Different API endpoints may require different access levels: + +- **Track API**: Default client works with `track` mode +- **Export API**: Requires `read` or `root` mode +- **Insights API**: Requires `read` or `root` mode + +The default client does not have access to the Export or Insights APIs. + +## Headers + +Include the following headers with your API requests: + +- `openpanel-client-id`: Your OpenPanel client ID +- `openpanel-client-secret`: Your OpenPanel client secret + +## Example + +```bash +curl 'https://api.openpanel.dev/insights/{projectId}/metrics' \ + -H 'openpanel-client-id: YOUR_CLIENT_ID' \ + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +``` + +## Security Best Practices + +1. **Store credentials securely**: Never expose your `clientId` and `clientSecret` in client-side code +2. **Use HTTPS**: Always use HTTPS to ensure secure communication +3. **Rotate credentials**: Regularly rotate your API credentials +4. **Limit access**: Use the minimum required access level for your use case + +## Error Responses + +If authentication fails, you'll receive a `401 Unauthorized` response: + +```json +{ + "error": "Unauthorized", + "message": "Invalid client credentials" +} +``` + +Common authentication errors: +- Invalid client ID or secret +- Client doesn't have required permissions +- Malformed client ID + +## Rate Limiting + +The API implements rate limiting to prevent abuse. Rate limits vary by endpoint: + +- **Track API**: Higher limits for event tracking +- **Export/Insights APIs**: Lower limits for data retrieval + +If you exceed the rate limit, you'll receive a `429 Too Many Requests` response. Implement exponential backoff for retries. + +Remember to replace `YOUR_CLIENT_ID` and `YOUR_CLIENT_SECRET` with your actual OpenPanel API credentials. diff --git a/apps/public/content/docs/api/export.mdx b/apps/public/content/docs/api/export.mdx index 4d69c150..491133d1 100644 --- a/apps/public/content/docs/api/export.mdx +++ b/apps/public/content/docs/api/export.mdx @@ -1,222 +1,418 @@ --- title: Export -description: The Export API allows you to retrieve event data and chart data from your OpenPanel projects. +description: The Export API allows you to retrieve event data and chart data from your OpenPanel projects for analysis, reporting, and data integration. --- ## Authentication To authenticate with the Export API, you need to use your `clientId` and `clientSecret`. Make sure your client has `read` or `root` mode. The default client does not have access to the Export API. +For detailed authentication information, see the [Authentication](/docs/api/authentication) guide. + Include the following headers with your requests: -- `openpanel-client-id`: Your OpenPanel client ID +- `openpanel-client-id`: Your OpenPanel client ID - `openpanel-client-secret`: Your OpenPanel client secret -Example: +## Base URL -```bash -curl 'https://api.openpanel.dev/export/events' \ - -H 'openpanel-client-id: YOUR_CLIENT_ID' \ - -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +All Export API requests should be made to: + +``` +https://api.openpanel.dev/export ``` -## Events +## Common Query Parameters -Get events from a specific project within a date range. +Most endpoints support the following query parameters: -Endpoint: `GET /export/events` +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `projectId` | string | The ID of the project (alternative: `project_id`) | Required | +| `startDate` | string | Start date (ISO format: YYYY-MM-DD) | Based on range | +| `endDate` | string | End date (ISO format: YYYY-MM-DD) | Based on range | +| `range` | string | Predefined date range (`7d`, `30d`, `today`, etc.) | None | -Parameters: -- project_id (required): The ID of the project -- event (optional): Filter by event name(s). Can be a single event or an array of events. -- start (optional): Start date (format: YYYY-MM-DD) -- end (optional): End date (format: YYYY-MM-DD) -- page (optional, default: 1): Page number for pagination -- limit (optional, default: 50, max: 50): Number of events per page -- includes (optional): Additional fields to include in the response +## Endpoints -Example: +### Get Events -```bash -curl 'https://api.openpanel.dev/export/events?project_id=abc&event=screen_view&start=2024-04-15&end=2024-04-18' \ - -H 'openpanel-client-id: YOUR_CLIENT_ID' \ - -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +Retrieve individual events from a specific project within a date range. This endpoint provides raw event data with optional filtering and pagination. + +``` +GET /export/events ``` -### Query Parameters +#### Query Parameters | Parameter | Type | Description | Example | |-----------|------|-------------|---------| -| projectId | string | The ID of the project to fetch events from | `abc123` | -| event | string or string[] | Event name(s) to filter | `screen_view` or `["screen_view","button_click"]` | -| start | string | Start date for the event range (ISO format) | `2024-04-15` | -| end | string | End date for the event range (ISO format) | `2024-04-18` | -| page | number | Page number for pagination (default: 1) | `2` | -| limit | number | Number of events per page (default: 50, max: 50) | `25` | -| includes | string or string[] | Additional fields to include in the response | `profile` or `["profile","meta"]` | +| `projectId` | string | The ID of the project to fetch events from | `abc123` | +| `profileId` | string | Filter events by specific profile/user ID | `user_123` | +| `event` | string or string[] | Event name(s) to filter | `screen_view` or `["screen_view","button_click"]` | +| `start` | string | Start date for the event range (ISO format) | `2024-04-15` | +| `end` | string | End date for the event range (ISO format) | `2024-04-18` | +| `page` | number | Page number for pagination (default: 1) | `2` | +| `limit` | number | Number of events per page (default: 50, max: 1000) | `100` | +| `includes` | string or string[] | Additional fields to include in the response | `profile` or `["profile","meta"]` | -### Example Request +#### Include Options + +The `includes` parameter allows you to fetch additional related data: + +- `profile`: Include user profile information +- `meta`: Include event metadata and configuration + +#### Example Request ```bash -curl 'https://api.openpanel.dev/export/events?project_id=abc123&event=screen_view&start=2024-04-15&end=2024-04-18&page=1&limit=50&includes=profile,meta' \ +curl 'https://api.openpanel.dev/export/events?projectId=abc123&event=screen_view&start=2024-04-15&end=2024-04-18&page=1&limit=100&includes=profile,meta' \ -H 'openpanel-client-id: YOUR_CLIENT_ID' \ -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' ``` -### Response +#### Response ```json { "meta": { - "count": number, - "totalCount": number, - "pages": number, - "current": number + "count": 50, + "totalCount": 1250, + "pages": 25, + "current": 1 }, - "data": Array + "data": [ + { + "id": "evt_123456789", + "name": "screen_view", + "deviceId": "device_abc123", + "profileId": "user_789", + "projectId": "abc123", + "sessionId": "session_xyz", + "properties": { + "path": "/dashboard", + "title": "Dashboard", + "url": "https://example.com/dashboard" + }, + "createdAt": "2024-04-15T10:30:00.000Z", + "country": "United States", + "city": "New York", + "region": "New York", + "os": "macOS", + "browser": "Chrome", + "device": "Desktop", + "duration": 0, + "path": "/dashboard", + "origin": "https://example.com", + "profile": { + "id": "user_789", + "email": "user@example.com", + "firstName": "John", + "lastName": "Doe", + "isExternal": true, + "createdAt": "2024-04-01T08:00:00.000Z" + }, + "meta": { + "name": "screen_view", + "description": "Page view tracking", + "conversion": false + } + } + ] } ``` -## Charts +### Get Charts -Retrieve chart data for a specific project. - -### Endpoint +Retrieve aggregated chart data for analytics and visualization. This endpoint provides time-series data with advanced filtering, breakdowns, and comparison capabilities. ``` GET /export/charts ``` -### Query Parameters +#### Query Parameters | Parameter | Type | Description | Example | |-----------|------|-------------|---------| -| projectId | string | The ID of the project to fetch chart data from | `abc123` | -| events | string[] | Array of event configurations to include in the chart | `[{"name":"sign_up","filters":[]}]` | -| breakdowns | object[] | Array of breakdown configurations | `[{"name":"country"}]` | -| interval | string | Time interval for data points | `day` | -| range | string | Predefined date range | `last_7_days` | -| previous | boolean | Include data from the previous period | `true` | -| startDate | string | Custom start date (ISO format) | `2024-04-01` | -| endDate | string | Custom end date (ISO format) | `2024-04-30` | -| chartType | string | Type of chart to generate | `linear` | -| metric | string | Metric to use for calculations | `sum` | -| limit | number | Limit the number of results | `10` | -| offset | number | Offset for pagination | `0` | +| `projectId` | string | The ID of the project to fetch chart data from | `abc123` | +| `events` | object[] | Array of event configurations to analyze | `[{"name":"screen_view","filters":[]}]` | +| `breakdowns` | object[] | Array of breakdown dimensions | `[{"name":"country"}]` | +| `interval` | string | Time interval for data points | `day` | +| `range` | string | Predefined date range | `7d` | +| `previous` | boolean | Include data from the previous period for comparison | `true` | +| `startDate` | string | Custom start date (ISO format) | `2024-04-01` | +| `endDate` | string | Custom end date (ISO format) | `2024-04-30` | -#### Events configuration +#### Event Configuration -Each event configuration object has the following properties: +Each event in the `events` array supports the following properties: + +| Property | Type | Description | Required | Default | +|----------|------|-------------|----------|---------| +| `name` | string | Name of the event to track | Yes | - | +| `filters` | Filter[] | Array of filters to apply to the event | No | `[]` | +| `segment` | string | Type of segmentation | No | `event` | +| `property` | string | Property name for property-based segments | No | - | + +#### Segmentation Options + +- `event`: Count individual events (default) +- `user`: Count unique users/profiles +- `session`: Count unique sessions +- `user_average`: Average events per user +- `one_event_per_user`: One event per user (deduplicated) +- `property_sum`: Sum of a numeric property +- `property_average`: Average of a numeric property +- `property_min`: Minimum value of a numeric property +- `property_max`: Maximum value of a numeric property + +#### Filter Configuration + +Each filter in the `filters` array supports: | Property | Type | Description | Required | |----------|------|-------------|----------| -| name | string | Name of the event to track | Yes | -| filters | Filter[] | Array of filters to apply to the event | No | -| segment | string | Type of segmentation. Options: `event`, `user`, `session`, `user_average`, `one_event_per_user`, `property_sum`, `property_average` | No (defaults to `event`) | -| property | string | Property name to analyze when using property-based segments | No | +| `name` | string | Property name to filter on | Yes | +| `operator` | string | Comparison operator | Yes | +| `value` | array | Array of values to compare against | Yes | -##### Filter Configuration +#### Filter Operators -Each filter in the `filters` array has the following structure: +- `is`: Exact match +- `isNot`: Not equal to +- `contains`: Contains substring +- `doesNotContain`: Does not contain substring +- `startsWith`: Starts with +- `endsWith`: Ends with +- `regex`: Regular expression match +- `isNull`: Property is null or empty +- `isNotNull`: Property has a value -| Property | Type | Description | Required | -|----------|------|-------------|----------| -| name | string | Name of the property to filter on | Yes | -| operator | string | Comparison operator. Valid values: `is`, `isNot`, `contains`, `doesNotContain`, `startsWith`, `endsWith`, `regex` | Yes | -| value | (string \| number \| boolean \| null)[] | Array of values to compare against | Yes | +#### Breakdown Dimensions -Example event configuration: -```json -{ - "name": "purchase", - "segment": "user", - "filters": [ - { - "name": "total", - "operator": "is", - "value": ["100"] - } - ] -} -``` +Common breakdown dimensions include: -The operators are used in the SQL builder (`chart.service.ts` lines 262-346) with the following mappings: -- `is`: Equals comparison -- `isNot`: Not equals comparison -- `contains`: LIKE %value% -- `doesNotContain`: NOT LIKE %value% -- `startsWith`: LIKE value% -- `endsWith`: LIKE %value -- `regex`: Match function +| Dimension | Description | Example Values | +|-----------|-------------|----------------| +| `country` | User's country | `United States`, `Canada` | +| `region` | User's region/state | `California`, `New York` | +| `city` | User's city | `San Francisco`, `New York` | +| `device` | Device type | `Desktop`, `Mobile`, `Tablet` | +| `browser` | Browser name | `Chrome`, `Firefox`, `Safari` | +| `os` | Operating system | `macOS`, `Windows`, `iOS` | +| `referrer` | Referrer URL | `google.com`, `facebook.com` | +| `path` | Page path | `/`, `/dashboard`, `/pricing` | -### Example Request +#### Time Intervals + +- `minute`: Minute-by-minute data +- `hour`: Hourly aggregation +- `day`: Daily aggregation (default) +- `week`: Weekly aggregation +- `month`: Monthly aggregation + +#### Date Ranges + +- `30min`: Last 30 minutes +- `lastHour`: Last hour +- `today`: Current day +- `yesterday`: Previous day +- `7d`: Last 7 days +- `30d`: Last 30 days +- `6m`: Last 6 months +- `12m`: Last 12 months +- `monthToDate`: Current month to date +- `lastMonth`: Previous month +- `yearToDate`: Current year to date +- `lastYear`: Previous year + +#### Example Request ```bash -curl 'https://api.openpanel.dev/export/charts?projectId=abc123&events=[{"name":"screen_view"}]&interval=day&range=last_30_days&chartType=linear&metric=sum' \ +curl 'https://api.openpanel.dev/export/charts?projectId=abc123&events=[{"name":"screen_view","segment":"user"}]&breakdowns=[{"name":"country"}]&interval=day&range=30d&previous=true' \ -H 'openpanel-client-id: YOUR_CLIENT_ID' \ -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' ``` -### Response - -The response will include chart data with series, metrics, and optional previous period comparisons based on the input parameters. - -## Funnel - -Retrieve funnel data for a specific project. - -### Endpoint - -``` -GET /export/funnel -``` - -### Query Parameters - -| Parameter | Type | Description | Example | -|-----------|------|-------------|---------| -| projectId | string | The ID of the project to fetch funnel data from | `abc123` | -| events | object[] | Array of event configurations for the funnel steps | `[{"name":"sign_up","filters":[]}]` | -| range | string | Predefined date range | `last_30_days` | -| startDate | string | Custom start date (ISO format) | `2024-04-01` | -| endDate | string | Custom end date (ISO format) | `2024-04-30` | - -### Example Request +#### Example Advanced Request ```bash -curl 'https://api.openpanel.dev/export/funnel?projectId=abc123&events=[{"name":"sign_up"},{"name":"purchase"}]&range=last_30_days' \ +curl 'https://api.openpanel.dev/export/charts' \ -H 'openpanel-client-id: YOUR_CLIENT_ID' \ - -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' \ + -G \ + --data-urlencode 'projectId=abc123' \ + --data-urlencode 'events=[{"name":"purchase","segment":"property_sum","property":"properties.total","filters":[{"name":"properties.total","operator":"isNotNull","value":[]}]}]' \ + --data-urlencode 'breakdowns=[{"name":"country"}]' \ + --data-urlencode 'interval=day' \ + --data-urlencode 'range=30d' ``` -### Response - -The response will include funnel data with total sessions and step-by-step breakdown of the funnel progression. +#### Response ```json { - "totalSessions": number, - "steps": [ + "series": [ { + "id": "screen_view-united-states", + "names": ["screen_view", "United States"], "event": { - "name": string, - "displayName": string + "id": "evt1", + "name": "screen_view" }, - "count": number, - "percent": number, - "dropoffCount": number, - "dropoffPercent": number, - "previousCount": number + "metrics": { + "sum": 1250, + "average": 41.67, + "min": 12, + "max": 89, + "previous": { + "sum": { + "value": 1100, + "change": 13.64 + }, + "average": { + "value": 36.67, + "change": 13.64 + } + } + }, + "data": [ + { + "date": "2024-04-01T00:00:00.000Z", + "count": 45, + "previous": { + "value": 38, + "change": 18.42 + } + }, + { + "date": "2024-04-02T00:00:00.000Z", + "count": 52, + "previous": { + "value": 41, + "change": 26.83 + } + } + ] + } + ], + "metrics": { + "sum": 1250, + "average": 41.67, + "min": 12, + "max": 89, + "previous": { + "sum": { + "value": 1100, + "change": 13.64 + } + } + } +} +``` + +## Error Handling + +The API uses standard HTTP response codes. Common error responses: + +### 400 Bad Request + +```json +{ + "error": "Bad Request", + "message": "Invalid query parameters", + "details": [ + { + "path": ["events", 0, "name"], + "message": "Required" } ] } ``` +### 401 Unauthorized + +```json +{ + "error": "Unauthorized", + "message": "Invalid client credentials" +} +``` + +### 403 Forbidden + +```json +{ + "error": "Forbidden", + "message": "You do not have access to this project" +} +``` + +### 404 Not Found + +```json +{ + "error": "Not Found", + "message": "Project not found" +} +``` + +### 429 Too Many Requests + +Rate limiting response includes headers indicating your rate limit status. + +## Rate Limiting + +The Export API implements rate limiting: +- **100 requests per 10 seconds** per client +- Rate limit headers included in responses +- Implement exponential backoff for retries + +## Data Types and Formats + +### Event Properties + +Event properties are stored as key-value pairs and can include: + +- **Built-in properties**: `path`, `origin`, `title`, `url`, `hash` +- **UTM parameters**: `utm_source`, `utm_medium`, `utm_campaign`, `utm_term`, `utm_content` +- **Custom properties**: Any custom data you track with your events + +### Property Access + +Properties can be accessed in filters and breakdowns using dot notation: + +- `properties.custom_field`: Access custom properties +- `profile.properties.user_type`: Access profile properties +- `properties.__query.utm_source`: Access query parameters + +### Date Handling + +- All dates are in ISO 8601 format +- Timezone handling is done server-side based on project settings +- Date ranges are inclusive of start and end dates + +### Geographic Data + +Geographic information is automatically collected when available: + +- `country`: Full country name +- `region`: State/province/region +- `city`: City name +- `longitude`/`latitude`: Coordinates (when available) + +### Device Information + +Device data is collected from user agents: + +- `device`: Device type (Desktop, Mobile, Tablet) +- `browser`: Browser name and version +- `os`: Operating system and version +- `brand`/`model`: Device brand and model (mobile devices) + ## Notes -- All date parameters should be in ISO format (YYYY-MM-DD). -- The `range` parameter accepts values like `today`, `yesterday`, `last_7_days`, `last_30_days`, `this_month`, `last_month`, `this_year`, `last_year`, `all_time`. -- The `interval` parameter accepts values like `minute`, `hour`, `day`, `month`. -- The `chartType` parameter can be `linear` or other supported chart types. -- The `metric` parameter can be `sum`, `average`, `min`, or `max`. +- Event data is typically available within seconds of tracking +- All timezone handling is done server-side based on project settings +- Property names are case-sensitive in filters and breakdowns Remember to replace `YOUR_CLIENT_ID` and `YOUR_CLIENT_SECRET` with your actual OpenPanel API credentials. \ No newline at end of file diff --git a/apps/public/content/docs/api/insights.mdx b/apps/public/content/docs/api/insights.mdx new file mode 100644 index 00000000..9d800ac1 --- /dev/null +++ b/apps/public/content/docs/api/insights.mdx @@ -0,0 +1,405 @@ +--- +title: Insights +description: The Insights API provides access to website analytics data including metrics, page views, visitor statistics, and detailed breakdowns by various dimensions. +--- + +## Authentication + +To authenticate with the Insights API, you need to use your `clientId` and `clientSecret`. Make sure your client has `read` or `root` mode. The default client does not have access to the Insights API. + +For detailed authentication information, see the [Authentication](/docs/api/authentication) guide. + +Include the following headers with your requests: +- `openpanel-client-id`: Your OpenPanel client ID +- `openpanel-client-secret`: Your OpenPanel client secret + +## Base URL + +All Insights API requests should be made to: + +``` +https://api.openpanel.dev/insights +``` + +## Common Query Parameters + +Most endpoints support the following query parameters: + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `startDate` | string | Start date (ISO format: YYYY-MM-DD) | Based on range | +| `endDate` | string | End date (ISO format: YYYY-MM-DD) | Based on range | +| `range` | string | Predefined date range (`7d`, `30d`, `90d`, etc.) | `7d` | +| `filters` | array | Event filters to apply | `[]` | +| `cursor` | number | Page number for pagination | `1` | +| `limit` | number | Number of results per page (max: 50) | `10` | + +### Filter Configuration + +Filters can be applied to narrow down results. Each filter has the following structure: + +```json +{ + "name": "property_name", + "operator": "is|isNot|contains|doesNotContain|startsWith|endsWith|regex", + "value": ["value1", "value2"] +} +``` + +## Endpoints + +### Get Metrics + +Retrieve comprehensive website metrics including visitors, sessions, page views, and engagement data. + +``` +GET /insights/{projectId}/metrics +``` + +#### Query Parameters + +| Parameter | Type | Description | Example | +|-----------|------|-------------|---------| +| `startDate` | string | Start date for metrics | `2024-01-01` | +| `endDate` | string | End date for metrics | `2024-01-31` | +| `range` | string | Predefined range | `7d` | +| `filters` | array | Event filters | `[{"name":"path","operator":"is","value":["/home"]}]` | + +#### Example Request + +```bash +curl 'https://api.openpanel.dev/insights/abc123/metrics?range=30d&filters=[{"name":"path","operator":"contains","value":["/product"]}]' \ + -H 'openpanel-client-id: YOUR_CLIENT_ID' \ + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +``` + +#### Response + +```json +{ + "metrics": { + "bounce_rate": 45.2, + "unique_visitors": 1250, + "total_sessions": 1580, + "avg_session_duration": 185.5, + "total_screen_views": 4230, + "views_per_session": 2.67 + }, + "series": [ + { + "date": "2024-01-01T00:00:00.000Z", + "bounce_rate": 42.1, + "unique_visitors": 85, + "total_sessions": 98, + "avg_session_duration": 195.2, + "total_screen_views": 156, + "views_per_session": 1.59 + } + ] +} +``` + +### Get Live Visitors + +Get the current number of active visitors on your website in real-time. + +``` +GET /insights/{projectId}/live +``` + +#### Example Request + +```bash +curl 'https://api.openpanel.dev/insights/abc123/live' \ + -H 'openpanel-client-id: YOUR_CLIENT_ID' \ + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +``` + +#### Response + +```json +{ + "visitors": 23 +} +``` + +### Get Top Pages + +Retrieve the most visited pages with detailed analytics including session count, bounce rate, and average time on page. + +``` +GET /insights/{projectId}/pages +``` + +#### Query Parameters + +| Parameter | Type | Description | Example | +|-----------|------|-------------|---------| +| `startDate` | string | Start date | `2024-01-01` | +| `endDate` | string | End date | `2024-01-31` | +| `range` | string | Predefined range | `7d` | +| `filters` | array | Event filters | `[]` | +| `cursor` | number | Page number | `1` | +| `limit` | number | Results per page (max: 50) | `10` | + +#### Example Request + +```bash +curl 'https://api.openpanel.dev/insights/abc123/pages?range=7d&limit=20' \ + -H 'openpanel-client-id: YOUR_CLIENT_ID' \ + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +``` + +#### Response + +```json +[ + { + "title": "Homepage - Example Site", + "origin": "https://example.com", + "path": "/", + "sessions": 456, + "bounce_rate": 35.2, + "avg_duration": 125.8 + }, + { + "title": "About Us", + "origin": "https://example.com", + "path": "/about", + "sessions": 234, + "bounce_rate": 45.1, + "avg_duration": 89.3 + } +] +``` + +### Get Referrer Data + +Retrieve referrer analytics to understand where your traffic is coming from. + +``` +GET /insights/{projectId}/referrer +GET /insights/{projectId}/referrer_name +GET /insights/{projectId}/referrer_type +``` + +#### Example Request + +```bash +curl 'https://api.openpanel.dev/insights/abc123/referrer?range=30d&limit=15' \ + -H 'openpanel-client-id: YOUR_CLIENT_ID' \ + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +``` + +#### Response + +```json +[ + { + "name": "google.com", + "sessions": 567, + "bounce_rate": 42.1, + "avg_session_duration": 156.7 + }, + { + "name": "facebook.com", + "sessions": 234, + "bounce_rate": 38.9, + "avg_session_duration": 189.2 + } +] +``` + +### Get UTM Campaign Data + +Analyze your marketing campaigns with UTM parameter breakdowns. + +``` +GET /insights/{projectId}/utm_source +GET /insights/{projectId}/utm_medium +GET /insights/{projectId}/utm_campaign +GET /insights/{projectId}/utm_term +GET /insights/{projectId}/utm_content +``` + +#### Example Request + +```bash +curl 'https://api.openpanel.dev/insights/abc123/utm_source?range=30d' \ + -H 'openpanel-client-id: YOUR_CLIENT_ID' \ + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +``` + +#### Response + +```json +[ + { + "name": "google", + "sessions": 890, + "bounce_rate": 35.4, + "avg_session_duration": 178.9 + }, + { + "name": "facebook", + "sessions": 456, + "bounce_rate": 41.2, + "avg_session_duration": 142.3 + } +] +``` + +### Get Geographic Data + +Understand your audience location with country, region, and city breakdowns. + +``` +GET /insights/{projectId}/country +GET /insights/{projectId}/region +GET /insights/{projectId}/city +``` + +#### Example Request + +```bash +curl 'https://api.openpanel.dev/insights/abc123/country?range=30d&limit=20' \ + -H 'openpanel-client-id: YOUR_CLIENT_ID' \ + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +``` + +#### Response + +```json +[ + { + "name": "United States", + "sessions": 1234, + "bounce_rate": 38.7, + "avg_session_duration": 167.4 + }, + { + "name": "United Kingdom", + "sessions": 567, + "bounce_rate": 42.1, + "avg_session_duration": 145.8 + } +] +``` + +For region and city endpoints, an additional `prefix` field may be included: + +```json +[ + { + "prefix": "United States", + "name": "California", + "sessions": 456, + "bounce_rate": 35.2, + "avg_session_duration": 172.1 + } +] +``` + +### Get Device & Technology Data + +Analyze visitor devices, browsers, and operating systems. + +``` +GET /insights/{projectId}/device +GET /insights/{projectId}/browser +GET /insights/{projectId}/browser_version +GET /insights/{projectId}/os +GET /insights/{projectId}/os_version +GET /insights/{projectId}/brand +GET /insights/{projectId}/model +``` + +#### Example Request + +```bash +curl 'https://api.openpanel.dev/insights/abc123/browser?range=7d' \ + -H 'openpanel-client-id: YOUR_CLIENT_ID' \ + -H 'openpanel-client-secret: YOUR_CLIENT_SECRET' +``` + +#### Response + +```json +[ + { + "name": "Chrome", + "sessions": 789, + "bounce_rate": 36.4, + "avg_session_duration": 162.3 + }, + { + "name": "Firefox", + "sessions": 234, + "bounce_rate": 41.7, + "avg_session_duration": 148.9 + } +] +``` + +For version-specific endpoints (browser_version, os_version), a `prefix` field shows the parent: + +```json +[ + { + "prefix": "Chrome", + "name": "118.0.0.0", + "sessions": 456, + "bounce_rate": 35.8, + "avg_session_duration": 165.7 + } +] +``` + +## Error Handling + +The API uses standard HTTP response codes. Common error responses: + +### 400 Bad Request + +```json +{ + "error": "Bad Request", + "message": "Invalid query parameters", + "details": { + "issues": [ + { + "path": ["range"], + "message": "Invalid enum value" + } + ] + } +} +``` + +### 401 Unauthorized + +```json +{ + "error": "Unauthorized", + "message": "Invalid client credentials" +} +``` + +### 429 Too Many Requests + +Rate limiting response includes headers indicating your rate limit status. + +## Rate Limiting + +The Insights API implements rate limiting: +- **100 requests per 10 seconds** per client +- Rate limit headers included in responses +- Implement exponential backoff for retries + +## Notes + +- All dates are returned in ISO 8601 format +- Durations are in seconds +- Bounce rates and percentages are returned as decimal numbers (e.g., 45.2 = 45.2%) +- Session duration is the average time spent on the website +- All timezone handling is done server-side based on project settings diff --git a/apps/public/content/pages/contact.mdx b/apps/public/content/pages/contact.mdx index 50b405cc..eee59c12 100644 --- a/apps/public/content/pages/contact.mdx +++ b/apps/public/content/pages/contact.mdx @@ -8,9 +8,9 @@ description: Get in touch with the founder of OpenPanel - a simple and affordabl - [Email](mailto:hello@openpanel.dev) - [X (@OpenPanelDev)](https://x.com/OpenPanelDev) - [X (@CarlLindesvard)](https://x.com/CarlLindesvard) -- [Discord](https://discord.gg/openpanel) +- [Discord](https://go.openpanel.dev/discord) - [Github](https://github.com/Openpanel-dev/openpanel/) ## Issues or feature requests -If you have any issues or feature requests, please let me know by [opening an issue on Github](https://github.com/Openpanel-dev/openpanel/issues) or join our [Discord](https://discord.gg/openpanel). \ No newline at end of file +If you have any issues or feature requests, please let me know by [opening an issue on Github](https://github.com/Openpanel-dev/openpanel/issues) or join our [Discord](https://go.openpanel.dev/discord). \ No newline at end of file diff --git a/packages/constants/index.ts b/packages/constants/index.ts index e3f68b23..998cb3a3 100644 --- a/packages/constants/index.ts +++ b/packages/constants/index.ts @@ -1,4 +1,4 @@ -import { isSameDay, isSameMonth } from 'date-fns'; +import { differenceInDays, isSameDay, isSameMonth } from 'date-fns'; export const DEFAULT_ASPECT_RATIO = 0.5625; export const NOT_SET_VALUE = '(not set)'; @@ -232,6 +232,9 @@ export function getDefaultIntervalByDates( if (isSameMonth(startDate, endDate)) { return 'day'; } + if (differenceInDays(endDate, startDate) <= 31) { + return 'day'; + } return 'month'; } diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts index fc53a402..38c7813e 100644 --- a/packages/db/src/services/chart.service.ts +++ b/packages/db/src/services/chart.service.ts @@ -1,8 +1,10 @@ import { escape } from 'sqlstring'; -import { stripLeadingAndTrailingSlashes } from '@openpanel/common'; +import { DateTime, stripLeadingAndTrailingSlashes } from '@openpanel/common'; import type { IChartEventFilter, + IChartInput, + IChartRange, IGetChartDataInput, } from '@openpanel/validation'; @@ -441,3 +443,240 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) { return where; } + +export function getChartStartEndDate( + { + startDate, + endDate, + range, + }: Pick, + timezone: string, +) { + const ranges = getDatesFromRange(range, timezone); + + if (startDate && endDate) { + return { startDate: startDate, endDate: endDate }; + } + + if (!startDate && endDate) { + return { startDate: ranges.startDate, endDate: endDate }; + } + + return ranges; +} + +export function getDatesFromRange(range: IChartRange, timezone: string) { + if (range === '30min' || range === 'lastHour') { + const minutes = range === '30min' ? 30 : 60; + const startDate = DateTime.now() + .minus({ minute: minutes }) + .startOf('minute') + .setZone(timezone) + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .setZone(timezone) + .endOf('minute') + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === 'today') { + const startDate = DateTime.now() + .setZone(timezone) + .startOf('day') + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .setZone(timezone) + .endOf('day') + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === 'yesterday') { + const startDate = DateTime.now() + .minus({ day: 1 }) + .setZone(timezone) + .startOf('day') + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .minus({ day: 1 }) + .setZone(timezone) + .endOf('day') + .toFormat('yyyy-MM-dd HH:mm:ss'); + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === '7d') { + const startDate = DateTime.now() + .minus({ day: 7 }) + .setZone(timezone) + .startOf('day') + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .setZone(timezone) + .endOf('day') + .plus({ millisecond: 1 }) + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === '6m') { + const startDate = DateTime.now() + .minus({ month: 6 }) + .setZone(timezone) + .startOf('day') + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .setZone(timezone) + .endOf('day') + .plus({ millisecond: 1 }) + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === '12m') { + const startDate = DateTime.now() + .minus({ month: 12 }) + .setZone(timezone) + .startOf('month') + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .setZone(timezone) + .endOf('month') + .plus({ millisecond: 1 }) + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === 'monthToDate') { + const startDate = DateTime.now() + .setZone(timezone) + .startOf('month') + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .setZone(timezone) + .endOf('day') + .plus({ millisecond: 1 }) + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === 'lastMonth') { + const month = DateTime.now() + .minus({ month: 1 }) + .setZone(timezone) + .startOf('month'); + + const startDate = month.toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = month + .endOf('month') + .plus({ millisecond: 1 }) + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === 'yearToDate') { + const startDate = DateTime.now() + .setZone(timezone) + .startOf('year') + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .setZone(timezone) + .endOf('day') + .plus({ millisecond: 1 }) + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + if (range === 'lastYear') { + const year = DateTime.now().minus({ year: 1 }).setZone(timezone); + const startDate = year.startOf('year').toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = year.endOf('year').toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; + } + + // range === '30d' + const startDate = DateTime.now() + .minus({ day: 30 }) + .setZone(timezone) + .startOf('day') + .toFormat('yyyy-MM-dd HH:mm:ss'); + const endDate = DateTime.now() + .setZone(timezone) + .endOf('day') + .plus({ millisecond: 1 }) + .toFormat('yyyy-MM-dd HH:mm:ss'); + + return { + startDate: startDate, + endDate: endDate, + }; +} + +export function getChartPrevStartEndDate({ + startDate, + endDate, +}: { + startDate: string; + endDate: string; +}) { + let diff = DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss').diff( + DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss'), + ); + + // this will make sure our start and end date's are correct + // otherwise if a day ends with 23:59:59.999 and starts with 00:00:00.000 + // the diff will be 23:59:59.999 and that will make the start date wrong + // so we add 1 millisecond to the diff + if ((diff.milliseconds / 1000) % 2 !== 0) { + diff = diff.plus({ millisecond: 1 }); + } + + return { + startDate: DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss') + .minus({ millisecond: diff.milliseconds }) + .toFormat('yyyy-MM-dd HH:mm:ss'), + endDate: DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss') + .minus({ millisecond: diff.milliseconds }) + .toFormat('yyyy-MM-dd HH:mm:ss'), + }; +} diff --git a/packages/db/src/services/overview.service.ts b/packages/db/src/services/overview.service.ts index 11f101c4..155c1651 100644 --- a/packages/db/src/services/overview.service.ts +++ b/packages/db/src/services/overview.service.ts @@ -24,7 +24,6 @@ export const zGetTopPagesInput = z.object({ filters: z.array(z.any()), startDate: z.string(), endDate: z.string(), - interval: zTimeInterval, cursor: z.number().optional(), limit: z.number().optional(), }); @@ -38,7 +37,6 @@ export const zGetTopEntryExitInput = z.object({ filters: z.array(z.any()), startDate: z.string(), endDate: z.string(), - interval: zTimeInterval, mode: z.enum(['entry', 'exit']), cursor: z.number().optional(), limit: z.number().optional(), @@ -53,7 +51,6 @@ export const zGetTopGenericInput = z.object({ filters: z.array(z.any()), startDate: z.string(), endDate: z.string(), - interval: zTimeInterval, column: z.enum([ // Referrers 'referrer', @@ -168,6 +165,16 @@ export class OverviewService { views_per_session: number; }[]; }> { + console.log('-----------------'); + console.log('getMetrics', { + projectId, + filters, + startDate, + endDate, + interval, + timezone, + }); + const where = this.getRawWhereClause('sessions', filters); if (this.isPageFilter(filters)) { // Session aggregation with bounce rates diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts index 95eb7cde..ce94e6b7 100644 --- a/packages/trpc/src/routers/chart.helpers.ts +++ b/packages/trpc/src/routers/chart.helpers.ts @@ -20,7 +20,9 @@ import { chQuery, createSqlBuilder, formatClickhouseDate, + getChartPrevStartEndDate, getChartSql, + getChartStartEndDate, getEventFiltersWhereClause, getOrganizationSubscriptionChartEndDate, getSettingsForProject, @@ -34,10 +36,6 @@ import type { IGetChartDataInput, } from '@openpanel/validation'; -function getEventLegend(event: IChartEvent) { - return event.displayName || event.name; -} - export function withFormula( { formula, events }: IChartInput, series: Awaited>, @@ -116,193 +114,6 @@ export function withFormula( ]; } -export function getDatesFromRange(range: IChartRange, timezone: string) { - if (range === '30min' || range === 'lastHour') { - const minutes = range === '30min' ? 30 : 60; - const startDate = DateTime.now() - .minus({ minute: minutes }) - .startOf('minute') - .setZone(timezone) - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .setZone(timezone) - .endOf('minute') - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === 'today') { - const startDate = DateTime.now() - .setZone(timezone) - .startOf('day') - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .setZone(timezone) - .endOf('day') - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === 'yesterday') { - const startDate = DateTime.now() - .minus({ day: 1 }) - .setZone(timezone) - .startOf('day') - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .minus({ day: 1 }) - .setZone(timezone) - .endOf('day') - .toFormat('yyyy-MM-dd HH:mm:ss'); - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === '7d') { - const startDate = DateTime.now() - .minus({ day: 7 }) - .setZone(timezone) - .startOf('day') - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .setZone(timezone) - .endOf('day') - .plus({ millisecond: 1 }) - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === '6m') { - const startDate = DateTime.now() - .minus({ month: 6 }) - .setZone(timezone) - .startOf('day') - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .setZone(timezone) - .endOf('day') - .plus({ millisecond: 1 }) - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === '12m') { - const startDate = DateTime.now() - .minus({ month: 12 }) - .setZone(timezone) - .startOf('month') - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .setZone(timezone) - .endOf('month') - .plus({ millisecond: 1 }) - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === 'monthToDate') { - const startDate = DateTime.now() - .setZone(timezone) - .startOf('month') - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .setZone(timezone) - .endOf('day') - .plus({ millisecond: 1 }) - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === 'lastMonth') { - const month = DateTime.now() - .minus({ month: 1 }) - .setZone(timezone) - .startOf('month'); - - const startDate = month.toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = month - .endOf('month') - .plus({ millisecond: 1 }) - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === 'yearToDate') { - const startDate = DateTime.now() - .setZone(timezone) - .startOf('year') - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .setZone(timezone) - .endOf('day') - .plus({ millisecond: 1 }) - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - if (range === 'lastYear') { - const year = DateTime.now().minus({ year: 1 }).setZone(timezone); - const startDate = year.startOf('year').toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = year.endOf('year').toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; - } - - // range === '30d' - const startDate = DateTime.now() - .minus({ day: 30 }) - .setZone(timezone) - .startOf('day') - .toFormat('yyyy-MM-dd HH:mm:ss'); - const endDate = DateTime.now() - .setZone(timezone) - .endOf('day') - .plus({ millisecond: 1 }) - .toFormat('yyyy-MM-dd HH:mm:ss'); - - return { - startDate: startDate, - endDate: endDate, - }; -} - function fillFunnel(funnel: { level: number; count: number }[], steps: number) { const filled = Array.from({ length: steps }, (_, index) => { const level = index + 1; @@ -325,56 +136,6 @@ function fillFunnel(funnel: { level: number; count: number }[], steps: number) { return filled.reverse(); } -export function getChartStartEndDate( - { - startDate, - endDate, - range, - }: Pick, - timezone: string, -) { - const ranges = getDatesFromRange(range, timezone); - - if (startDate && endDate) { - return { startDate: startDate, endDate: endDate }; - } - - if (!startDate && endDate) { - return { startDate: ranges.startDate, endDate: endDate }; - } - - return ranges; -} - -export function getChartPrevStartEndDate({ - startDate, - endDate, -}: { - startDate: string; - endDate: string; -}) { - let diff = DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss').diff( - DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss'), - ); - - // this will make sure our start and end date's are correct - // otherwise if a day ends with 23:59:59.999 and starts with 00:00:00.000 - // the diff will be 23:59:59.999 and that will make the start date wrong - // so we add 1 millisecond to the diff - if ((diff.milliseconds / 1000) % 2 !== 0) { - diff = diff.plus({ millisecond: 1 }); - } - - return { - startDate: DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss') - .minus({ millisecond: diff.milliseconds }) - .toFormat('yyyy-MM-dd HH:mm:ss'), - endDate: DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss') - .minus({ millisecond: diff.milliseconds }) - .toFormat('yyyy-MM-dd HH:mm:ss'), - }; -} - export async function getFunnelData({ projectId, startDate, diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index 922d4b37..056e440e 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -10,13 +10,13 @@ import { chQuery, clix, conversionService, - createSqlBuilder, db, funnelService, + getChartPrevStartEndDate, + getChartStartEndDate, getEventMetasCached, getSelectPropertyKey, getSettingsForProject, - toDate, } from '@openpanel/db'; import { zChartInput, @@ -40,11 +40,7 @@ import { protectedProcedure, publicProcedure, } from '../trpc'; -import { - getChart, - getChartPrevStartEndDate, - getChartStartEndDate, -} from './chart.helpers'; +import { getChart } from './chart.helpers'; function utc(date: string | Date) { if (typeof date === 'string') { diff --git a/packages/trpc/src/routers/event.ts b/packages/trpc/src/routers/event.ts index ce50c220..5496a768 100644 --- a/packages/trpc/src/routers/event.ts +++ b/packages/trpc/src/routers/event.ts @@ -10,6 +10,7 @@ import { db, eventService, formatClickhouseDate, + getChartStartEndDate, getConversionEventNames, getEventList, getEventMetasCached, @@ -28,7 +29,6 @@ import { clone } from 'ramda'; import { getProjectAccessCached } from '../access'; import { TRPCAccessError } from '../errors'; import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc'; -import { getChartStartEndDate } from './chart.helpers'; export const eventRouter = createTRPCRouter({ updateEventMeta: protectedProcedure @@ -289,7 +289,6 @@ export const eventRouter = createTRPCRouter({ filters: input.filters, startDate, endDate, - interval: input.interval, cursor: input.cursor || 1, limit: input.take, timezone, diff --git a/packages/trpc/src/routers/overview.ts b/packages/trpc/src/routers/overview.ts index 46d56888..807820b8 100644 --- a/packages/trpc/src/routers/overview.ts +++ b/packages/trpc/src/routers/overview.ts @@ -1,4 +1,6 @@ import { + getChartPrevStartEndDate, + getChartStartEndDate, getOrganizationSubscriptionChartEndDate, getSettingsForProject, overviewService, @@ -10,10 +12,6 @@ import { type IChartRange, zRange } from '@openpanel/validation'; import { format } from 'date-fns'; import { z } from 'zod'; import { cacheMiddleware, createTRPCRouter, publicProcedure } from '../trpc'; -import { - getChartPrevStartEndDate, - getChartStartEndDate, -} from './chart.helpers'; const cacher = cacheMiddleware((input) => { const range = input.range as IChartRange; diff --git a/packages/trpc/src/routers/reference.ts b/packages/trpc/src/routers/reference.ts index 439230b4..7fbbf2dd 100644 --- a/packages/trpc/src/routers/reference.ts +++ b/packages/trpc/src/routers/reference.ts @@ -1,12 +1,16 @@ import { z } from 'zod'; -import { db, getReferences, getSettingsForProject } from '@openpanel/db'; +import { + db, + getChartStartEndDate, + getReferences, + getSettingsForProject, +} from '@openpanel/db'; import { zCreateReference, zRange } from '@openpanel/validation'; import { getProjectAccess } from '../access'; import { TRPCAccessError } from '../errors'; import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc'; -import { getChartStartEndDate } from './chart.helpers'; export const referenceRouter = createTRPCRouter({ create: protectedProcedure