zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feature(dashboard): add integrations and notifications

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-10-02 22:12:05 +02:00
parent d920f6951c
commit f65a633403
94 changed files with 3692 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/db/index.ts
View File

@@ -15,4 +15,6 @@ export * from './src/services/user.service';
export * from './src/services/reference.service';
export * from './src/services/id.service';
export * from './src/services/retention.service';
export * from './src/services/notification.service';
export * from './src/buffers';
export * from './src/types';

2
packages/db/package.json
View File

@@ -18,9 +18,11 @@
"@openpanel/common": "workspace:*",
"@openpanel/constants": "workspace:*",
"@openpanel/logger": "workspace:*",
"@openpanel/queue": "workspace:^",
"@openpanel/redis": "workspace:*",
"@openpanel/validation": "workspace:*",
"@prisma/client": "^5.1.1",
"prisma-json-types-generator": "^3.1.1",
"ramda": "^0.29.1",
"sqlstring": "^2.3.3",
"superjson": "^1.13.3",

71
packages/db/prisma/migrations/20240925202841_notifications/migration.sql Normal file
View File

@@ -0,0 +1,71 @@
-- CreateEnum
CREATE TYPE "IntegrationType" AS ENUM ('app', 'mail', 'custom');
-- CreateTable
CREATE TABLE "notification_settings" (
"id" UUID NOT NULL DEFAULT gen_random_uuid(),
"projectId" TEXT NOT NULL,
"settings" JSONB NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "notification_settings_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "notifications" (
"id" UUID NOT NULL DEFAULT gen_random_uuid(),
"projectId" TEXT NOT NULL,
"title" TEXT NOT NULL,
"message" TEXT NOT NULL,
"isReadAt" TIMESTAMP(3),
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"integrationId" UUID,
"integrationType" "IntegrationType" NOT NULL,
CONSTRAINT "notifications_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "integrations" (
"id" UUID NOT NULL DEFAULT gen_random_uuid(),
"name" TEXT NOT NULL,
"type" TEXT NOT NULL,
"config" JSONB NOT NULL,
"organizationId" TEXT NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "integrations_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "_IntegrationToNotificationControl" (
"A" UUID NOT NULL,
"B" UUID NOT NULL
);
-- CreateIndex
CREATE UNIQUE INDEX "_IntegrationToNotificationControl_AB_unique" ON "_IntegrationToNotificationControl"("A", "B");
-- CreateIndex
CREATE INDEX "_IntegrationToNotificationControl_B_index" ON "_IntegrationToNotificationControl"("B");
-- AddForeignKey
ALTER TABLE "notification_settings" ADD CONSTRAINT "notification_settings_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "notifications" ADD CONSTRAINT "notifications_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "notifications" ADD CONSTRAINT "notifications_integrationId_fkey" FOREIGN KEY ("integrationId") REFERENCES "integrations"("id") ON DELETE SET NULL ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "integrations" ADD CONSTRAINT "integrations_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "_IntegrationToNotificationControl" ADD CONSTRAINT "_IntegrationToNotificationControl_A_fkey" FOREIGN KEY ("A") REFERENCES "integrations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "_IntegrationToNotificationControl" ADD CONSTRAINT "_IntegrationToNotificationControl_B_fkey" FOREIGN KEY ("B") REFERENCES "notification_settings"("id") ON DELETE CASCADE ON UPDATE CASCADE;

14
packages/db/prisma/migrations/20240925204316_notification_send_to_app_and_email/migration.sql Normal file
View File

@@ -0,0 +1,14 @@
/*
Warnings:
- You are about to drop the column `integrationType` on the `notifications` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "notification_settings" ADD COLUMN "sendToApp" BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN "sendToEmail" BOOLEAN NOT NULL DEFAULT false;
-- AlterTable
ALTER TABLE "notifications" DROP COLUMN "integrationType",
ADD COLUMN "sendToApp" BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN "sendToEmail" BOOLEAN NOT NULL DEFAULT false;

14
packages/db/prisma/migrations/20240926175415_renaming/migration.sql Normal file
View File

@@ -0,0 +1,14 @@
/*
Warnings:
- You are about to drop the column `type` on the `integrations` table. All the data in the column will be lost.
- You are about to drop the column `settings` on the `notification_settings` table. All the data in the column will be lost.
- Added the required column `config` to the `notification_settings` table without a default value. This is not possible if the table is not empty.
*/
-- AlterTable
ALTER TABLE "integrations" DROP COLUMN "type";
-- AlterTable
ALTER TABLE "notification_settings" DROP COLUMN "settings",
ADD COLUMN "config" JSONB NOT NULL;

55
packages/db/prisma/migrations/20240927190558_rename_notification_controls/migration.sql Normal file
View File

@@ -0,0 +1,55 @@
/*
Warnings:
- You are about to drop the `_IntegrationToNotificationControl` table. If the table is not empty, all the data it contains will be lost.
- You are about to drop the `notification_settings` table. If the table is not empty, all the data it contains will be lost.
*/
-- DropForeignKey
ALTER TABLE "_IntegrationToNotificationControl" DROP CONSTRAINT "_IntegrationToNotificationControl_A_fkey";
-- DropForeignKey
ALTER TABLE "_IntegrationToNotificationControl" DROP CONSTRAINT "_IntegrationToNotificationControl_B_fkey";
-- DropForeignKey
ALTER TABLE "notification_settings" DROP CONSTRAINT "notification_settings_projectId_fkey";
-- DropTable
DROP TABLE "_IntegrationToNotificationControl";
-- DropTable
DROP TABLE "notification_settings";
-- CreateTable
CREATE TABLE "notification_rules" (
"id" UUID NOT NULL DEFAULT gen_random_uuid(),
"projectId" TEXT NOT NULL,
"sendToApp" BOOLEAN NOT NULL DEFAULT false,
"sendToEmail" BOOLEAN NOT NULL DEFAULT false,
"config" JSONB NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "notification_rules_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "_IntegrationToNotificationRule" (
"A" UUID NOT NULL,
"B" UUID NOT NULL
);
-- CreateIndex
CREATE UNIQUE INDEX "_IntegrationToNotificationRule_AB_unique" ON "_IntegrationToNotificationRule"("A", "B");
-- CreateIndex
CREATE INDEX "_IntegrationToNotificationRule_B_index" ON "_IntegrationToNotificationRule"("B");
-- AddForeignKey
ALTER TABLE "notification_rules" ADD CONSTRAINT "notification_rules_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "_IntegrationToNotificationRule" ADD CONSTRAINT "_IntegrationToNotificationRule_A_fkey" FOREIGN KEY ("A") REFERENCES "integrations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "_IntegrationToNotificationRule" ADD CONSTRAINT "_IntegrationToNotificationRule_B_fkey" FOREIGN KEY ("B") REFERENCES "notification_rules"("id") ON DELETE CASCADE ON UPDATE CASCADE;

8
packages/db/prisma/migrations/20240927195752_add_name_to_rules/migration.sql Normal file
View File

@@ -0,0 +1,8 @@
/*
Warnings:
- Added the required column `name` to the `notification_rules` table without a default value. This is not possible if the table is not empty.
*/
-- AlterTable
ALTER TABLE "notification_rules" ADD COLUMN "name" TEXT NOT NULL;

2
packages/db/prisma/migrations/20241001215748_add_payload_to_notifications/migration.sql Normal file
View File

@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "notifications" ADD COLUMN "payload" JSONB;

64
packages/db/prisma/schema.prisma
View File

@@ -5,6 +5,10 @@ generator client {
provider = "prisma-client-js"
}
generator json {
provider = "prisma-json-types-generator"
}
datasource db {
provider = "postgresql"
url = env("DATABASE_URL")
@@ -30,6 +34,7 @@ model Organization {
Client Client[]
Dashboard Dashboard[]
ShareOverview ShareOverview[]
integrations Integration[]
@@map("organizations")
}
@@ -87,6 +92,9 @@ model Project {
references Reference[]
access ProjectAccess[]
notificationRules NotificationRule[]
notifications Notification[]
createdAt DateTime @default(now())
updatedAt DateTime @default(now()) @updatedAt
@@ -299,3 +307,59 @@ model Reference {
@@map("references")
}
enum IntegrationType {
app
mail
custom
}
model NotificationRule {
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
name String
projectId String
project Project @relation(fields: [projectId], references: [id])
integrations Integration[]
sendToApp Boolean @default(false)
sendToEmail Boolean @default(false)
/// [IPrismaNotificationRuleConfig]
config Json
createdAt DateTime @default(now())
updatedAt DateTime @default(now()) @updatedAt
@@map("notification_rules")
}
model Notification {
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
projectId String
project Project @relation(fields: [projectId], references: [id])
title String
message String
isReadAt DateTime?
createdAt DateTime @default(now())
updatedAt DateTime @default(now()) @updatedAt
sendToApp Boolean @default(false)
sendToEmail Boolean @default(false)
integration Integration? @relation(fields: [integrationId], references: [id])
integrationId String? @db.Uuid
/// [IPrismaNotificationPayload]
payload Json?
@@map("notifications")
}
model Integration {
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
name String
/// [IPrismaIntegrationConfig]
config Json
organization Organization @relation(fields: [organizationId], references: [id])
organizationId String
notificationRules NotificationRule[]
notifications Notification[]
createdAt DateTime @default(now())
updatedAt DateTime @default(now()) @updatedAt
@@map("integrations")
}

10
packages/db/src/services/chart.service.ts
View File

@@ -1,6 +1,9 @@
import { escape } from 'sqlstring';
import { getTimezoneFromDateString } from '@openpanel/common';
import {
getTimezoneFromDateString,
stripLeadingAndTrailingSlashes,
} from '@openpanel/common';
import type {
IChartEventFilter,
IGetChartDataInput,
@@ -328,7 +331,10 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
}
case 'regex': {
where[id] = value
.map((val) => `match(${name}, ${escape(String(val).trim())})`)
.map(
(val) =>
`match(${name}, ${escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`,
)
.join(' OR ');
break;
}

2
packages/db/src/services/event.service.ts
View File

@@ -17,8 +17,8 @@ import type { EventMeta, Prisma } from '../prisma-client';
import { db } from '../prisma-client';
import { createSqlBuilder } from '../sql-builder';
import { getEventFiltersWhereClause } from './chart.service';
import { getProfiles, upsertProfile } from './profile.service';
import type { IServiceProfile } from './profile.service';
import { getProfiles, upsertProfile } from './profile.service';
export type IImportedEvent = Omit<
IClickhouseEvent,

314
packages/db/src/services/notification.service.ts Normal file
View File

@@ -0,0 +1,314 @@
import { stripLeadingAndTrailingSlashes } from '@openpanel/common';
import { notificationQueue } from '@openpanel/queue';
import { cacheable } from '@openpanel/redis';
import type { IChartEvent, IChartEventFilter } from '@openpanel/validation';
import { pathOr } from 'ramda';
import {
type Integration,
type Notification,
Prisma,
db,
} from '../prisma-client';
import type {
IServiceCreateEventPayload,
IServiceEvent,
} from './event.service';
import { getProjectByIdCached } from './project.service';
type ICreateNotification = Pick<
Notification,
'projectId' | 'title' | 'message' | 'integrationId' | 'payload'
>;
export type INotificationPayload =
| {
type: 'event';
event: IServiceCreateEventPayload;
}
| {
type: 'funnel';
funnel: IServiceEvent[];
};
export const APP_NOTIFICATION_INTEGRATION_ID = 'app';
export const EMAIL_NOTIFICATION_INTEGRATION_ID = 'email';
export const BASE_INTEGRATIONS: Integration[] = [
{
id: APP_NOTIFICATION_INTEGRATION_ID,
name: 'App',
createdAt: new Date(),
updatedAt: new Date(),
config: {
type: APP_NOTIFICATION_INTEGRATION_ID,
},
organizationId: '',
},
{
id: EMAIL_NOTIFICATION_INTEGRATION_ID,
name: 'Email',
createdAt: new Date(),
updatedAt: new Date(),
config: {
type: EMAIL_NOTIFICATION_INTEGRATION_ID,
},
organizationId: '',
},
];
export const isBaseIntegration = (id: string) =>
BASE_INTEGRATIONS.find((i) => i.id === id);
export const getNotificationRulesByProjectId = cacheable(
function getNotificationRulesByProjectId(projectId: string) {
return db.notificationRule.findMany({
where: {
projectId,
},
select: {
id: true,
name: true,
sendToApp: true,
sendToEmail: true,
config: true,
integrations: {
select: {
id: true,
},
},
},
});
},
60 * 24,
);
function getIntegration(integrationId: string | null) {
if (integrationId === APP_NOTIFICATION_INTEGRATION_ID) {
return {
integrationId: null,
sendToApp: true,
sendToEmail: false,
};
}
if (integrationId === EMAIL_NOTIFICATION_INTEGRATION_ID) {
return {
integrationId: null,
sendToApp: false,
sendToEmail: true,
};
}
return {
sendToApp: false,
sendToEmail: false,
integrationId,
};
}
export async function createNotification(notification: ICreateNotification) {
const res = await db.notification.create({
data: {
title: notification.title,
message: notification.message,
projectId: notification.projectId,
payload: notification.payload || Prisma.DbNull,
...getIntegration(notification.integrationId),
},
});
return triggerNotification(res);
}
export function triggerNotification(notification: Notification) {
return notificationQueue.add('sendNotification', {
type: 'sendNotification',
payload: {
notification,
},
});
}
function matchEventFilters(
event: IServiceCreateEventPayload,
filters: IChartEventFilter[],
) {
return filters.every((filter) => {
const { name, value, operator } = filter;
if (value.length === 0) return true;
if (name === 'has_profile') {
if (value.includes('true')) {
return event.profileId !== event.deviceId;
}
return event.profileId === event.deviceId;
}
const propertyValue = (
name.startsWith('properties.')
? pathOr('', name.split('.'), event)
: pathOr('', [name], event)
).trim();
switch (operator) {
case 'is':
return value.includes(propertyValue);
case 'isNot':
return !value.includes(propertyValue);
case 'contains':
return value.some((val) => propertyValue.includes(String(val)));
case 'doesNotContain':
return !value.some((val) => propertyValue.includes(String(val)));
case 'startsWith':
return value.some((val) => propertyValue.startsWith(String(val)));
case 'endsWith':
return value.some((val) => propertyValue.endsWith(String(val)));
case 'regex': {
return value
.map((val) => stripLeadingAndTrailingSlashes(String(val)))
.some((val) => new RegExp(val).test(propertyValue));
}
default:
return false;
}
});
}
export function matchEvent(
event: IServiceCreateEventPayload,
chartEvent: IChartEvent,
) {
if (event.name !== chartEvent.name) {
return false;
}
if (chartEvent.filters.length > 0) {
return matchEventFilters(event, chartEvent.filters);
}
return true;
}
export async function checkNotificationRulesForEvent(
payload: IServiceCreateEventPayload,
) {
const project = await getProjectByIdCached(payload.projectId);
const rules = await getNotificationRulesByProjectId(payload.projectId);
await Promise.all(
rules.flatMap((rule) => {
if (rule.config.type === 'events') {
const match = rule.config.events.find((event) => {
return matchEvent(payload, event);
});
if (!match) {
return [];
}
const notification = {
title: `You received a new "${payload.name}" event`,
message: project?.name ? `Project: ${project?.name}` : '',
projectId: payload.projectId,
payload: {
type: 'event',
event: payload,
},
} as const;
const promises = rule.integrations.map((integration) =>
createNotification({
...notification,
integrationId: integration.id,
}),
);
if (rule.sendToApp) {
promises.push(
createNotification({
...notification,
integrationId: APP_NOTIFICATION_INTEGRATION_ID,
}),
);
}
if (rule.sendToEmail) {
promises.push(
createNotification({
...notification,
integrationId: EMAIL_NOTIFICATION_INTEGRATION_ID,
}),
);
}
return promises;
}
}),
);
}
export async function checkNotificationRulesForSessionEnd(
events: IServiceEvent[],
) {
const sortedEvents = events.sort(
(a, b) => a.createdAt.getTime() - b.createdAt.getTime(),
);
const projectId = sortedEvents[0]?.projectId;
if (!projectId) return null;
const [project, rules] = await Promise.all([
getProjectByIdCached(projectId),
getNotificationRulesByProjectId(projectId),
]);
const funnelRules = rules.filter((rule) => rule.config.type === 'funnel');
const notificationPromises = funnelRules.flatMap((rule) => {
// Match funnel events
let funnelIndex = 0;
const matchedEvents = [];
for (const event of sortedEvents) {
if (matchEvent(event, rule.config.events[funnelIndex]!)) {
matchedEvents.push(event);
funnelIndex++;
if (funnelIndex === rule.config.events.length) break;
}
}
// If funnel not completed, skip this rule
if (funnelIndex < rule.config.events.length) return [];
// Create notification object
const notification = {
title: `Funnel "${rule.name}" completed`,
message: project?.name ? `Project: ${project?.name}` : '',
projectId,
payload: { type: 'funnel', funnel: matchedEvents } as const,
};
// Generate notification promises
return [
...rule.integrations.map((integration) =>
createNotification({ ...notification, integrationId: integration.id }),
),
...(rule.sendToApp
? [
createNotification({
...notification,
integrationId: APP_NOTIFICATION_INTEGRATION_ID,
}),
]
: []),
...(rule.sendToEmail
? [
createNotification({
...notification,
integrationId: EMAIL_NOTIFICATION_INTEGRATION_ID,
}),
]
: []),
];
});
await Promise.all(notificationPromises);
}

3
packages/db/src/services/project.service.ts
View File

@@ -1,5 +1,6 @@
import { auth } from '@clerk/nextjs/server';
import { cacheable } from '@openpanel/redis';
import type { Prisma, Project } from '../prisma-client';
import { db } from '../prisma-client';
@@ -24,6 +25,8 @@ export async function getProjectById(id: string) {
return res;
}
export const getProjectByIdCached = cacheable(getProjectById, 60 * 60 * 24);
export async function getProjectWithClients(id: string) {
const res = await db.project.findUnique({
where: {

13
packages/db/src/types.ts Normal file
View File

@@ -0,0 +1,13 @@
import type {
IIntegrationConfig,
INotificationRuleConfig,
} from '@openpanel/validation';
import type { INotificationPayload } from './services/notification.service';
declare global {
namespace PrismaJson {
type IPrismaNotificationRuleConfig = INotificationRuleConfig;
type IPrismaIntegrationConfig = IIntegrationConfig;
type IPrismaNotificationPayload = INotificationPayload;
}
}

2
packages/db/tsconfig.json
View File

@@ -7,6 +7,6 @@
},
"tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
},
"include": ["."],
"include": [".", "./src/types.ts"],
"exclude": ["node_modules"]
}