batching events

apps/dashboard/Dockerfile

@@ -1,37 +1,14 @@
-FROM --platform=linux/amd64 node:20-slim AS base
+ARG NODE_VERSION=20
 
-ARG NEXT_PUBLIC_DASHBOARD_URL
-ENV NEXT_PUBLIC_DASHBOARD_URL=$NEXT_PUBLIC_DASHBOARD_URL
+FROM --platform=linux/amd64 node:${NODE_VERSION}-slim AS base
 
-ARG NEXT_PUBLIC_API_URL
-ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL
+ENV SKIP_ENV_VALIDATION="1"
 
 ARG DATABASE_URL
 ENV DATABASE_URL=$DATABASE_URL
 
-ARG CLICKHOUSE_DB 
-ENV CLICKHOUSE_DB=$CLICKHOUSE_DB
-
-ARG CLICKHOUSE_PASSWORD 
-ENV CLICKHOUSE_PASSWORD=$CLICKHOUSE_PASSWORD
-
-ARG CLICKHOUSE_URL 
-ENV CLICKHOUSE_URL=$CLICKHOUSE_URL
-
-ARG CLICKHOUSE_USER 
-ENV CLICKHOUSE_USER=$CLICKHOUSE_USER
-
-ARG REDIS_URL
-ENV REDIS_URL=$REDIS_URL
-
-ARG NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
-ENV NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=$NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
-
-ARG CLERK_SECRET_KEY
-ENV CLERK_SECRET_KEY=$CLERK_SECRET_KEY
-
-ARG CLERK_SIGNING_SECRET
-ENV CLERK_SIGNING_SECRET=$CLERK_SIGNING_SECRET
+ARG ENABLE_INSTRUMENTATION_HOOK
+ENV ENABLE_INSTRUMENTATION_HOOK=$ENABLE_INSTRUMENTATION_HOOK
 
 ENV PNPM_HOME="/pnpm"
 
@@ -39,8 +16,6 @@ ENV PATH="$PNPM_HOME:$PATH"
 
 RUN corepack enable
 
-ARG NODE_VERSION=20
-
 RUN apt update \
     && apt install -y curl \
     && curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n \
@@ -72,12 +47,19 @@ WORKDIR /app/apps/dashboard
 RUN pnpm install --frozen-lockfile --ignore-scripts
 
 WORKDIR  /app
-COPY apps apps
+COPY apps/dashboard apps/dashboard
 COPY packages packages
 COPY tooling tooling
 RUN pnpm db:codegen
 
 WORKDIR /app/apps/dashboard
+
+# Will be replaced on runtime
+ENV NEXT_PUBLIC_DASHBOARD_URL="__NEXT_PUBLIC_DASHBOARD_URL__"
+ENV NEXT_PUBLIC_API_URL="__NEXT_PUBLIC_API_URL__"
+# Check entrypoint for this little fellow
+ENV NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY="pk_test_eW9sby5jb20k"
+
 RUN pnpm run build
 
 # PROD
@@ -116,4 +98,7 @@ WORKDIR /app/apps/dashboard
 
 EXPOSE 3000
 
-CMD ["pnpm", "start"]
+# CMD ["pnpm", "start"]
+COPY --from=build /app/apps/dashboard/entrypoint.sh /usr/bin/
+RUN chmod +x /usr/bin/entrypoint.sh
+ENTRYPOINT ["entrypoint.sh", "pnpm", "start"]

apps/dashboard/entrypoint.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+
+echo "> Replace env variable placeholders with runtime values..."
+# Define an array of environment variables to check
+variables_to_replace=(
+  "NEXT_PUBLIC_DASHBOARD_URL"
+  "NEXT_PUBLIC_API_URL"
+  "NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"
+)
+
+# Replace env variable placeholders with real values
+for key in "${variables_to_replace[@]}"; do
+  value=$(printenv $key)
+  if [ ! -z "$value" ]; then
+    echo "  - Searching for $key with value $value..."
+    # Use a custom placeholder for 'NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY' or use the actual key otherwise
+    if [ "$key" = "NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY" ]; then
+      placeholder="pk_test_eW9sby5jb20k"
+    else
+      placeholder="__${key}__"
+    fi
+    # Run the replacement
+    find /app/apps/dashboard/.next/ -type f \( -name "*.js" -o -name "*.html" \) -exec sed -i "s|$placeholder|$value|g" {} \;
+
+  else
+    echo "  - Skipping $key as it has no value set."
+  fi
+done
+
+# Execute the container's main process (CMD in Dockerfile)
+exec "$@"

apps/dashboard/next.config.mjs

@@ -30,7 +30,7 @@ const config = {
   experimental: {
     // Avoid "Critical dependency: the request of a dependency is an expression"
     serverComponentsExternalPackages: ['bullmq', 'ioredis'],
-    instrumentationHook: true,
+    instrumentationHook: !!process.env.ENABLE_INSTRUMENTATION_HOOK,
   },
   /**
    * If you are using `appDir` then you must comment the below `i18n` config out.

apps/dashboard/package.json

@@ -15,7 +15,7 @@
   "dependencies": {
     "@baselime/node-opentelemetry": "^0.5.8",
     "@clerk/nextjs": "^5.0.12",
-    "@clickhouse/client": "^0.2.9",
+    "@clickhouse/client": "^1.2.0",
     "@hookform/resolvers": "^3.3.4",
     "@openpanel/common": "workspace:^",
     "@openpanel/constants": "workspace:^",
@@ -24,6 +24,7 @@
     "@openpanel/queue": "workspace:^",
     "@openpanel/sdk-info": "workspace:^",
     "@openpanel/validation": "workspace:^",
+    "@prisma/nextjs-monorepo-workaround-plugin": "^5.12.1",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
     "@radix-ui/react-aspect-ratio": "^1.0.3",
@@ -112,7 +113,6 @@
     "@openpanel/prettier-config": "workspace:*",
     "@openpanel/trpc": "workspace:*",
     "@openpanel/tsconfig": "workspace:*",
-    "@prisma/nextjs-monorepo-workaround-plugin": "^5.12.1",
     "@types/bcrypt": "^5.0.2",
     "@types/lodash.debounce": "^4.0.9",
     "@types/lodash.throttle": "^4.1.9",
@@ -145,4 +145,4 @@
     ]
   },
   "prettier": "@openpanel/prettier-config"
-}
+}

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-list/event-listener.tsx

@@ -1,12 +1,12 @@
 'use client';
 
-import { useState } from 'react';
 import {
   Tooltip,
   TooltipContent,
   TooltipTrigger,
 } from '@/components/ui/tooltip';
 import { useAppParams } from '@/hooks/useAppParams';
+import { useDebounceVal } from '@/hooks/useDebounceVal';
 import useWS from '@/hooks/useWS';
 import { cn } from '@/utils/cn';
 import dynamic from 'next/dynamic';
@@ -22,11 +22,13 @@ const AnimatedNumbers = dynamic(() => import('react-animated-numbers'), {
 export default function EventListener() {
   const router = useRouter();
   const { projectId } = useAppParams();
-  const [counter, setCounter] = useState(0);
+  const counter = useDebounceVal(0, 1000, {
+    maxWait: 5000,
+  });
 
   useWS<IServiceEventMinimal>(`/live/events/${projectId}`, (event) => {
     if (event?.name) {
-      setCounter((prev) => prev + 1);
+      counter.set((prev) => prev + 1);
     }
   });
 
@@ -35,7 +37,7 @@ export default function EventListener() {
       <TooltipTrigger asChild>
         <button
           onClick={() => {
-            setCounter(0);
+            counter.set(0);
             router.refresh();
           }}
           className="flex h-8 items-center gap-2 rounded border border-border bg-card px-3 text-sm font-medium leading-none"
@@ -52,7 +54,7 @@ export default function EventListener() {
               )}
             ></div>
           </div>
-          {counter === 0 ? (
+          {counter.debounced === 0 ? (
             'Listening'
           ) : (
             <>
@@ -61,11 +63,10 @@ export default function EventListener() {
                 transitions={(index) => ({
                   type: 'spring',
                   duration: index + 0.3,
-
                   damping: 10,
                   stiffness: 200,
                 })}
-                animateToNumber={counter}
+                animateToNumber={counter.debounced}
                 locale="en"
               />
               new events
@@ -74,7 +75,9 @@ export default function EventListener() {
         </button>
       </TooltipTrigger>
       <TooltipContent side="bottom">
-        {counter === 0 ? 'Listening to new events' : 'Click to refresh'}
+        {counter.debounced === 0
+          ? 'Listening to new events'
+          : 'Click to refresh'}
       </TooltipContent>
     </Tooltip>
   );

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx

@@ -5,7 +5,7 @@ import { getEvents } from '@openpanel/db';
 import LiveEvents from './live-events';
 
 type Props = {
-  projectId?: string;
+  projectId: string;
   limit?: number;
 };
 const RealtimeLiveEventsServer = async ({ projectId, limit = 30 }: Props) => {

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/live-events.tsx

@@ -12,14 +12,14 @@ import type {
 
 type Props = {
   events: (IServiceEventMinimal | IServiceCreateEventPayload)[];
-  projectId?: string;
+  projectId: string;
   limit: number;
 };
 
 const RealtimeLiveEvents = ({ events, projectId, limit }: Props) => {
   const [state, setState] = useState(events ?? []);
   useWS<IServiceEventMinimal | IServiceCreateEventPayload>(
-    projectId ? `/live/events/${projectId}` : '/live/events',
+    `/live/events/${projectId}`,
     (event) => {
       setState((p) => [event, ...p].slice(0, limit));
     }

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-reloader.tsx

@@ -12,12 +12,21 @@ const RealtimeReloader = ({ projectId }: Props) => {
   const client = useQueryClient();
   const router = useRouter();
 
-  useWS<number>(`/live/visitors/${projectId}`, (value) => {
-    router.refresh();
-    client.refetchQueries({
-      type: 'active',
-    });
-  });
+  useWS<number>(
+    `/live/events/${projectId}`,
+    () => {
+      router.refresh();
+      client.refetchQueries({
+        type: 'active',
+      });
+    },
+    {
+      debounce: {
+        maxWait: 15000,
+        delay: 15000,
+      },
+    }
+  );
 
   return null;
 };

apps/dashboard/src/app/api/clerk/webhook/route.ts

@@ -3,6 +3,8 @@ import { pathOr } from 'ramda';
 
 import { AccessLevel, db } from '@openpanel/db';
 
+export const dynamic = 'force-dynamic';
+
 export async function POST(request: Request) {
   const payload: WebhookEvent = await request.json();
 

apps/dashboard/src/app/providers.tsx

@@ -63,7 +63,8 @@ function AllProviders({ children }: { children: React.ReactNode }) {
       disableTransitionOnChange
     >
       <OpenpanelProvider
-        clientId="301c6dc1-424c-4bc3-9886-a8beab09b615"
+        url="https://op.coderax.se/api"
+        clientId="d32780cb-1c60-4a1b-bb5a-ffc11973255e"
         profileId={userId || undefined}
         trackScreenViews
         trackOutgoingLinks

apps/dashboard/src/hooks/useDebounceVal.ts

@@ -0,0 +1,30 @@
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import debounce from 'lodash.debounce';
+
+interface DebouncedState<T> {
+  value: T;
+  debounced: T;
+  set: React.Dispatch<React.SetStateAction<T>>;
+}
+
+export function useDebounceVal<T>(
+  initialValue: T,
+  delay = 500,
+  options?: Parameters<typeof debounce>[2]
+): DebouncedState<T> {
+  const [value, setValue] = useState<T>(initialValue);
+  const [debouncedValue, _setDebouncedValue] = useState<T>(initialValue);
+  const setDebouncedValue = useMemo(
+    () => debounce(_setDebouncedValue, delay, options),
+    []
+  );
+  useEffect(() => {
+    setDebouncedValue(value);
+  }, [value]);
+
+  return {
+    value,
+    debounced: debouncedValue,
+    set: setValue,
+  };
+}

apps/dashboard/src/hooks/useWS.ts

@@ -2,11 +2,22 @@
 
 import { use, useEffect, useMemo, useState } from 'react';
 import { useAuth } from '@clerk/nextjs';
+import debounce from 'lodash.debounce';
 import useWebSocket from 'react-use-websocket';
 
 import { getSuperJson } from '@openpanel/common';
 
-export default function useWS<T>(path: string, onMessage: (event: T) => void) {
+type UseWSOptions = {
+  debounce?: {
+    delay: number;
+  } & Parameters<typeof debounce>[2];
+};
+
+export default function useWS<T>(
+  path: string,
+  onMessage: (event: T) => void,
+  options?: UseWSOptions
+) {
   const auth = useAuth();
   const ws = String(process.env.NEXT_PUBLIC_API_URL)
     .replace(/^https/, 'wss')
@@ -18,6 +29,13 @@ export default function useWS<T>(path: string, onMessage: (event: T) => void) {
     [baseUrl, token]
   );
 
+  const debouncedOnMessage = useMemo(() => {
+    if (options?.debounce) {
+      return debounce(onMessage, options.debounce.delay, options.debounce);
+    }
+    return onMessage;
+  }, [options?.debounce?.delay]);
+
   useEffect(() => {
     if (auth.isSignedIn) {
       auth.getToken().then(setToken);
@@ -35,7 +53,7 @@ export default function useWS<T>(path: string, onMessage: (event: T) => void) {
       try {
         const data = getSuperJson<T>(event.data);
         if (data) {
-          onMessage(data);
+          debouncedOnMessage(data);
         }
       } catch (error) {
         console.error('Error parsing message', error);