diff --git a/apps/api/src/controllers/gsc-oauth-callback.controller.ts b/apps/api/src/controllers/gsc-oauth-callback.controller.ts new file mode 100644 index 00000000..639fc50f --- /dev/null +++ b/apps/api/src/controllers/gsc-oauth-callback.controller.ts @@ -0,0 +1,132 @@ +import { COOKIE_OPTIONS, googleGsc } from '@openpanel/auth'; +import { db } from '@openpanel/db'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import { z } from 'zod'; +import { LogError } from '@/utils/errors'; + +export async function gscInitiate(req: FastifyRequest, reply: FastifyReply) { + const schema = z.object({ + state: z.string(), + code_verifier: z.string(), + project_id: z.string(), + redirect: z.string().url(), + }); + + const query = schema.safeParse(req.query); + if (!query.success) { + return reply.status(400).send({ error: 'Invalid parameters' }); + } + + const { state, code_verifier, project_id, redirect } = query.data; + + reply.setCookie('gsc_oauth_state', state, { maxAge: 60 * 10, ...COOKIE_OPTIONS }); + reply.setCookie('gsc_code_verifier', code_verifier, { maxAge: 60 * 10, ...COOKIE_OPTIONS }); + reply.setCookie('gsc_project_id', project_id, { maxAge: 60 * 10, ...COOKIE_OPTIONS }); + + return reply.redirect(redirect); +} + +export async function gscGoogleCallback( + req: FastifyRequest, + reply: FastifyReply +) { + try { + const schema = z.object({ + code: z.string(), + state: z.string(), + }); + + const query = schema.safeParse(req.query); + if (!query.success) { + throw new LogError('Invalid GSC callback query params', { + error: query.error, + query: req.query, + }); + } + + const { code, state } = query.data; + const storedState = req.cookies.gsc_oauth_state ?? null; + const codeVerifier = req.cookies.gsc_code_verifier ?? null; + const projectId = req.cookies.gsc_project_id ?? null; + + if (!storedState || !codeVerifier || !projectId) { + throw new LogError('Missing GSC OAuth cookies', { + storedState: storedState === null, + codeVerifier: codeVerifier === null, + projectId: projectId === null, + }); + } + + if (state !== storedState) { + throw new LogError('GSC OAuth state mismatch', { state, storedState }); + } + + const tokens = await googleGsc.validateAuthorizationCode( + code, + codeVerifier + ); + + const accessToken = tokens.accessToken(); + const refreshToken = tokens.hasRefreshToken() + ? tokens.refreshToken() + : null; + const accessTokenExpiresAt = tokens.accessTokenExpiresAt(); + + if (!refreshToken) { + throw new LogError('No refresh token returned from Google GSC OAuth'); + } + + const project = await db.project.findUnique({ + where: { id: projectId }, + select: { id: true, organizationId: true }, + }); + + if (!project) { + throw new LogError('Project not found for GSC connection', { projectId }); + } + + await db.gscConnection.upsert({ + where: { projectId }, + create: { + projectId, + accessToken, + refreshToken, + accessTokenExpiresAt, + siteUrl: '', + }, + update: { + accessToken, + refreshToken, + accessTokenExpiresAt, + lastSyncStatus: null, + lastSyncError: null, + }, + }); + + reply.clearCookie('gsc_oauth_state'); + reply.clearCookie('gsc_code_verifier'); + reply.clearCookie('gsc_project_id'); + + const dashboardUrl = + process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!; + const redirectUrl = `${dashboardUrl}/${project.organizationId}/${projectId}/settings/gsc`; + return reply.redirect(redirectUrl); + } catch (error) { + req.log.error(error); + return redirectWithError(reply, error); + } +} + +function redirectWithError(reply: FastifyReply, error: LogError | unknown) { + const url = new URL( + process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL! + ); + url.pathname = '/login'; + if (error instanceof LogError) { + url.searchParams.set('error', error.message); + } else { + url.searchParams.set('error', 'Failed to connect Google Search Console'); + } + url.searchParams.set('correlationId', reply.request.id); + return reply.redirect(url.toString()); +} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index bf933329..ee5e0fd7 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -42,6 +42,7 @@ import liveRouter from './routes/live.router'; import manageRouter from './routes/manage.router'; import miscRouter from './routes/misc.router'; import oauthRouter from './routes/oauth-callback.router'; +import gscCallbackRouter from './routes/gsc-callback.router'; import profileRouter from './routes/profile.router'; import trackRouter from './routes/track.router'; import webhookRouter from './routes/webhook.router'; @@ -194,6 +195,7 @@ const startServer = async () => { instance.register(liveRouter, { prefix: '/live' }); instance.register(webhookRouter, { prefix: '/webhook' }); instance.register(oauthRouter, { prefix: '/oauth' }); + instance.register(gscCallbackRouter, { prefix: '/gsc' }); instance.register(miscRouter, { prefix: '/misc' }); instance.register(aiRouter, { prefix: '/ai' }); }); diff --git a/apps/api/src/routes/gsc-callback.router.ts b/apps/api/src/routes/gsc-callback.router.ts new file mode 100644 index 00000000..0becfb77 --- /dev/null +++ b/apps/api/src/routes/gsc-callback.router.ts @@ -0,0 +1,17 @@ +import { gscGoogleCallback, gscInitiate } from '@/controllers/gsc-oauth-callback.controller'; +import type { FastifyPluginCallback } from 'fastify'; + +const router: FastifyPluginCallback = async (fastify) => { + fastify.route({ + method: 'GET', + url: '/initiate', + handler: gscInitiate, + }); + fastify.route({ + method: 'GET', + url: '/callback', + handler: gscGoogleCallback, + }); +}; + +export default router; diff --git a/apps/start/src/components/sidebar-project-menu.tsx b/apps/start/src/components/sidebar-project-menu.tsx index 50498c89..8c0a5594 100644 --- a/apps/start/src/components/sidebar-project-menu.tsx +++ b/apps/start/src/components/sidebar-project-menu.tsx @@ -14,6 +14,7 @@ import { LayoutDashboardIcon, LayoutPanelTopIcon, PlusIcon, + SearchIcon, SparklesIcon, TrendingUpDownIcon, UndoDotIcon, @@ -59,6 +60,7 @@ export default function SidebarProjectMenu({ +
Manage
diff --git a/apps/start/src/routeTree.gen.ts b/apps/start/src/routeTree.gen.ts index 221cc39d..8742b541 100644 --- a/apps/start/src/routeTree.gen.ts +++ b/apps/start/src/routeTree.gen.ts @@ -42,6 +42,7 @@ import { Route as AppOrganizationIdProfileTabsRouteImport } from './routes/_app. import { Route as AppOrganizationIdMembersTabsRouteImport } from './routes/_app.$organizationId.members._tabs' import { Route as AppOrganizationIdIntegrationsTabsRouteImport } from './routes/_app.$organizationId.integrations._tabs' import { Route as AppOrganizationIdProjectIdSessionsRouteImport } from './routes/_app.$organizationId.$projectId.sessions' +import { Route as AppOrganizationIdProjectIdSeoRouteImport } from './routes/_app.$organizationId.$projectId.seo' import { Route as AppOrganizationIdProjectIdReportsRouteImport } from './routes/_app.$organizationId.$projectId.reports' import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references' import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime' @@ -71,6 +72,7 @@ import { Route as AppOrganizationIdProjectIdEventsTabsIndexRouteImport } from '. import { Route as AppOrganizationIdProjectIdSettingsTabsWidgetsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.widgets' import { Route as AppOrganizationIdProjectIdSettingsTabsTrackingRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.tracking' import { Route as AppOrganizationIdProjectIdSettingsTabsImportsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.imports' +import { Route as AppOrganizationIdProjectIdSettingsTabsGscRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.gsc' import { Route as AppOrganizationIdProjectIdSettingsTabsEventsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.events' import { Route as AppOrganizationIdProjectIdSettingsTabsDetailsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.details' import { Route as AppOrganizationIdProjectIdSettingsTabsClientsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.clients' @@ -312,6 +314,12 @@ const AppOrganizationIdProjectIdSessionsRoute = path: '/sessions', getParentRoute: () => AppOrganizationIdProjectIdRoute, } as any) +const AppOrganizationIdProjectIdSeoRoute = + AppOrganizationIdProjectIdSeoRouteImport.update({ + id: '/seo', + path: '/seo', + getParentRoute: () => AppOrganizationIdProjectIdRoute, + } as any) const AppOrganizationIdProjectIdReportsRoute = AppOrganizationIdProjectIdReportsRouteImport.update({ id: '/reports', @@ -488,6 +496,12 @@ const AppOrganizationIdProjectIdSettingsTabsImportsRoute = path: '/imports', getParentRoute: () => AppOrganizationIdProjectIdSettingsTabsRoute, } as any) +const AppOrganizationIdProjectIdSettingsTabsGscRoute = + AppOrganizationIdProjectIdSettingsTabsGscRouteImport.update({ + id: '/gsc', + path: '/gsc', + getParentRoute: () => AppOrganizationIdProjectIdSettingsTabsRoute, + } as any) const AppOrganizationIdProjectIdSettingsTabsEventsRoute = AppOrganizationIdProjectIdSettingsTabsEventsRouteImport.update({ id: '/events', @@ -606,6 +620,7 @@ export interface FileRoutesByFullPath { '/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/$organizationId/$projectId/reports': typeof AppOrganizationIdProjectIdReportsRoute + '/$organizationId/$projectId/seo': typeof AppOrganizationIdProjectIdSeoRoute '/$organizationId/$projectId/sessions': typeof AppOrganizationIdProjectIdSessionsRoute '/$organizationId/integrations': typeof AppOrganizationIdIntegrationsTabsRouteWithChildren '/$organizationId/members': typeof AppOrganizationIdMembersTabsRouteWithChildren @@ -640,6 +655,7 @@ export interface FileRoutesByFullPath { '/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute '/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute '/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute + '/$organizationId/$projectId/settings/gsc': typeof AppOrganizationIdProjectIdSettingsTabsGscRoute '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute '/$organizationId/$projectId/settings/tracking': typeof AppOrganizationIdProjectIdSettingsTabsTrackingRoute '/$organizationId/$projectId/settings/widgets': typeof AppOrganizationIdProjectIdSettingsTabsWidgetsRoute @@ -677,6 +693,7 @@ export interface FileRoutesByTo { '/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/$organizationId/$projectId/reports': typeof AppOrganizationIdProjectIdReportsRoute + '/$organizationId/$projectId/seo': typeof AppOrganizationIdProjectIdSeoRoute '/$organizationId/$projectId/sessions': typeof AppOrganizationIdProjectIdSessionsRoute '/$organizationId/integrations': typeof AppOrganizationIdIntegrationsTabsIndexRoute '/$organizationId/members': typeof AppOrganizationIdMembersTabsIndexRoute @@ -708,6 +725,7 @@ export interface FileRoutesByTo { '/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute '/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute '/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute + '/$organizationId/$projectId/settings/gsc': typeof AppOrganizationIdProjectIdSettingsTabsGscRoute '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute '/$organizationId/$projectId/settings/tracking': typeof AppOrganizationIdProjectIdSettingsTabsTrackingRoute '/$organizationId/$projectId/settings/widgets': typeof AppOrganizationIdProjectIdSettingsTabsWidgetsRoute @@ -747,6 +765,7 @@ export interface FileRoutesById { '/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/_app/$organizationId/$projectId/reports': typeof AppOrganizationIdProjectIdReportsRoute + '/_app/$organizationId/$projectId/seo': typeof AppOrganizationIdProjectIdSeoRoute '/_app/$organizationId/$projectId/sessions': typeof AppOrganizationIdProjectIdSessionsRoute '/_app/$organizationId/integrations': typeof AppOrganizationIdIntegrationsRouteWithChildren '/_app/$organizationId/integrations/_tabs': typeof AppOrganizationIdIntegrationsTabsRouteWithChildren @@ -789,6 +808,7 @@ export interface FileRoutesById { '/_app/$organizationId/$projectId/settings/_tabs/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute '/_app/$organizationId/$projectId/settings/_tabs/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute '/_app/$organizationId/$projectId/settings/_tabs/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute + '/_app/$organizationId/$projectId/settings/_tabs/gsc': typeof AppOrganizationIdProjectIdSettingsTabsGscRoute '/_app/$organizationId/$projectId/settings/_tabs/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute '/_app/$organizationId/$projectId/settings/_tabs/tracking': typeof AppOrganizationIdProjectIdSettingsTabsTrackingRoute '/_app/$organizationId/$projectId/settings/_tabs/widgets': typeof AppOrganizationIdProjectIdSettingsTabsWidgetsRoute @@ -830,6 +850,7 @@ export interface FileRouteTypes { | '/$organizationId/$projectId/realtime' | '/$organizationId/$projectId/references' | '/$organizationId/$projectId/reports' + | '/$organizationId/$projectId/seo' | '/$organizationId/$projectId/sessions' | '/$organizationId/integrations' | '/$organizationId/members' @@ -864,6 +885,7 @@ export interface FileRouteTypes { | '/$organizationId/$projectId/settings/clients' | '/$organizationId/$projectId/settings/details' | '/$organizationId/$projectId/settings/events' + | '/$organizationId/$projectId/settings/gsc' | '/$organizationId/$projectId/settings/imports' | '/$organizationId/$projectId/settings/tracking' | '/$organizationId/$projectId/settings/widgets' @@ -901,6 +923,7 @@ export interface FileRouteTypes { | '/$organizationId/$projectId/realtime' | '/$organizationId/$projectId/references' | '/$organizationId/$projectId/reports' + | '/$organizationId/$projectId/seo' | '/$organizationId/$projectId/sessions' | '/$organizationId/integrations' | '/$organizationId/members' @@ -932,6 +955,7 @@ export interface FileRouteTypes { | '/$organizationId/$projectId/settings/clients' | '/$organizationId/$projectId/settings/details' | '/$organizationId/$projectId/settings/events' + | '/$organizationId/$projectId/settings/gsc' | '/$organizationId/$projectId/settings/imports' | '/$organizationId/$projectId/settings/tracking' | '/$organizationId/$projectId/settings/widgets' @@ -970,6 +994,7 @@ export interface FileRouteTypes { | '/_app/$organizationId/$projectId/realtime' | '/_app/$organizationId/$projectId/references' | '/_app/$organizationId/$projectId/reports' + | '/_app/$organizationId/$projectId/seo' | '/_app/$organizationId/$projectId/sessions' | '/_app/$organizationId/integrations' | '/_app/$organizationId/integrations/_tabs' @@ -1012,6 +1037,7 @@ export interface FileRouteTypes { | '/_app/$organizationId/$projectId/settings/_tabs/clients' | '/_app/$organizationId/$projectId/settings/_tabs/details' | '/_app/$organizationId/$projectId/settings/_tabs/events' + | '/_app/$organizationId/$projectId/settings/_tabs/gsc' | '/_app/$organizationId/$projectId/settings/_tabs/imports' | '/_app/$organizationId/$projectId/settings/_tabs/tracking' | '/_app/$organizationId/$projectId/settings/_tabs/widgets' @@ -1310,6 +1336,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AppOrganizationIdProjectIdSessionsRouteImport parentRoute: typeof AppOrganizationIdProjectIdRoute } + '/_app/$organizationId/$projectId/seo': { + id: '/_app/$organizationId/$projectId/seo' + path: '/seo' + fullPath: '/$organizationId/$projectId/seo' + preLoaderRoute: typeof AppOrganizationIdProjectIdSeoRouteImport + parentRoute: typeof AppOrganizationIdProjectIdRoute + } '/_app/$organizationId/$projectId/reports': { id: '/_app/$organizationId/$projectId/reports' path: '/reports' @@ -1520,6 +1553,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRouteImport parentRoute: typeof AppOrganizationIdProjectIdSettingsTabsRoute } + '/_app/$organizationId/$projectId/settings/_tabs/gsc': { + id: '/_app/$organizationId/$projectId/settings/_tabs/gsc' + path: '/gsc' + fullPath: '/$organizationId/$projectId/settings/gsc' + preLoaderRoute: typeof AppOrganizationIdProjectIdSettingsTabsGscRouteImport + parentRoute: typeof AppOrganizationIdProjectIdSettingsTabsRoute + } '/_app/$organizationId/$projectId/settings/_tabs/events': { id: '/_app/$organizationId/$projectId/settings/_tabs/events' path: '/events' @@ -1785,6 +1825,7 @@ interface AppOrganizationIdProjectIdSettingsTabsRouteChildren { AppOrganizationIdProjectIdSettingsTabsClientsRoute: typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute AppOrganizationIdProjectIdSettingsTabsDetailsRoute: typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute AppOrganizationIdProjectIdSettingsTabsEventsRoute: typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute + AppOrganizationIdProjectIdSettingsTabsGscRoute: typeof AppOrganizationIdProjectIdSettingsTabsGscRoute AppOrganizationIdProjectIdSettingsTabsImportsRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute AppOrganizationIdProjectIdSettingsTabsTrackingRoute: typeof AppOrganizationIdProjectIdSettingsTabsTrackingRoute AppOrganizationIdProjectIdSettingsTabsWidgetsRoute: typeof AppOrganizationIdProjectIdSettingsTabsWidgetsRoute @@ -1799,6 +1840,8 @@ const AppOrganizationIdProjectIdSettingsTabsRouteChildren: AppOrganizationIdProj AppOrganizationIdProjectIdSettingsTabsDetailsRoute, AppOrganizationIdProjectIdSettingsTabsEventsRoute: AppOrganizationIdProjectIdSettingsTabsEventsRoute, + AppOrganizationIdProjectIdSettingsTabsGscRoute: + AppOrganizationIdProjectIdSettingsTabsGscRoute, AppOrganizationIdProjectIdSettingsTabsImportsRoute: AppOrganizationIdProjectIdSettingsTabsImportsRoute, AppOrganizationIdProjectIdSettingsTabsTrackingRoute: @@ -1837,6 +1880,7 @@ interface AppOrganizationIdProjectIdRouteChildren { AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute AppOrganizationIdProjectIdReportsRoute: typeof AppOrganizationIdProjectIdReportsRoute + AppOrganizationIdProjectIdSeoRoute: typeof AppOrganizationIdProjectIdSeoRoute AppOrganizationIdProjectIdSessionsRoute: typeof AppOrganizationIdProjectIdSessionsRoute AppOrganizationIdProjectIdIndexRoute: typeof AppOrganizationIdProjectIdIndexRoute AppOrganizationIdProjectIdDashboardsDashboardIdRoute: typeof AppOrganizationIdProjectIdDashboardsDashboardIdRoute @@ -1862,6 +1906,7 @@ const AppOrganizationIdProjectIdRouteChildren: AppOrganizationIdProjectIdRouteCh AppOrganizationIdProjectIdReferencesRoute, AppOrganizationIdProjectIdReportsRoute: AppOrganizationIdProjectIdReportsRoute, + AppOrganizationIdProjectIdSeoRoute: AppOrganizationIdProjectIdSeoRoute, AppOrganizationIdProjectIdSessionsRoute: AppOrganizationIdProjectIdSessionsRoute, AppOrganizationIdProjectIdIndexRoute: AppOrganizationIdProjectIdIndexRoute, diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.seo.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.seo.tsx new file mode 100644 index 00000000..ba87e5f9 --- /dev/null +++ b/apps/start/src/routes/_app.$organizationId.$projectId.seo.tsx @@ -0,0 +1,289 @@ +import { PageContainer } from '@/components/page-container'; +import { PageHeader } from '@/components/page-header'; +import { Skeleton } from '@/components/skeleton'; +import { Button } from '@/components/ui/button'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { useAppParams } from '@/hooks/use-app-params'; +import { useTRPC } from '@/integrations/trpc/react'; +import { createProjectTitle } from '@/utils/title'; +import { useQuery } from '@tanstack/react-query'; +import { createFileRoute, useNavigate } from '@tanstack/react-router'; +import { subDays, format } from 'date-fns'; +import { + Area, + AreaChart, + CartesianGrid, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; + +export const Route = createFileRoute( + '/_app/$organizationId/$projectId/seo' +)({ + component: SeoPage, + head: () => ({ + meta: [{ title: createProjectTitle('SEO') }], + }), +}); + +const startDate = format(subDays(new Date(), 30), 'yyyy-MM-dd'); +const endDate = format(subDays(new Date(), 1), 'yyyy-MM-dd'); + +function SeoPage() { + const { projectId, organizationId } = useAppParams(); + const trpc = useTRPC(); + const navigate = useNavigate(); + + const connectionQuery = useQuery( + trpc.gsc.getConnection.queryOptions({ projectId }) + ); + + const connection = connectionQuery.data; + const isConnected = connection && connection.siteUrl; + + const overviewQuery = useQuery( + trpc.gsc.getOverview.queryOptions( + { projectId, startDate, endDate }, + { enabled: !!isConnected } + ) + ); + + const pagesQuery = useQuery( + trpc.gsc.getPages.queryOptions( + { projectId, startDate, endDate, limit: 50 }, + { enabled: !!isConnected } + ) + ); + + const queriesQuery = useQuery( + trpc.gsc.getQueries.queryOptions( + { projectId, startDate, endDate, limit: 50 }, + { enabled: !!isConnected } + ) + ); + + if (connectionQuery.isLoading) { + return ( + + +
+ + +
+ + ); + } + + if (!isConnected) { + return ( + + +
+
+ + + +
+

No SEO data yet

+

+ Connect Google Search Console to track your search impressions, clicks, and keyword rankings. +

+ +
+ + ); + } + + const overview = overviewQuery.data ?? []; + const pages = pagesQuery.data ?? []; + const queries = queriesQuery.data ?? []; + + return ( + + + +
+ {/* Summary metrics */} +
+ {(['clicks', 'impressions', 'ctr', 'position'] as const).map((metric) => { + const total = overview.reduce((sum, row) => { + if (metric === 'ctr' || metric === 'position') { + return sum + row[metric]; + } + return sum + row[metric]; + }, 0); + const display = + metric === 'ctr' + ? `${((total / Math.max(overview.length, 1)) * 100).toFixed(1)}%` + : metric === 'position' + ? (total / Math.max(overview.length, 1)).toFixed(1) + : total.toLocaleString(); + const label = + metric === 'ctr' + ? 'Avg CTR' + : metric === 'position' + ? 'Avg Position' + : metric.charAt(0).toUpperCase() + metric.slice(1); + + return ( +
+
{label}
+
{overviewQuery.isLoading ? : display}
+
+ ); + })} +
+ + {/* Clicks over time chart */} +
+

Clicks over time

+ {overviewQuery.isLoading ? ( + + ) : ( + + + + + + + + + + + + + + + + )} +
+ + {/* Pages and Queries tables */} +
+ + +
+
+ + ); +} + +interface GscTableRow { + clicks: number; + impressions: number; + ctr: number; + position: number; + [key: string]: string | number; +} + +function GscTable({ + title, + rows, + keyLabel, + keyField, + isLoading, +}: { + title: string; + rows: GscTableRow[]; + keyLabel: string; + keyField: string; + isLoading: boolean; +}) { + return ( +
+
+

{title}

+
+ + + + {keyLabel} + Clicks + Impressions + CTR + Position + + + + {isLoading && + Array.from({ length: 5 }).map((_, i) => ( + + {Array.from({ length: 5 }).map((_, j) => ( + + + + ))} + + ))} + {!isLoading && rows.length === 0 && ( + + + No data yet + + + )} + {rows.map((row) => ( + + + {String(row[keyField])} + + {row.clicks.toLocaleString()} + {row.impressions.toLocaleString()} + {(row.ctr * 100).toFixed(1)}% + {row.position.toFixed(1)} + + ))} + +
+
+ ); +} diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.gsc.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.gsc.tsx new file mode 100644 index 00000000..af18c356 --- /dev/null +++ b/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.gsc.tsx @@ -0,0 +1,274 @@ +import { Skeleton } from '@/components/skeleton'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { useAppParams } from '@/hooks/use-app-params'; +import { useTRPC } from '@/integrations/trpc/react'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { createFileRoute } from '@tanstack/react-router'; +import { formatDistanceToNow } from 'date-fns'; +import { CheckCircleIcon, Loader2Icon, XCircleIcon } from 'lucide-react'; +import { useState } from 'react'; +import { toast } from 'sonner'; + +export const Route = createFileRoute( + '/_app/$organizationId/$projectId/settings/_tabs/gsc' +)({ + component: GscSettings, +}); + +function GscSettings() { + const { projectId } = useAppParams(); + const trpc = useTRPC(); + const queryClient = useQueryClient(); + const [selectedSite, setSelectedSite] = useState(''); + + const connectionQuery = useQuery( + trpc.gsc.getConnection.queryOptions( + { projectId }, + { refetchInterval: 5000 } + ) + ); + + const sitesQuery = useQuery( + trpc.gsc.getSites.queryOptions( + { projectId }, + { enabled: !!connectionQuery.data && !connectionQuery.data.siteUrl } + ) + ); + + const initiateOAuth = useMutation( + trpc.gsc.initiateOAuth.mutationOptions({ + onSuccess: (data) => { + // Route through the API /gsc/initiate endpoint which sets cookies then redirects to Google + const apiUrl = (import.meta.env.VITE_API_URL as string) ?? ''; + const initiateUrl = new URL(`${apiUrl}/gsc/initiate`); + initiateUrl.searchParams.set('state', data.state); + initiateUrl.searchParams.set('code_verifier', data.codeVerifier); + initiateUrl.searchParams.set('project_id', data.projectId); + initiateUrl.searchParams.set('redirect', data.url); + window.location.href = initiateUrl.toString(); + }, + onError: () => { + toast.error('Failed to initiate Google Search Console connection'); + }, + }) + ); + + const selectSite = useMutation( + trpc.gsc.selectSite.mutationOptions({ + onSuccess: () => { + toast.success('Site connected', { + description: 'Backfill of 6 months of data has started.', + }); + queryClient.invalidateQueries(trpc.gsc.getConnection.pathFilter()); + }, + onError: () => { + toast.error('Failed to select site'); + }, + }) + ); + + const disconnect = useMutation( + trpc.gsc.disconnect.mutationOptions({ + onSuccess: () => { + toast.success('Disconnected from Google Search Console'); + queryClient.invalidateQueries(trpc.gsc.getConnection.pathFilter()); + }, + onError: () => { + toast.error('Failed to disconnect'); + }, + }) + ); + + const connection = connectionQuery.data; + + if (connectionQuery.isLoading) { + return ( +
+ +
+ ); + } + + // Not connected at all + if (!connection) { + return ( +
+
+

Google Search Console

+

+ Connect your Google Search Console property to import search performance data. +

+
+
+

+ You will be redirected to Google to authorize access. Only read-only access to Search Console data is requested. +

+ +
+
+ ); + } + + // Connected but no site selected yet + if (!connection.siteUrl) { + const sites = sitesQuery.data ?? []; + return ( +
+
+

Select a property

+

+ Choose which Google Search Console property to connect to this project. +

+
+
+ {sitesQuery.isLoading ? ( + + ) : sites.length === 0 ? ( +

+ No Search Console properties found for this Google account. +

+ ) : ( + <> + + + + )} +
+ +
+ ); + } + + // Fully connected + const syncStatusIcon = + connection.lastSyncStatus === 'success' ? ( + + ) : connection.lastSyncStatus === 'error' ? ( + + ) : null; + + const syncStatusVariant = + connection.lastSyncStatus === 'success' + ? 'success' + : connection.lastSyncStatus === 'error' + ? 'destructive' + : 'secondary'; + + return ( +
+
+

Google Search Console

+

+ Connected to Google Search Console. +

+
+ +
+
+
Property
+
+ {connection.siteUrl} +
+
+ + {connection.backfillStatus && ( +
+
Backfill
+ + {connection.backfillStatus === 'running' && ( + + )} + {connection.backfillStatus} + +
+ )} + + {connection.lastSyncedAt && ( +
+
Last synced
+
+ {connection.lastSyncStatus && ( + + {syncStatusIcon} + {connection.lastSyncStatus} + + )} + + {formatDistanceToNow(new Date(connection.lastSyncedAt), { + addSuffix: true, + })} + +
+
+ )} + + {connection.lastSyncError && ( +
+
Last error
+
+ {connection.lastSyncError} +
+
+ )} +
+ + +
+ ); +} diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.tsx index 205fb7bc..b0037ed9 100644 --- a/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.tsx +++ b/apps/start/src/routes/_app.$organizationId.$projectId.settings._tabs.tsx @@ -45,6 +45,7 @@ function ProjectDashboard() { { id: 'tracking', label: 'Tracking script' }, { id: 'widgets', label: 'Widgets' }, { id: 'imports', label: 'Imports' }, + { id: 'gsc', label: 'Google Search' }, ]; const handleTabChange = (tabId: string) => { diff --git a/apps/worker/src/boot-cron.ts b/apps/worker/src/boot-cron.ts index 106cec07..e3835e91 100644 --- a/apps/worker/src/boot-cron.ts +++ b/apps/worker/src/boot-cron.ts @@ -78,6 +78,11 @@ export async function bootCron() { type: 'onboarding', pattern: '0 * * * *', }, + { + name: 'gscSync', + type: 'gscSync', + pattern: '0 3 * * *', + }, ]; if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') { diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts index 6d96dd61..5b4285ca 100644 --- a/apps/worker/src/boot-workers.ts +++ b/apps/worker/src/boot-workers.ts @@ -6,6 +6,7 @@ import { type EventsQueuePayloadIncomingEvent, cronQueue, eventsGroupQueues, + gscQueue, importQueue, insightsQueue, miscQueue, @@ -20,6 +21,7 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { Worker as GroupWorker } from 'groupmq'; import { cronJob } from './jobs/cron'; +import { gscJob } from './jobs/gsc'; import { incomingEvent } from './jobs/events.incoming-event'; import { importJob } from './jobs/import'; import { insightsProjectJob } from './jobs/insights'; @@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] { 'misc', 'import', 'insights', + 'gsc', ]; } @@ -208,6 +211,17 @@ export async function bootWorkers() { logger.info('Started worker for insights', { concurrency }); } + // Start gsc worker + if (enabledQueues.includes('gsc')) { + const concurrency = getConcurrencyFor('gsc', 5); + const gscWorker = new Worker(gscQueue.name, gscJob, { + ...workerOptions, + concurrency, + }); + workers.push(gscWorker); + logger.info('Started worker for gsc', { concurrency }); + } + if (workers.length === 0) { logger.warn( 'No workers started. Check ENABLED_QUEUES environment variable.', diff --git a/apps/worker/src/jobs/cron.ts b/apps/worker/src/jobs/cron.ts index 8d69afec..f9428614 100644 --- a/apps/worker/src/jobs/cron.ts +++ b/apps/worker/src/jobs/cron.ts @@ -4,6 +4,7 @@ import { eventBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessio import type { CronQueuePayload } from '@openpanel/queue'; import { jobdeleteProjects } from './cron.delete-projects'; +import { gscSyncAllJob } from './gsc'; import { onboardingJob } from './cron.onboarding'; import { ping } from './cron.ping'; import { salt } from './cron.salt'; @@ -41,5 +42,8 @@ export async function cronJob(job: Job) { case 'onboarding': { return await onboardingJob(job); } + case 'gscSync': { + return await gscSyncAllJob(); + } } } diff --git a/apps/worker/src/jobs/gsc.ts b/apps/worker/src/jobs/gsc.ts new file mode 100644 index 00000000..6ed9f51a --- /dev/null +++ b/apps/worker/src/jobs/gsc.ts @@ -0,0 +1,142 @@ +import { db, syncGscData } from '@openpanel/db'; +import { gscQueue } from '@openpanel/queue'; +import type { GscQueuePayload } from '@openpanel/queue'; +import type { Job } from 'bullmq'; +import { logger } from '../utils/logger'; + +const BACKFILL_MONTHS = 6; +const CHUNK_DAYS = 14; + +export async function gscJob(job: Job) { + switch (job.data.type) { + case 'gscProjectSync': + return gscProjectSyncJob(job.data.payload.projectId); + case 'gscProjectBackfill': + return gscProjectBackfillJob(job.data.payload.projectId); + } +} + +async function gscProjectSyncJob(projectId: string) { + const conn = await db.gscConnection.findUnique({ where: { projectId } }); + if (!conn?.siteUrl) { + logger.warn('GSC sync skipped: no connection or siteUrl', { projectId }); + return; + } + + try { + // Sync rolling 3-day window (GSC data can arrive late) + const endDate = new Date(); + endDate.setDate(endDate.getDate() - 1); // yesterday + const startDate = new Date(endDate); + startDate.setDate(startDate.getDate() - 2); // 3 days total + + await syncGscData(projectId, startDate, endDate); + + await db.gscConnection.update({ + where: { projectId }, + data: { + lastSyncedAt: new Date(), + lastSyncStatus: 'success', + lastSyncError: null, + }, + }); + logger.info('GSC sync completed', { projectId }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await db.gscConnection.update({ + where: { projectId }, + data: { + lastSyncedAt: new Date(), + lastSyncStatus: 'error', + lastSyncError: message, + }, + }); + logger.error('GSC sync failed', { projectId, error }); + throw error; + } +} + +async function gscProjectBackfillJob(projectId: string) { + const conn = await db.gscConnection.findUnique({ where: { projectId } }); + if (!conn?.siteUrl) { + logger.warn('GSC backfill skipped: no connection or siteUrl', { projectId }); + return; + } + + await db.gscConnection.update({ + where: { projectId }, + data: { backfillStatus: 'running' }, + }); + + try { + const endDate = new Date(); + endDate.setDate(endDate.getDate() - 1); // yesterday + + const startDate = new Date(endDate); + startDate.setMonth(startDate.getMonth() - BACKFILL_MONTHS); + + // Process in chunks to avoid timeouts and respect API limits + let chunkEnd = new Date(endDate); + while (chunkEnd > startDate) { + const chunkStart = new Date(chunkEnd); + chunkStart.setDate(chunkStart.getDate() - CHUNK_DAYS + 1); + if (chunkStart < startDate) { + chunkStart.setTime(startDate.getTime()); + } + + logger.info('GSC backfill chunk', { + projectId, + from: chunkStart.toISOString().slice(0, 10), + to: chunkEnd.toISOString().slice(0, 10), + }); + + await syncGscData(projectId, chunkStart, chunkEnd); + + chunkEnd = new Date(chunkStart); + chunkEnd.setDate(chunkEnd.getDate() - 1); + } + + await db.gscConnection.update({ + where: { projectId }, + data: { + backfillStatus: 'completed', + lastSyncedAt: new Date(), + lastSyncStatus: 'success', + lastSyncError: null, + }, + }); + logger.info('GSC backfill completed', { projectId }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await db.gscConnection.update({ + where: { projectId }, + data: { + backfillStatus: 'failed', + lastSyncStatus: 'error', + lastSyncError: message, + }, + }); + logger.error('GSC backfill failed', { projectId, error }); + throw error; + } +} + +export async function gscSyncAllJob() { + const connections = await db.gscConnection.findMany({ + where: { + siteUrl: { not: '' }, + }, + select: { projectId: true }, + }); + + logger.info('GSC nightly sync: enqueuing projects', { + count: connections.length, + }); + + for (const conn of connections) { + await gscQueue.add('gscProjectSync', { + type: 'gscProjectSync', + payload: { projectId: conn.projectId }, + }); + } +} diff --git a/packages/auth/src/oauth.ts b/packages/auth/src/oauth.ts index 18464f6a..9bd8cd14 100644 --- a/packages/auth/src/oauth.ts +++ b/packages/auth/src/oauth.ts @@ -16,3 +16,9 @@ export const google = new Arctic.Google( process.env.GOOGLE_CLIENT_SECRET ?? '', process.env.GOOGLE_REDIRECT_URI ?? '', ); + +export const googleGsc = new Arctic.Google( + process.env.GOOGLE_CLIENT_ID ?? '', + process.env.GOOGLE_CLIENT_SECRET ?? '', + process.env.GSC_GOOGLE_REDIRECT_URI ?? '', +); diff --git a/packages/db/code-migrations/12-add-gsc.ts b/packages/db/code-migrations/12-add-gsc.ts new file mode 100644 index 00000000..43d82d9f --- /dev/null +++ b/packages/db/code-migrations/12-add-gsc.ts @@ -0,0 +1,85 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createTable, runClickhouseMigrationCommands } from '../src/clickhouse/migration'; +import { getIsCluster } from './helpers'; + +export async function up() { + const isClustered = getIsCluster(); + + const commonMetricColumns = [ + '`clicks` UInt32 CODEC(Delta(4), LZ4)', + '`impressions` UInt32 CODEC(Delta(4), LZ4)', + '`ctr` Float32 CODEC(Gorilla, LZ4)', + '`position` Float32 CODEC(Gorilla, LZ4)', + '`synced_at` DateTime DEFAULT now() CODEC(Delta(4), LZ4)', + ]; + + const sqls: string[] = [ + // Daily totals — accurate overview numbers + ...createTable({ + name: 'gsc_daily', + columns: [ + '`project_id` String CODEC(ZSTD(3))', + '`date` Date CODEC(Delta(2), LZ4)', + ...commonMetricColumns, + ], + orderBy: ['project_id', 'date'], + partitionBy: 'toYYYYMM(date)', + engine: 'ReplacingMergeTree(synced_at)', + distributionHash: 'cityHash64(project_id)', + replicatedVersion: '1', + isClustered, + }), + + // Per-page breakdown + ...createTable({ + name: 'gsc_pages_daily', + columns: [ + '`project_id` String CODEC(ZSTD(3))', + '`date` Date CODEC(Delta(2), LZ4)', + '`page` String CODEC(ZSTD(3))', + ...commonMetricColumns, + ], + orderBy: ['project_id', 'date', 'page'], + partitionBy: 'toYYYYMM(date)', + engine: 'ReplacingMergeTree(synced_at)', + distributionHash: 'cityHash64(project_id)', + replicatedVersion: '1', + isClustered, + }), + + // Per-query breakdown + ...createTable({ + name: 'gsc_queries_daily', + columns: [ + '`project_id` String CODEC(ZSTD(3))', + '`date` Date CODEC(Delta(2), LZ4)', + '`query` String CODEC(ZSTD(3))', + ...commonMetricColumns, + ], + orderBy: ['project_id', 'date', 'query'], + partitionBy: 'toYYYYMM(date)', + engine: 'ReplacingMergeTree(synced_at)', + distributionHash: 'cityHash64(project_id)', + replicatedVersion: '1', + isClustered, + }), + ]; + + fs.writeFileSync( + path.join(__filename.replace('.ts', '.sql')), + sqls + .map((sql) => + sql + .trim() + .replace(/;$/, '') + .replace(/\n{2,}/g, '\n') + .concat(';'), + ) + .join('\n\n---\n\n'), + ); + + if (!process.argv.includes('--dry')) { + await runClickhouseMigrationCommands(sqls); + } +} diff --git a/packages/db/index.ts b/packages/db/index.ts index f0e461c3..33e8902a 100644 --- a/packages/db/index.ts +++ b/packages/db/index.ts @@ -31,3 +31,4 @@ export * from './src/services/overview.service'; export * from './src/services/pages.service'; export * from './src/services/insights'; export * from './src/session-context'; +export * from './src/gsc'; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 784f94fb..8bf72e14 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -203,6 +203,7 @@ model Project { notificationRules NotificationRule[] notifications Notification[] imports Import[] + gscConnection GscConnection? // When deleteAt > now(), the project will be deleted deleteAt DateTime? @@ -612,6 +613,24 @@ model InsightEvent { @@map("insight_events") } +model GscConnection { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + projectId String @unique + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + siteUrl String @default("") + accessToken String + refreshToken String + accessTokenExpiresAt DateTime? + lastSyncedAt DateTime? + lastSyncStatus String? + lastSyncError String? + backfillStatus String? + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + + @@map("gsc_connections") +} + model EmailUnsubscribe { id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid email String diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts index d2899f82..3a0ba20a 100644 --- a/packages/db/src/clickhouse/client.ts +++ b/packages/db/src/clickhouse/client.ts @@ -58,6 +58,9 @@ export const TABLE_NAMES = { sessions: 'sessions', events_imports: 'events_imports', session_replay_chunks: 'session_replay_chunks', + gsc_daily: 'gsc_daily', + gsc_pages_daily: 'gsc_pages_daily', + gsc_queries_daily: 'gsc_queries_daily', }; /** diff --git a/packages/db/src/gsc.ts b/packages/db/src/gsc.ts new file mode 100644 index 00000000..6a8e5e08 --- /dev/null +++ b/packages/db/src/gsc.ts @@ -0,0 +1,341 @@ +import { originalCh } from './clickhouse/client'; +import { db } from './prisma-client'; + +export interface GscSite { + siteUrl: string; + permissionLevel: string; +} + +async function refreshGscToken( + refreshToken: string +): Promise<{ accessToken: string; expiresAt: Date }> { + const params = new URLSearchParams({ + client_id: process.env.GOOGLE_CLIENT_ID ?? '', + client_secret: process.env.GOOGLE_CLIENT_SECRET ?? '', + refresh_token: refreshToken, + grant_type: 'refresh_token', + }); + + const res = await fetch('https://oauth2.googleapis.com/token', { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: params.toString(), + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Failed to refresh GSC token: ${text}`); + } + + const data = (await res.json()) as { + access_token: string; + expires_in: number; + }; + const expiresAt = new Date(Date.now() + data.expires_in * 1000); + return { accessToken: data.access_token, expiresAt }; +} + +export async function getGscAccessToken(projectId: string): Promise { + const conn = await db.gscConnection.findUniqueOrThrow({ + where: { projectId }, + }); + + if ( + conn.accessTokenExpiresAt && + conn.accessTokenExpiresAt.getTime() > Date.now() + 60_000 + ) { + return conn.accessToken; + } + + const { accessToken, expiresAt } = await refreshGscToken(conn.refreshToken); + await db.gscConnection.update({ + where: { projectId }, + data: { accessToken, accessTokenExpiresAt: expiresAt }, + }); + return accessToken; +} + +export async function listGscSites(projectId: string): Promise { + const accessToken = await getGscAccessToken(projectId); + const res = await fetch('https://www.googleapis.com/webmaster/v3/sites', { + headers: { Authorization: `Bearer ${accessToken}` }, + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Failed to list GSC sites: ${text}`); + } + + const data = (await res.json()) as { + siteEntry?: Array<{ siteUrl: string; permissionLevel: string }>; + }; + return data.siteEntry ?? []; +} + +interface GscApiRow { + keys: string[]; + clicks: number; + impressions: number; + ctr: number; + position: number; +} + +async function queryGscSearchAnalytics( + accessToken: string, + siteUrl: string, + startDate: string, + endDate: string, + dimensions: string[] +): Promise { + const encodedSiteUrl = encodeURIComponent(siteUrl); + const url = `https://www.googleapis.com/webmaster/v3/sites/${encodedSiteUrl}/searchAnalytics/query`; + + const allRows: GscApiRow[] = []; + let startRow = 0; + const rowLimit = 25000; + + while (true) { + const res = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + startDate, + endDate, + dimensions, + rowLimit, + startRow, + dataState: 'all', + }), + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`GSC query failed for dimensions [${dimensions.join(',')}]: ${text}`); + } + + const data = (await res.json()) as { rows?: GscApiRow[] }; + const rows = data.rows ?? []; + allRows.push(...rows); + + if (rows.length < rowLimit) break; + startRow += rowLimit; + } + + return allRows; +} + +function formatDate(date: Date): string { + return date.toISOString().slice(0, 10); +} + +function nowString(): string { + return new Date().toISOString().replace('T', ' ').replace('Z', ''); +} + +export async function syncGscData( + projectId: string, + startDate: Date, + endDate: Date +): Promise { + const conn = await db.gscConnection.findUniqueOrThrow({ + where: { projectId }, + }); + + if (!conn.siteUrl) { + throw new Error('No GSC site URL configured for this project'); + } + + const accessToken = await getGscAccessToken(projectId); + const start = formatDate(startDate); + const end = formatDate(endDate); + const syncedAt = nowString(); + + // 1. Daily totals — authoritative numbers for overview chart + const dailyRows = await queryGscSearchAnalytics( + accessToken, + conn.siteUrl, + start, + end, + ['date'] + ); + + if (dailyRows.length > 0) { + await originalCh.insert({ + table: 'gsc_daily', + values: dailyRows.map((row) => ({ + project_id: projectId, + date: row.keys[0] ?? '', + clicks: row.clicks, + impressions: row.impressions, + ctr: row.ctr, + position: row.position, + synced_at: syncedAt, + })), + format: 'JSONEachRow', + }); + } + + // 2. Per-page breakdown + const pageRows = await queryGscSearchAnalytics( + accessToken, + conn.siteUrl, + start, + end, + ['date', 'page'] + ); + + if (pageRows.length > 0) { + await originalCh.insert({ + table: 'gsc_pages_daily', + values: pageRows.map((row) => ({ + project_id: projectId, + date: row.keys[0] ?? '', + page: row.keys[1] ?? '', + clicks: row.clicks, + impressions: row.impressions, + ctr: row.ctr, + position: row.position, + synced_at: syncedAt, + })), + format: 'JSONEachRow', + }); + } + + // 3. Per-query breakdown + const queryRows = await queryGscSearchAnalytics( + accessToken, + conn.siteUrl, + start, + end, + ['date', 'query'] + ); + + if (queryRows.length > 0) { + await originalCh.insert({ + table: 'gsc_queries_daily', + values: queryRows.map((row) => ({ + project_id: projectId, + date: row.keys[0] ?? '', + query: row.keys[1] ?? '', + clicks: row.clicks, + impressions: row.impressions, + ctr: row.ctr, + position: row.position, + synced_at: syncedAt, + })), + format: 'JSONEachRow', + }); + } +} + +export async function getGscOverview( + projectId: string, + startDate: string, + endDate: string +): Promise< + Array<{ + date: string; + clicks: number; + impressions: number; + ctr: number; + position: number; + }> +> { + const result = await originalCh.query({ + query: ` + SELECT + date, + sum(clicks) as clicks, + sum(impressions) as impressions, + avg(ctr) as ctr, + avg(position) as position + FROM gsc_daily + FINAL + WHERE project_id = {projectId: String} + AND date >= {startDate: String} + AND date <= {endDate: String} + GROUP BY date + ORDER BY date ASC + `, + query_params: { projectId, startDate, endDate }, + format: 'JSONEachRow', + }); + return result.json(); +} + +export async function getGscPages( + projectId: string, + startDate: string, + endDate: string, + limit = 100 +): Promise< + Array<{ + page: string; + clicks: number; + impressions: number; + ctr: number; + position: number; + }> +> { + const result = await originalCh.query({ + query: ` + SELECT + page, + sum(clicks) as clicks, + sum(impressions) as impressions, + avg(ctr) as ctr, + avg(position) as position + FROM gsc_pages_daily + FINAL + WHERE project_id = {projectId: String} + AND date >= {startDate: String} + AND date <= {endDate: String} + GROUP BY page + ORDER BY clicks DESC + LIMIT {limit: UInt32} + `, + query_params: { projectId, startDate, endDate, limit }, + format: 'JSONEachRow', + }); + return result.json(); +} + +export async function getGscQueries( + projectId: string, + startDate: string, + endDate: string, + limit = 100 +): Promise< + Array<{ + query: string; + clicks: number; + impressions: number; + ctr: number; + position: number; + }> +> { + const result = await originalCh.query({ + query: ` + SELECT + query, + sum(clicks) as clicks, + sum(impressions) as impressions, + avg(ctr) as ctr, + avg(position) as position + FROM gsc_queries_daily + FINAL + WHERE project_id = {projectId: String} + AND date >= {startDate: String} + AND date <= {endDate: String} + GROUP BY query + ORDER BY clicks DESC + LIMIT {limit: UInt32} + `, + query_params: { projectId, startDate, endDate, limit }, + format: 'JSONEachRow', + }); + return result.json(); +} diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts index 4ba92b6a..aa769b48 100644 --- a/packages/queue/src/queues.ts +++ b/packages/queue/src/queues.ts @@ -126,6 +126,10 @@ export type CronQueuePayloadFlushReplay = { type: 'flushReplay'; payload: undefined; }; +export type CronQueuePayloadGscSync = { + type: 'gscSync'; + payload: undefined; +}; export type CronQueuePayload = | CronQueuePayloadSalt | CronQueuePayloadFlushEvents @@ -136,7 +140,8 @@ export type CronQueuePayload = | CronQueuePayloadPing | CronQueuePayloadProject | CronQueuePayloadInsightsDaily - | CronQueuePayloadOnboarding; + | CronQueuePayloadOnboarding + | CronQueuePayloadGscSync; export type MiscQueuePayloadTrialEndingSoon = { type: 'trialEndingSoon'; @@ -268,3 +273,21 @@ export const insightsQueue = new Queue( }, } ); + +export type GscQueuePayloadSync = { + type: 'gscProjectSync'; + payload: { projectId: string }; +}; +export type GscQueuePayloadBackfill = { + type: 'gscProjectBackfill'; + payload: { projectId: string }; +}; +export type GscQueuePayload = GscQueuePayloadSync | GscQueuePayloadBackfill; + +export const gscQueue = new Queue(getQueueName('gsc'), { + connection: getRedisQueue(), + defaultJobOptions: { + removeOnComplete: 50, + removeOnFail: 100, + }, +}); diff --git a/packages/trpc/src/root.ts b/packages/trpc/src/root.ts index 068a321d..626c1312 100644 --- a/packages/trpc/src/root.ts +++ b/packages/trpc/src/root.ts @@ -1,4 +1,5 @@ import { authRouter } from './routers/auth'; +import { gscRouter } from './routers/gsc'; import { chartRouter } from './routers/chart'; import { chatRouter } from './routers/chat'; import { clientRouter } from './routers/client'; @@ -53,6 +54,7 @@ export const appRouter = createTRPCRouter({ insight: insightRouter, widget: widgetRouter, email: emailRouter, + gsc: gscRouter, }); // export type definition of API diff --git a/packages/trpc/src/routers/gsc.ts b/packages/trpc/src/routers/gsc.ts new file mode 100644 index 00000000..2cab43cf --- /dev/null +++ b/packages/trpc/src/routers/gsc.ts @@ -0,0 +1,201 @@ +import { Arctic, googleGsc } from '@openpanel/auth'; +import { + db, + getGscOverview, + getGscPages, + getGscQueries, + listGscSites, +} from '@openpanel/db'; +import { gscQueue } from '@openpanel/queue'; +import { z } from 'zod'; +import { getProjectAccess } from '../access'; +import { TRPCAccessError, TRPCNotFoundError } from '../errors'; +import { createTRPCRouter, protectedProcedure } from '../trpc'; + +export const gscRouter = createTRPCRouter({ + getConnection: protectedProcedure + .input(z.object({ projectId: z.string() })) + .query(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + return db.gscConnection.findUnique({ + where: { projectId: input.projectId }, + select: { + id: true, + siteUrl: true, + lastSyncedAt: true, + lastSyncStatus: true, + lastSyncError: true, + backfillStatus: true, + createdAt: true, + updatedAt: true, + }, + }); + }), + + initiateOAuth: protectedProcedure + .input(z.object({ projectId: z.string() })) + .mutation(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + + const state = Arctic.generateState(); + const codeVerifier = Arctic.generateCodeVerifier(); + const url = googleGsc.createAuthorizationURL(state, codeVerifier, [ + 'https://www.googleapis.com/auth/webmaster.readonly', + ]); + url.searchParams.set('access_type', 'offline'); + url.searchParams.set('prompt', 'consent'); + + return { + url: url.toString(), + state, + codeVerifier, + projectId: input.projectId, + }; + }), + + getSites: protectedProcedure + .input(z.object({ projectId: z.string() })) + .query(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + return listGscSites(input.projectId); + }), + + selectSite: protectedProcedure + .input(z.object({ projectId: z.string(), siteUrl: z.string() })) + .mutation(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + + const conn = await db.gscConnection.findUnique({ + where: { projectId: input.projectId }, + }); + if (!conn) { + throw TRPCNotFoundError('GSC connection not found'); + } + + await db.gscConnection.update({ + where: { projectId: input.projectId }, + data: { + siteUrl: input.siteUrl, + backfillStatus: 'pending', + }, + }); + + await gscQueue.add('gscProjectBackfill', { + type: 'gscProjectBackfill', + payload: { projectId: input.projectId }, + }); + + return { ok: true }; + }), + + disconnect: protectedProcedure + .input(z.object({ projectId: z.string() })) + .mutation(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + + await db.gscConnection.deleteMany({ + where: { projectId: input.projectId }, + }); + + return { ok: true }; + }), + + getOverview: protectedProcedure + .input( + z.object({ + projectId: z.string(), + startDate: z.string(), + endDate: z.string(), + }) + ) + .query(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + return getGscOverview(input.projectId, input.startDate, input.endDate); + }), + + getPages: protectedProcedure + .input( + z.object({ + projectId: z.string(), + startDate: z.string(), + endDate: z.string(), + limit: z.number().min(1).max(1000).optional().default(100), + }) + ) + .query(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + return getGscPages( + input.projectId, + input.startDate, + input.endDate, + input.limit + ); + }), + + getQueries: protectedProcedure + .input( + z.object({ + projectId: z.string(), + startDate: z.string(), + endDate: z.string(), + limit: z.number().min(1).max(1000).optional().default(100), + }) + ) + .query(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + return getGscQueries( + input.projectId, + input.startDate, + input.endDate, + input.limit + ); + }), +});