This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-06 13:59:03 +01:00
parent 70ca44f039
commit 2981638893
21 changed files with 1609 additions and 1 deletions

View File

@@ -0,0 +1,132 @@
import { COOKIE_OPTIONS, googleGsc } from '@openpanel/auth';
import { db } from '@openpanel/db';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { z } from 'zod';
import { LogError } from '@/utils/errors';
export async function gscInitiate(req: FastifyRequest, reply: FastifyReply) {
const schema = z.object({
state: z.string(),
code_verifier: z.string(),
project_id: z.string(),
redirect: z.string().url(),
});
const query = schema.safeParse(req.query);
if (!query.success) {
return reply.status(400).send({ error: 'Invalid parameters' });
}
const { state, code_verifier, project_id, redirect } = query.data;
reply.setCookie('gsc_oauth_state', state, { maxAge: 60 * 10, ...COOKIE_OPTIONS });
reply.setCookie('gsc_code_verifier', code_verifier, { maxAge: 60 * 10, ...COOKIE_OPTIONS });
reply.setCookie('gsc_project_id', project_id, { maxAge: 60 * 10, ...COOKIE_OPTIONS });
return reply.redirect(redirect);
}
export async function gscGoogleCallback(
req: FastifyRequest,
reply: FastifyReply
) {
try {
const schema = z.object({
code: z.string(),
state: z.string(),
});
const query = schema.safeParse(req.query);
if (!query.success) {
throw new LogError('Invalid GSC callback query params', {
error: query.error,
query: req.query,
});
}
const { code, state } = query.data;
const storedState = req.cookies.gsc_oauth_state ?? null;
const codeVerifier = req.cookies.gsc_code_verifier ?? null;
const projectId = req.cookies.gsc_project_id ?? null;
if (!storedState || !codeVerifier || !projectId) {
throw new LogError('Missing GSC OAuth cookies', {
storedState: storedState === null,
codeVerifier: codeVerifier === null,
projectId: projectId === null,
});
}
if (state !== storedState) {
throw new LogError('GSC OAuth state mismatch', { state, storedState });
}
const tokens = await googleGsc.validateAuthorizationCode(
code,
codeVerifier
);
const accessToken = tokens.accessToken();
const refreshToken = tokens.hasRefreshToken()
? tokens.refreshToken()
: null;
const accessTokenExpiresAt = tokens.accessTokenExpiresAt();
if (!refreshToken) {
throw new LogError('No refresh token returned from Google GSC OAuth');
}
const project = await db.project.findUnique({
where: { id: projectId },
select: { id: true, organizationId: true },
});
if (!project) {
throw new LogError('Project not found for GSC connection', { projectId });
}
await db.gscConnection.upsert({
where: { projectId },
create: {
projectId,
accessToken,
refreshToken,
accessTokenExpiresAt,
siteUrl: '',
},
update: {
accessToken,
refreshToken,
accessTokenExpiresAt,
lastSyncStatus: null,
lastSyncError: null,
},
});
reply.clearCookie('gsc_oauth_state');
reply.clearCookie('gsc_code_verifier');
reply.clearCookie('gsc_project_id');
const dashboardUrl =
process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!;
const redirectUrl = `${dashboardUrl}/${project.organizationId}/${projectId}/settings/gsc`;
return reply.redirect(redirectUrl);
} catch (error) {
req.log.error(error);
return redirectWithError(reply, error);
}
}
function redirectWithError(reply: FastifyReply, error: LogError | unknown) {
const url = new URL(
process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!
);
url.pathname = '/login';
if (error instanceof LogError) {
url.searchParams.set('error', error.message);
} else {
url.searchParams.set('error', 'Failed to connect Google Search Console');
}
url.searchParams.set('correlationId', reply.request.id);
return reply.redirect(url.toString());
}

View File

@@ -42,6 +42,7 @@ import liveRouter from './routes/live.router';
import manageRouter from './routes/manage.router';
import miscRouter from './routes/misc.router';
import oauthRouter from './routes/oauth-callback.router';
import gscCallbackRouter from './routes/gsc-callback.router';
import profileRouter from './routes/profile.router';
import trackRouter from './routes/track.router';
import webhookRouter from './routes/webhook.router';
@@ -194,6 +195,7 @@ const startServer = async () => {
instance.register(liveRouter, { prefix: '/live' });
instance.register(webhookRouter, { prefix: '/webhook' });
instance.register(oauthRouter, { prefix: '/oauth' });
instance.register(gscCallbackRouter, { prefix: '/gsc' });
instance.register(miscRouter, { prefix: '/misc' });
instance.register(aiRouter, { prefix: '/ai' });
});

View File

@@ -0,0 +1,17 @@
import { gscGoogleCallback, gscInitiate } from '@/controllers/gsc-oauth-callback.controller';
import type { FastifyPluginCallback } from 'fastify';
const router: FastifyPluginCallback = async (fastify) => {
fastify.route({
method: 'GET',
url: '/initiate',
handler: gscInitiate,
});
fastify.route({
method: 'GET',
url: '/callback',
handler: gscGoogleCallback,
});
};
export default router;

View File

@@ -14,6 +14,7 @@ import {
LayoutDashboardIcon,
LayoutPanelTopIcon,
PlusIcon,
SearchIcon,
SparklesIcon,
TrendingUpDownIcon,
UndoDotIcon,
@@ -59,6 +60,7 @@ export default function SidebarProjectMenu({
<SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" />
<SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" />
<SidebarLink href={'/profiles'} icon={UsersIcon} label="Profiles" />
<SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" />
<div className="mt-4 mb-2 font-medium text-muted-foreground text-sm">
Manage
</div>

View File

@@ -42,6 +42,7 @@ import { Route as AppOrganizationIdProfileTabsRouteImport } from './routes/_app.
import { Route as AppOrganizationIdMembersTabsRouteImport } from './routes/_app.$organizationId.members._tabs'
import { Route as AppOrganizationIdIntegrationsTabsRouteImport } from './routes/_app.$organizationId.integrations._tabs'
import { Route as AppOrganizationIdProjectIdSessionsRouteImport } from './routes/_app.$organizationId.$projectId.sessions'
import { Route as AppOrganizationIdProjectIdSeoRouteImport } from './routes/_app.$organizationId.$projectId.seo'
import { Route as AppOrganizationIdProjectIdReportsRouteImport } from './routes/_app.$organizationId.$projectId.reports'
import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references'
import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime'
@@ -71,6 +72,7 @@ import { Route as AppOrganizationIdProjectIdEventsTabsIndexRouteImport } from '.
import { Route as AppOrganizationIdProjectIdSettingsTabsWidgetsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.widgets'
import { Route as AppOrganizationIdProjectIdSettingsTabsTrackingRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.tracking'
import { Route as AppOrganizationIdProjectIdSettingsTabsImportsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.imports'
import { Route as AppOrganizationIdProjectIdSettingsTabsGscRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.gsc'
import { Route as AppOrganizationIdProjectIdSettingsTabsEventsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.events'
import { Route as AppOrganizationIdProjectIdSettingsTabsDetailsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.details'
import { Route as AppOrganizationIdProjectIdSettingsTabsClientsRouteImport } from './routes/_app.$organizationId.$projectId.settings._tabs.clients'
@@ -312,6 +314,12 @@ const AppOrganizationIdProjectIdSessionsRoute =
path: '/sessions',
getParentRoute: () => AppOrganizationIdProjectIdRoute,
} as any)
const AppOrganizationIdProjectIdSeoRoute =
AppOrganizationIdProjectIdSeoRouteImport.update({
id: '/seo',
path: '/seo',
getParentRoute: () => AppOrganizationIdProjectIdRoute,
} as any)
const AppOrganizationIdProjectIdReportsRoute =
AppOrganizationIdProjectIdReportsRouteImport.update({
id: '/reports',
@@ -488,6 +496,12 @@ const AppOrganizationIdProjectIdSettingsTabsImportsRoute =
path: '/imports',
getParentRoute: () => AppOrganizationIdProjectIdSettingsTabsRoute,
} as any)
const AppOrganizationIdProjectIdSettingsTabsGscRoute =
AppOrganizationIdProjectIdSettingsTabsGscRouteImport.update({
id: '/gsc',
path: '/gsc',
getParentRoute: () => AppOrganizationIdProjectIdSettingsTabsRoute,
} as any)
const AppOrganizationIdProjectIdSettingsTabsEventsRoute =
AppOrganizationIdProjectIdSettingsTabsEventsRouteImport.update({
id: '/events',
@@ -606,6 +620,7 @@ export interface FileRoutesByFullPath {
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
'/$organizationId/$projectId/reports': typeof AppOrganizationIdProjectIdReportsRoute
'/$organizationId/$projectId/seo': typeof AppOrganizationIdProjectIdSeoRoute
'/$organizationId/$projectId/sessions': typeof AppOrganizationIdProjectIdSessionsRoute
'/$organizationId/integrations': typeof AppOrganizationIdIntegrationsTabsRouteWithChildren
'/$organizationId/members': typeof AppOrganizationIdMembersTabsRouteWithChildren
@@ -640,6 +655,7 @@ export interface FileRoutesByFullPath {
'/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
'/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
'/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
'/$organizationId/$projectId/settings/gsc': typeof AppOrganizationIdProjectIdSettingsTabsGscRoute
'/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
'/$organizationId/$projectId/settings/tracking': typeof AppOrganizationIdProjectIdSettingsTabsTrackingRoute
'/$organizationId/$projectId/settings/widgets': typeof AppOrganizationIdProjectIdSettingsTabsWidgetsRoute
@@ -677,6 +693,7 @@ export interface FileRoutesByTo {
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
'/$organizationId/$projectId/reports': typeof AppOrganizationIdProjectIdReportsRoute
'/$organizationId/$projectId/seo': typeof AppOrganizationIdProjectIdSeoRoute
'/$organizationId/$projectId/sessions': typeof AppOrganizationIdProjectIdSessionsRoute
'/$organizationId/integrations': typeof AppOrganizationIdIntegrationsTabsIndexRoute
'/$organizationId/members': typeof AppOrganizationIdMembersTabsIndexRoute
@@ -708,6 +725,7 @@ export interface FileRoutesByTo {
'/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
'/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
'/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
'/$organizationId/$projectId/settings/gsc': typeof AppOrganizationIdProjectIdSettingsTabsGscRoute
'/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
'/$organizationId/$projectId/settings/tracking': typeof AppOrganizationIdProjectIdSettingsTabsTrackingRoute
'/$organizationId/$projectId/settings/widgets': typeof AppOrganizationIdProjectIdSettingsTabsWidgetsRoute
@@ -747,6 +765,7 @@ export interface FileRoutesById {
'/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
'/_app/$organizationId/$projectId/reports': typeof AppOrganizationIdProjectIdReportsRoute
'/_app/$organizationId/$projectId/seo': typeof AppOrganizationIdProjectIdSeoRoute
'/_app/$organizationId/$projectId/sessions': typeof AppOrganizationIdProjectIdSessionsRoute
'/_app/$organizationId/integrations': typeof AppOrganizationIdIntegrationsRouteWithChildren
'/_app/$organizationId/integrations/_tabs': typeof AppOrganizationIdIntegrationsTabsRouteWithChildren
@@ -789,6 +808,7 @@ export interface FileRoutesById {
'/_app/$organizationId/$projectId/settings/_tabs/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
'/_app/$organizationId/$projectId/settings/_tabs/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
'/_app/$organizationId/$projectId/settings/_tabs/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
'/_app/$organizationId/$projectId/settings/_tabs/gsc': typeof AppOrganizationIdProjectIdSettingsTabsGscRoute
'/_app/$organizationId/$projectId/settings/_tabs/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
'/_app/$organizationId/$projectId/settings/_tabs/tracking': typeof AppOrganizationIdProjectIdSettingsTabsTrackingRoute
'/_app/$organizationId/$projectId/settings/_tabs/widgets': typeof AppOrganizationIdProjectIdSettingsTabsWidgetsRoute
@@ -830,6 +850,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/realtime'
| '/$organizationId/$projectId/references'
| '/$organizationId/$projectId/reports'
| '/$organizationId/$projectId/seo'
| '/$organizationId/$projectId/sessions'
| '/$organizationId/integrations'
| '/$organizationId/members'
@@ -864,6 +885,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/settings/clients'
| '/$organizationId/$projectId/settings/details'
| '/$organizationId/$projectId/settings/events'
| '/$organizationId/$projectId/settings/gsc'
| '/$organizationId/$projectId/settings/imports'
| '/$organizationId/$projectId/settings/tracking'
| '/$organizationId/$projectId/settings/widgets'
@@ -901,6 +923,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/realtime'
| '/$organizationId/$projectId/references'
| '/$organizationId/$projectId/reports'
| '/$organizationId/$projectId/seo'
| '/$organizationId/$projectId/sessions'
| '/$organizationId/integrations'
| '/$organizationId/members'
@@ -932,6 +955,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/settings/clients'
| '/$organizationId/$projectId/settings/details'
| '/$organizationId/$projectId/settings/events'
| '/$organizationId/$projectId/settings/gsc'
| '/$organizationId/$projectId/settings/imports'
| '/$organizationId/$projectId/settings/tracking'
| '/$organizationId/$projectId/settings/widgets'
@@ -970,6 +994,7 @@ export interface FileRouteTypes {
| '/_app/$organizationId/$projectId/realtime'
| '/_app/$organizationId/$projectId/references'
| '/_app/$organizationId/$projectId/reports'
| '/_app/$organizationId/$projectId/seo'
| '/_app/$organizationId/$projectId/sessions'
| '/_app/$organizationId/integrations'
| '/_app/$organizationId/integrations/_tabs'
@@ -1012,6 +1037,7 @@ export interface FileRouteTypes {
| '/_app/$organizationId/$projectId/settings/_tabs/clients'
| '/_app/$organizationId/$projectId/settings/_tabs/details'
| '/_app/$organizationId/$projectId/settings/_tabs/events'
| '/_app/$organizationId/$projectId/settings/_tabs/gsc'
| '/_app/$organizationId/$projectId/settings/_tabs/imports'
| '/_app/$organizationId/$projectId/settings/_tabs/tracking'
| '/_app/$organizationId/$projectId/settings/_tabs/widgets'
@@ -1310,6 +1336,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppOrganizationIdProjectIdSessionsRouteImport
parentRoute: typeof AppOrganizationIdProjectIdRoute
}
'/_app/$organizationId/$projectId/seo': {
id: '/_app/$organizationId/$projectId/seo'
path: '/seo'
fullPath: '/$organizationId/$projectId/seo'
preLoaderRoute: typeof AppOrganizationIdProjectIdSeoRouteImport
parentRoute: typeof AppOrganizationIdProjectIdRoute
}
'/_app/$organizationId/$projectId/reports': {
id: '/_app/$organizationId/$projectId/reports'
path: '/reports'
@@ -1520,6 +1553,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRouteImport
parentRoute: typeof AppOrganizationIdProjectIdSettingsTabsRoute
}
'/_app/$organizationId/$projectId/settings/_tabs/gsc': {
id: '/_app/$organizationId/$projectId/settings/_tabs/gsc'
path: '/gsc'
fullPath: '/$organizationId/$projectId/settings/gsc'
preLoaderRoute: typeof AppOrganizationIdProjectIdSettingsTabsGscRouteImport
parentRoute: typeof AppOrganizationIdProjectIdSettingsTabsRoute
}
'/_app/$organizationId/$projectId/settings/_tabs/events': {
id: '/_app/$organizationId/$projectId/settings/_tabs/events'
path: '/events'
@@ -1785,6 +1825,7 @@ interface AppOrganizationIdProjectIdSettingsTabsRouteChildren {
AppOrganizationIdProjectIdSettingsTabsClientsRoute: typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
AppOrganizationIdProjectIdSettingsTabsDetailsRoute: typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
AppOrganizationIdProjectIdSettingsTabsEventsRoute: typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
AppOrganizationIdProjectIdSettingsTabsGscRoute: typeof AppOrganizationIdProjectIdSettingsTabsGscRoute
AppOrganizationIdProjectIdSettingsTabsImportsRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
AppOrganizationIdProjectIdSettingsTabsTrackingRoute: typeof AppOrganizationIdProjectIdSettingsTabsTrackingRoute
AppOrganizationIdProjectIdSettingsTabsWidgetsRoute: typeof AppOrganizationIdProjectIdSettingsTabsWidgetsRoute
@@ -1799,6 +1840,8 @@ const AppOrganizationIdProjectIdSettingsTabsRouteChildren: AppOrganizationIdProj
AppOrganizationIdProjectIdSettingsTabsDetailsRoute,
AppOrganizationIdProjectIdSettingsTabsEventsRoute:
AppOrganizationIdProjectIdSettingsTabsEventsRoute,
AppOrganizationIdProjectIdSettingsTabsGscRoute:
AppOrganizationIdProjectIdSettingsTabsGscRoute,
AppOrganizationIdProjectIdSettingsTabsImportsRoute:
AppOrganizationIdProjectIdSettingsTabsImportsRoute,
AppOrganizationIdProjectIdSettingsTabsTrackingRoute:
@@ -1837,6 +1880,7 @@ interface AppOrganizationIdProjectIdRouteChildren {
AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute
AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute
AppOrganizationIdProjectIdReportsRoute: typeof AppOrganizationIdProjectIdReportsRoute
AppOrganizationIdProjectIdSeoRoute: typeof AppOrganizationIdProjectIdSeoRoute
AppOrganizationIdProjectIdSessionsRoute: typeof AppOrganizationIdProjectIdSessionsRoute
AppOrganizationIdProjectIdIndexRoute: typeof AppOrganizationIdProjectIdIndexRoute
AppOrganizationIdProjectIdDashboardsDashboardIdRoute: typeof AppOrganizationIdProjectIdDashboardsDashboardIdRoute
@@ -1862,6 +1906,7 @@ const AppOrganizationIdProjectIdRouteChildren: AppOrganizationIdProjectIdRouteCh
AppOrganizationIdProjectIdReferencesRoute,
AppOrganizationIdProjectIdReportsRoute:
AppOrganizationIdProjectIdReportsRoute,
AppOrganizationIdProjectIdSeoRoute: AppOrganizationIdProjectIdSeoRoute,
AppOrganizationIdProjectIdSessionsRoute:
AppOrganizationIdProjectIdSessionsRoute,
AppOrganizationIdProjectIdIndexRoute: AppOrganizationIdProjectIdIndexRoute,

View File

@@ -0,0 +1,289 @@
import { PageContainer } from '@/components/page-container';
import { PageHeader } from '@/components/page-header';
import { Skeleton } from '@/components/skeleton';
import { Button } from '@/components/ui/button';
import {
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
} from '@/components/ui/table';
import { useAppParams } from '@/hooks/use-app-params';
import { useTRPC } from '@/integrations/trpc/react';
import { createProjectTitle } from '@/utils/title';
import { useQuery } from '@tanstack/react-query';
import { createFileRoute, useNavigate } from '@tanstack/react-router';
import { subDays, format } from 'date-fns';
import {
Area,
AreaChart,
CartesianGrid,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
} from 'recharts';
export const Route = createFileRoute(
'/_app/$organizationId/$projectId/seo'
)({
component: SeoPage,
head: () => ({
meta: [{ title: createProjectTitle('SEO') }],
}),
});
const startDate = format(subDays(new Date(), 30), 'yyyy-MM-dd');
const endDate = format(subDays(new Date(), 1), 'yyyy-MM-dd');
function SeoPage() {
const { projectId, organizationId } = useAppParams();
const trpc = useTRPC();
const navigate = useNavigate();
const connectionQuery = useQuery(
trpc.gsc.getConnection.queryOptions({ projectId })
);
const connection = connectionQuery.data;
const isConnected = connection && connection.siteUrl;
const overviewQuery = useQuery(
trpc.gsc.getOverview.queryOptions(
{ projectId, startDate, endDate },
{ enabled: !!isConnected }
)
);
const pagesQuery = useQuery(
trpc.gsc.getPages.queryOptions(
{ projectId, startDate, endDate, limit: 50 },
{ enabled: !!isConnected }
)
);
const queriesQuery = useQuery(
trpc.gsc.getQueries.queryOptions(
{ projectId, startDate, endDate, limit: 50 },
{ enabled: !!isConnected }
)
);
if (connectionQuery.isLoading) {
return (
<PageContainer>
<PageHeader title="SEO" description="Google Search Console data" />
<div className="space-y-4 mt-8">
<Skeleton className="h-64 w-full" />
<Skeleton className="h-96 w-full" />
</div>
</PageContainer>
);
}
if (!isConnected) {
return (
<PageContainer>
<PageHeader title="SEO" description="Connect Google Search Console to see your search performance data." />
<div className="mt-16 flex flex-col items-center gap-4 text-center">
<div className="rounded-full bg-muted p-6">
<svg
className="h-12 w-12 text-muted-foreground"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={1.5}
d="M21 21l-4.35-4.35M17 11A6 6 0 1 1 5 11a6 6 0 0 1 12 0z"
/>
</svg>
</div>
<h2 className="text-xl font-semibold">No SEO data yet</h2>
<p className="text-muted-foreground max-w-sm">
Connect Google Search Console to track your search impressions, clicks, and keyword rankings.
</p>
<Button
onClick={() =>
navigate({
to: '/$organizationId/$projectId/settings/gsc',
params: { organizationId, projectId },
})
}
>
Connect Google Search Console
</Button>
</div>
</PageContainer>
);
}
const overview = overviewQuery.data ?? [];
const pages = pagesQuery.data ?? [];
const queries = queriesQuery.data ?? [];
return (
<PageContainer>
<PageHeader
title="SEO"
description={`Search performance for ${connection.siteUrl}`}
/>
<div className="mt-8 space-y-8">
{/* Summary metrics */}
<div className="grid grid-cols-2 gap-4 md:grid-cols-4">
{(['clicks', 'impressions', 'ctr', 'position'] as const).map((metric) => {
const total = overview.reduce((sum, row) => {
if (metric === 'ctr' || metric === 'position') {
return sum + row[metric];
}
return sum + row[metric];
}, 0);
const display =
metric === 'ctr'
? `${((total / Math.max(overview.length, 1)) * 100).toFixed(1)}%`
: metric === 'position'
? (total / Math.max(overview.length, 1)).toFixed(1)
: total.toLocaleString();
const label =
metric === 'ctr'
? 'Avg CTR'
: metric === 'position'
? 'Avg Position'
: metric.charAt(0).toUpperCase() + metric.slice(1);
return (
<div key={metric} className="rounded-lg border p-4">
<div className="text-sm text-muted-foreground">{label}</div>
<div className="text-2xl font-semibold mt-1">{overviewQuery.isLoading ? <Skeleton className="h-8 w-24" /> : display}</div>
</div>
);
})}
</div>
{/* Clicks over time chart */}
<div className="rounded-lg border p-4">
<h3 className="font-medium mb-4">Clicks over time</h3>
{overviewQuery.isLoading ? (
<Skeleton className="h-48 w-full" />
) : (
<ResponsiveContainer width="100%" height={200}>
<AreaChart data={overview}>
<defs>
<linearGradient id="colorClicks" x1="0" y1="0" x2="0" y2="1">
<stop offset="5%" stopColor="hsl(var(--primary))" stopOpacity={0.3} />
<stop offset="95%" stopColor="hsl(var(--primary))" stopOpacity={0} />
</linearGradient>
</defs>
<CartesianGrid strokeDasharray="3 3" className="stroke-border" />
<XAxis dataKey="date" tick={{ fontSize: 12 }} />
<YAxis tick={{ fontSize: 12 }} />
<Tooltip />
<Area
type="monotone"
dataKey="clicks"
stroke="hsl(var(--primary))"
fill="url(#colorClicks)"
strokeWidth={2}
/>
</AreaChart>
</ResponsiveContainer>
)}
</div>
{/* Pages and Queries tables */}
<div className="grid grid-cols-1 gap-8 lg:grid-cols-2">
<GscTable
title="Top pages"
rows={pages}
keyLabel="Page"
keyField="page"
isLoading={pagesQuery.isLoading}
/>
<GscTable
title="Top queries"
rows={queries}
keyLabel="Query"
keyField="query"
isLoading={queriesQuery.isLoading}
/>
</div>
</div>
</PageContainer>
);
}
interface GscTableRow {
clicks: number;
impressions: number;
ctr: number;
position: number;
[key: string]: string | number;
}
function GscTable({
title,
rows,
keyLabel,
keyField,
isLoading,
}: {
title: string;
rows: GscTableRow[];
keyLabel: string;
keyField: string;
isLoading: boolean;
}) {
return (
<div className="rounded-lg border">
<div className="p-4 border-b">
<h3 className="font-medium">{title}</h3>
</div>
<Table>
<TableHeader>
<TableRow>
<TableHead>{keyLabel}</TableHead>
<TableHead className="text-right">Clicks</TableHead>
<TableHead className="text-right">Impressions</TableHead>
<TableHead className="text-right">CTR</TableHead>
<TableHead className="text-right">Position</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{isLoading &&
Array.from({ length: 5 }).map((_, i) => (
<TableRow key={i}>
{Array.from({ length: 5 }).map((_, j) => (
<TableCell key={j}>
<Skeleton className="h-4 w-full" />
</TableCell>
))}
</TableRow>
))}
{!isLoading && rows.length === 0 && (
<TableRow>
<TableCell colSpan={5} className="text-center text-muted-foreground py-8">
No data yet
</TableCell>
</TableRow>
)}
{rows.map((row) => (
<TableRow key={String(row[keyField])}>
<TableCell className="max-w-[200px] truncate font-mono text-xs">
{String(row[keyField])}
</TableCell>
<TableCell className="text-right">{row.clicks.toLocaleString()}</TableCell>
<TableCell className="text-right">{row.impressions.toLocaleString()}</TableCell>
<TableCell className="text-right">{(row.ctr * 100).toFixed(1)}%</TableCell>
<TableCell className="text-right">{row.position.toFixed(1)}</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
);
}

View File

@@ -0,0 +1,274 @@
import { Skeleton } from '@/components/skeleton';
import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import { useAppParams } from '@/hooks/use-app-params';
import { useTRPC } from '@/integrations/trpc/react';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { formatDistanceToNow } from 'date-fns';
import { CheckCircleIcon, Loader2Icon, XCircleIcon } from 'lucide-react';
import { useState } from 'react';
import { toast } from 'sonner';
export const Route = createFileRoute(
'/_app/$organizationId/$projectId/settings/_tabs/gsc'
)({
component: GscSettings,
});
function GscSettings() {
const { projectId } = useAppParams();
const trpc = useTRPC();
const queryClient = useQueryClient();
const [selectedSite, setSelectedSite] = useState('');
const connectionQuery = useQuery(
trpc.gsc.getConnection.queryOptions(
{ projectId },
{ refetchInterval: 5000 }
)
);
const sitesQuery = useQuery(
trpc.gsc.getSites.queryOptions(
{ projectId },
{ enabled: !!connectionQuery.data && !connectionQuery.data.siteUrl }
)
);
const initiateOAuth = useMutation(
trpc.gsc.initiateOAuth.mutationOptions({
onSuccess: (data) => {
// Route through the API /gsc/initiate endpoint which sets cookies then redirects to Google
const apiUrl = (import.meta.env.VITE_API_URL as string) ?? '';
const initiateUrl = new URL(`${apiUrl}/gsc/initiate`);
initiateUrl.searchParams.set('state', data.state);
initiateUrl.searchParams.set('code_verifier', data.codeVerifier);
initiateUrl.searchParams.set('project_id', data.projectId);
initiateUrl.searchParams.set('redirect', data.url);
window.location.href = initiateUrl.toString();
},
onError: () => {
toast.error('Failed to initiate Google Search Console connection');
},
})
);
const selectSite = useMutation(
trpc.gsc.selectSite.mutationOptions({
onSuccess: () => {
toast.success('Site connected', {
description: 'Backfill of 6 months of data has started.',
});
queryClient.invalidateQueries(trpc.gsc.getConnection.pathFilter());
},
onError: () => {
toast.error('Failed to select site');
},
})
);
const disconnect = useMutation(
trpc.gsc.disconnect.mutationOptions({
onSuccess: () => {
toast.success('Disconnected from Google Search Console');
queryClient.invalidateQueries(trpc.gsc.getConnection.pathFilter());
},
onError: () => {
toast.error('Failed to disconnect');
},
})
);
const connection = connectionQuery.data;
if (connectionQuery.isLoading) {
return (
<div className="space-y-4">
<Skeleton className="h-32 w-full" />
</div>
);
}
// Not connected at all
if (!connection) {
return (
<div className="space-y-6">
<div>
<h3 className="text-lg font-medium">Google Search Console</h3>
<p className="text-sm text-muted-foreground mt-1">
Connect your Google Search Console property to import search performance data.
</p>
</div>
<div className="rounded-lg border p-6 flex flex-col gap-4">
<p className="text-sm text-muted-foreground">
You will be redirected to Google to authorize access. Only read-only access to Search Console data is requested.
</p>
<Button
className="w-fit"
onClick={() => initiateOAuth.mutate({ projectId })}
disabled={initiateOAuth.isPending}
>
{initiateOAuth.isPending && (
<Loader2Icon className="mr-2 h-4 w-4 animate-spin" />
)}
Connect Google Search Console
</Button>
</div>
</div>
);
}
// Connected but no site selected yet
if (!connection.siteUrl) {
const sites = sitesQuery.data ?? [];
return (
<div className="space-y-6">
<div>
<h3 className="text-lg font-medium">Select a property</h3>
<p className="text-sm text-muted-foreground mt-1">
Choose which Google Search Console property to connect to this project.
</p>
</div>
<div className="rounded-lg border p-6 space-y-4">
{sitesQuery.isLoading ? (
<Skeleton className="h-10 w-full" />
) : sites.length === 0 ? (
<p className="text-sm text-muted-foreground">
No Search Console properties found for this Google account.
</p>
) : (
<>
<Select value={selectedSite} onValueChange={setSelectedSite}>
<SelectTrigger>
<SelectValue placeholder="Select a property..." />
</SelectTrigger>
<SelectContent>
{sites.map((site) => (
<SelectItem key={site.siteUrl} value={site.siteUrl}>
{site.siteUrl}
</SelectItem>
))}
</SelectContent>
</Select>
<Button
disabled={!selectedSite || selectSite.isPending}
onClick={() => selectSite.mutate({ projectId, siteUrl: selectedSite })}
>
{selectSite.isPending && (
<Loader2Icon className="mr-2 h-4 w-4 animate-spin" />
)}
Connect property
</Button>
</>
)}
</div>
<Button
variant="ghost"
size="sm"
onClick={() => disconnect.mutate({ projectId })}
>
Cancel
</Button>
</div>
);
}
// Fully connected
const syncStatusIcon =
connection.lastSyncStatus === 'success' ? (
<CheckCircleIcon className="h-4 w-4" />
) : connection.lastSyncStatus === 'error' ? (
<XCircleIcon className="h-4 w-4" />
) : null;
const syncStatusVariant =
connection.lastSyncStatus === 'success'
? 'success'
: connection.lastSyncStatus === 'error'
? 'destructive'
: 'secondary';
return (
<div className="space-y-6">
<div>
<h3 className="text-lg font-medium">Google Search Console</h3>
<p className="text-sm text-muted-foreground mt-1">
Connected to Google Search Console.
</p>
</div>
<div className="rounded-lg border divide-y">
<div className="p-4 flex items-center justify-between">
<div className="text-sm font-medium">Property</div>
<div className="font-mono text-sm text-muted-foreground">
{connection.siteUrl}
</div>
</div>
{connection.backfillStatus && (
<div className="p-4 flex items-center justify-between">
<div className="text-sm font-medium">Backfill</div>
<Badge className="capitalize" variant={
connection.backfillStatus === 'completed' ? 'success' :
connection.backfillStatus === 'failed' ? 'destructive' :
connection.backfillStatus === 'running' ? 'default' : 'secondary'
}>
{connection.backfillStatus === 'running' && (
<Loader2Icon className="mr-1 h-3 w-3 animate-spin" />
)}
{connection.backfillStatus}
</Badge>
</div>
)}
{connection.lastSyncedAt && (
<div className="p-4 flex items-center justify-between">
<div className="text-sm font-medium">Last synced</div>
<div className="flex items-center gap-2">
{connection.lastSyncStatus && (
<Badge className="capitalize" variant={syncStatusVariant as any}>
{syncStatusIcon}
{connection.lastSyncStatus}
</Badge>
)}
<span className="text-sm text-muted-foreground">
{formatDistanceToNow(new Date(connection.lastSyncedAt), {
addSuffix: true,
})}
</span>
</div>
</div>
)}
{connection.lastSyncError && (
<div className="p-4">
<div className="text-sm font-medium text-destructive">Last error</div>
<div className="mt-1 text-sm text-muted-foreground font-mono break-words">
{connection.lastSyncError}
</div>
</div>
)}
</div>
<Button
variant="destructive"
size="sm"
onClick={() => disconnect.mutate({ projectId })}
disabled={disconnect.isPending}
>
{disconnect.isPending && (
<Loader2Icon className="mr-2 h-4 w-4 animate-spin" />
)}
Disconnect
</Button>
</div>
);
}

View File

@@ -45,6 +45,7 @@ function ProjectDashboard() {
{ id: 'tracking', label: 'Tracking script' },
{ id: 'widgets', label: 'Widgets' },
{ id: 'imports', label: 'Imports' },
{ id: 'gsc', label: 'Google Search' },
];
const handleTabChange = (tabId: string) => {

View File

@@ -78,6 +78,11 @@ export async function bootCron() {
type: 'onboarding',
pattern: '0 * * * *',
},
{
name: 'gscSync',
type: 'gscSync',
pattern: '0 3 * * *',
},
];
if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {

View File

@@ -6,6 +6,7 @@ import {
type EventsQueuePayloadIncomingEvent,
cronQueue,
eventsGroupQueues,
gscQueue,
importQueue,
insightsQueue,
miscQueue,
@@ -20,6 +21,7 @@ import { setTimeout as sleep } from 'node:timers/promises';
import { Worker as GroupWorker } from 'groupmq';
import { cronJob } from './jobs/cron';
import { gscJob } from './jobs/gsc';
import { incomingEvent } from './jobs/events.incoming-event';
import { importJob } from './jobs/import';
import { insightsProjectJob } from './jobs/insights';
@@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] {
'misc',
'import',
'insights',
'gsc',
];
}
@@ -208,6 +211,17 @@ export async function bootWorkers() {
logger.info('Started worker for insights', { concurrency });
}
// Start gsc worker
if (enabledQueues.includes('gsc')) {
const concurrency = getConcurrencyFor('gsc', 5);
const gscWorker = new Worker(gscQueue.name, gscJob, {
...workerOptions,
concurrency,
});
workers.push(gscWorker);
logger.info('Started worker for gsc', { concurrency });
}
if (workers.length === 0) {
logger.warn(
'No workers started. Check ENABLED_QUEUES environment variable.',

View File

@@ -4,6 +4,7 @@ import { eventBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessio
import type { CronQueuePayload } from '@openpanel/queue';
import { jobdeleteProjects } from './cron.delete-projects';
import { gscSyncAllJob } from './gsc';
import { onboardingJob } from './cron.onboarding';
import { ping } from './cron.ping';
import { salt } from './cron.salt';
@@ -41,5 +42,8 @@ export async function cronJob(job: Job<CronQueuePayload>) {
case 'onboarding': {
return await onboardingJob(job);
}
case 'gscSync': {
return await gscSyncAllJob();
}
}
}

142
apps/worker/src/jobs/gsc.ts Normal file
View File

@@ -0,0 +1,142 @@
import { db, syncGscData } from '@openpanel/db';
import { gscQueue } from '@openpanel/queue';
import type { GscQueuePayload } from '@openpanel/queue';
import type { Job } from 'bullmq';
import { logger } from '../utils/logger';
const BACKFILL_MONTHS = 6;
const CHUNK_DAYS = 14;
export async function gscJob(job: Job<GscQueuePayload>) {
switch (job.data.type) {
case 'gscProjectSync':
return gscProjectSyncJob(job.data.payload.projectId);
case 'gscProjectBackfill':
return gscProjectBackfillJob(job.data.payload.projectId);
}
}
async function gscProjectSyncJob(projectId: string) {
const conn = await db.gscConnection.findUnique({ where: { projectId } });
if (!conn?.siteUrl) {
logger.warn('GSC sync skipped: no connection or siteUrl', { projectId });
return;
}
try {
// Sync rolling 3-day window (GSC data can arrive late)
const endDate = new Date();
endDate.setDate(endDate.getDate() - 1); // yesterday
const startDate = new Date(endDate);
startDate.setDate(startDate.getDate() - 2); // 3 days total
await syncGscData(projectId, startDate, endDate);
await db.gscConnection.update({
where: { projectId },
data: {
lastSyncedAt: new Date(),
lastSyncStatus: 'success',
lastSyncError: null,
},
});
logger.info('GSC sync completed', { projectId });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
await db.gscConnection.update({
where: { projectId },
data: {
lastSyncedAt: new Date(),
lastSyncStatus: 'error',
lastSyncError: message,
},
});
logger.error('GSC sync failed', { projectId, error });
throw error;
}
}
async function gscProjectBackfillJob(projectId: string) {
const conn = await db.gscConnection.findUnique({ where: { projectId } });
if (!conn?.siteUrl) {
logger.warn('GSC backfill skipped: no connection or siteUrl', { projectId });
return;
}
await db.gscConnection.update({
where: { projectId },
data: { backfillStatus: 'running' },
});
try {
const endDate = new Date();
endDate.setDate(endDate.getDate() - 1); // yesterday
const startDate = new Date(endDate);
startDate.setMonth(startDate.getMonth() - BACKFILL_MONTHS);
// Process in chunks to avoid timeouts and respect API limits
let chunkEnd = new Date(endDate);
while (chunkEnd > startDate) {
const chunkStart = new Date(chunkEnd);
chunkStart.setDate(chunkStart.getDate() - CHUNK_DAYS + 1);
if (chunkStart < startDate) {
chunkStart.setTime(startDate.getTime());
}
logger.info('GSC backfill chunk', {
projectId,
from: chunkStart.toISOString().slice(0, 10),
to: chunkEnd.toISOString().slice(0, 10),
});
await syncGscData(projectId, chunkStart, chunkEnd);
chunkEnd = new Date(chunkStart);
chunkEnd.setDate(chunkEnd.getDate() - 1);
}
await db.gscConnection.update({
where: { projectId },
data: {
backfillStatus: 'completed',
lastSyncedAt: new Date(),
lastSyncStatus: 'success',
lastSyncError: null,
},
});
logger.info('GSC backfill completed', { projectId });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
await db.gscConnection.update({
where: { projectId },
data: {
backfillStatus: 'failed',
lastSyncStatus: 'error',
lastSyncError: message,
},
});
logger.error('GSC backfill failed', { projectId, error });
throw error;
}
}
export async function gscSyncAllJob() {
const connections = await db.gscConnection.findMany({
where: {
siteUrl: { not: '' },
},
select: { projectId: true },
});
logger.info('GSC nightly sync: enqueuing projects', {
count: connections.length,
});
for (const conn of connections) {
await gscQueue.add('gscProjectSync', {
type: 'gscProjectSync',
payload: { projectId: conn.projectId },
});
}
}

View File

@@ -16,3 +16,9 @@ export const google = new Arctic.Google(
process.env.GOOGLE_CLIENT_SECRET ?? '',
process.env.GOOGLE_REDIRECT_URI ?? '',
);
export const googleGsc = new Arctic.Google(
process.env.GOOGLE_CLIENT_ID ?? '',
process.env.GOOGLE_CLIENT_SECRET ?? '',
process.env.GSC_GOOGLE_REDIRECT_URI ?? '',
);

View File

@@ -0,0 +1,85 @@
import fs from 'node:fs';
import path from 'node:path';
import { createTable, runClickhouseMigrationCommands } from '../src/clickhouse/migration';
import { getIsCluster } from './helpers';
export async function up() {
const isClustered = getIsCluster();
const commonMetricColumns = [
'`clicks` UInt32 CODEC(Delta(4), LZ4)',
'`impressions` UInt32 CODEC(Delta(4), LZ4)',
'`ctr` Float32 CODEC(Gorilla, LZ4)',
'`position` Float32 CODEC(Gorilla, LZ4)',
'`synced_at` DateTime DEFAULT now() CODEC(Delta(4), LZ4)',
];
const sqls: string[] = [
// Daily totals — accurate overview numbers
...createTable({
name: 'gsc_daily',
columns: [
'`project_id` String CODEC(ZSTD(3))',
'`date` Date CODEC(Delta(2), LZ4)',
...commonMetricColumns,
],
orderBy: ['project_id', 'date'],
partitionBy: 'toYYYYMM(date)',
engine: 'ReplacingMergeTree(synced_at)',
distributionHash: 'cityHash64(project_id)',
replicatedVersion: '1',
isClustered,
}),
// Per-page breakdown
...createTable({
name: 'gsc_pages_daily',
columns: [
'`project_id` String CODEC(ZSTD(3))',
'`date` Date CODEC(Delta(2), LZ4)',
'`page` String CODEC(ZSTD(3))',
...commonMetricColumns,
],
orderBy: ['project_id', 'date', 'page'],
partitionBy: 'toYYYYMM(date)',
engine: 'ReplacingMergeTree(synced_at)',
distributionHash: 'cityHash64(project_id)',
replicatedVersion: '1',
isClustered,
}),
// Per-query breakdown
...createTable({
name: 'gsc_queries_daily',
columns: [
'`project_id` String CODEC(ZSTD(3))',
'`date` Date CODEC(Delta(2), LZ4)',
'`query` String CODEC(ZSTD(3))',
...commonMetricColumns,
],
orderBy: ['project_id', 'date', 'query'],
partitionBy: 'toYYYYMM(date)',
engine: 'ReplacingMergeTree(synced_at)',
distributionHash: 'cityHash64(project_id)',
replicatedVersion: '1',
isClustered,
}),
];
fs.writeFileSync(
path.join(__filename.replace('.ts', '.sql')),
sqls
.map((sql) =>
sql
.trim()
.replace(/;$/, '')
.replace(/\n{2,}/g, '\n')
.concat(';'),
)
.join('\n\n---\n\n'),
);
if (!process.argv.includes('--dry')) {
await runClickhouseMigrationCommands(sqls);
}
}

View File

@@ -31,3 +31,4 @@ export * from './src/services/overview.service';
export * from './src/services/pages.service';
export * from './src/services/insights';
export * from './src/session-context';
export * from './src/gsc';

View File

@@ -203,6 +203,7 @@ model Project {
notificationRules NotificationRule[]
notifications Notification[]
imports Import[]
gscConnection GscConnection?
// When deleteAt > now(), the project will be deleted
deleteAt DateTime?
@@ -612,6 +613,24 @@ model InsightEvent {
@@map("insight_events")
}
model GscConnection {
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
projectId String @unique
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
siteUrl String @default("")
accessToken String
refreshToken String
accessTokenExpiresAt DateTime?
lastSyncedAt DateTime?
lastSyncStatus String?
lastSyncError String?
backfillStatus String?
createdAt DateTime @default(now())
updatedAt DateTime @default(now()) @updatedAt
@@map("gsc_connections")
}
model EmailUnsubscribe {
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
email String

View File

@@ -58,6 +58,9 @@ export const TABLE_NAMES = {
sessions: 'sessions',
events_imports: 'events_imports',
session_replay_chunks: 'session_replay_chunks',
gsc_daily: 'gsc_daily',
gsc_pages_daily: 'gsc_pages_daily',
gsc_queries_daily: 'gsc_queries_daily',
};
/**

341
packages/db/src/gsc.ts Normal file
View File

@@ -0,0 +1,341 @@
import { originalCh } from './clickhouse/client';
import { db } from './prisma-client';
export interface GscSite {
siteUrl: string;
permissionLevel: string;
}
async function refreshGscToken(
refreshToken: string
): Promise<{ accessToken: string; expiresAt: Date }> {
const params = new URLSearchParams({
client_id: process.env.GOOGLE_CLIENT_ID ?? '',
client_secret: process.env.GOOGLE_CLIENT_SECRET ?? '',
refresh_token: refreshToken,
grant_type: 'refresh_token',
});
const res = await fetch('https://oauth2.googleapis.com/token', {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: params.toString(),
});
if (!res.ok) {
const text = await res.text();
throw new Error(`Failed to refresh GSC token: ${text}`);
}
const data = (await res.json()) as {
access_token: string;
expires_in: number;
};
const expiresAt = new Date(Date.now() + data.expires_in * 1000);
return { accessToken: data.access_token, expiresAt };
}
export async function getGscAccessToken(projectId: string): Promise<string> {
const conn = await db.gscConnection.findUniqueOrThrow({
where: { projectId },
});
if (
conn.accessTokenExpiresAt &&
conn.accessTokenExpiresAt.getTime() > Date.now() + 60_000
) {
return conn.accessToken;
}
const { accessToken, expiresAt } = await refreshGscToken(conn.refreshToken);
await db.gscConnection.update({
where: { projectId },
data: { accessToken, accessTokenExpiresAt: expiresAt },
});
return accessToken;
}
export async function listGscSites(projectId: string): Promise<GscSite[]> {
const accessToken = await getGscAccessToken(projectId);
const res = await fetch('https://www.googleapis.com/webmaster/v3/sites', {
headers: { Authorization: `Bearer ${accessToken}` },
});
if (!res.ok) {
const text = await res.text();
throw new Error(`Failed to list GSC sites: ${text}`);
}
const data = (await res.json()) as {
siteEntry?: Array<{ siteUrl: string; permissionLevel: string }>;
};
return data.siteEntry ?? [];
}
interface GscApiRow {
keys: string[];
clicks: number;
impressions: number;
ctr: number;
position: number;
}
async function queryGscSearchAnalytics(
accessToken: string,
siteUrl: string,
startDate: string,
endDate: string,
dimensions: string[]
): Promise<GscApiRow[]> {
const encodedSiteUrl = encodeURIComponent(siteUrl);
const url = `https://www.googleapis.com/webmaster/v3/sites/${encodedSiteUrl}/searchAnalytics/query`;
const allRows: GscApiRow[] = [];
let startRow = 0;
const rowLimit = 25000;
while (true) {
const res = await fetch(url, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
startDate,
endDate,
dimensions,
rowLimit,
startRow,
dataState: 'all',
}),
});
if (!res.ok) {
const text = await res.text();
throw new Error(`GSC query failed for dimensions [${dimensions.join(',')}]: ${text}`);
}
const data = (await res.json()) as { rows?: GscApiRow[] };
const rows = data.rows ?? [];
allRows.push(...rows);
if (rows.length < rowLimit) break;
startRow += rowLimit;
}
return allRows;
}
function formatDate(date: Date): string {
return date.toISOString().slice(0, 10);
}
function nowString(): string {
return new Date().toISOString().replace('T', ' ').replace('Z', '');
}
export async function syncGscData(
projectId: string,
startDate: Date,
endDate: Date
): Promise<void> {
const conn = await db.gscConnection.findUniqueOrThrow({
where: { projectId },
});
if (!conn.siteUrl) {
throw new Error('No GSC site URL configured for this project');
}
const accessToken = await getGscAccessToken(projectId);
const start = formatDate(startDate);
const end = formatDate(endDate);
const syncedAt = nowString();
// 1. Daily totals — authoritative numbers for overview chart
const dailyRows = await queryGscSearchAnalytics(
accessToken,
conn.siteUrl,
start,
end,
['date']
);
if (dailyRows.length > 0) {
await originalCh.insert({
table: 'gsc_daily',
values: dailyRows.map((row) => ({
project_id: projectId,
date: row.keys[0] ?? '',
clicks: row.clicks,
impressions: row.impressions,
ctr: row.ctr,
position: row.position,
synced_at: syncedAt,
})),
format: 'JSONEachRow',
});
}
// 2. Per-page breakdown
const pageRows = await queryGscSearchAnalytics(
accessToken,
conn.siteUrl,
start,
end,
['date', 'page']
);
if (pageRows.length > 0) {
await originalCh.insert({
table: 'gsc_pages_daily',
values: pageRows.map((row) => ({
project_id: projectId,
date: row.keys[0] ?? '',
page: row.keys[1] ?? '',
clicks: row.clicks,
impressions: row.impressions,
ctr: row.ctr,
position: row.position,
synced_at: syncedAt,
})),
format: 'JSONEachRow',
});
}
// 3. Per-query breakdown
const queryRows = await queryGscSearchAnalytics(
accessToken,
conn.siteUrl,
start,
end,
['date', 'query']
);
if (queryRows.length > 0) {
await originalCh.insert({
table: 'gsc_queries_daily',
values: queryRows.map((row) => ({
project_id: projectId,
date: row.keys[0] ?? '',
query: row.keys[1] ?? '',
clicks: row.clicks,
impressions: row.impressions,
ctr: row.ctr,
position: row.position,
synced_at: syncedAt,
})),
format: 'JSONEachRow',
});
}
}
export async function getGscOverview(
projectId: string,
startDate: string,
endDate: string
): Promise<
Array<{
date: string;
clicks: number;
impressions: number;
ctr: number;
position: number;
}>
> {
const result = await originalCh.query({
query: `
SELECT
date,
sum(clicks) as clicks,
sum(impressions) as impressions,
avg(ctr) as ctr,
avg(position) as position
FROM gsc_daily
FINAL
WHERE project_id = {projectId: String}
AND date >= {startDate: String}
AND date <= {endDate: String}
GROUP BY date
ORDER BY date ASC
`,
query_params: { projectId, startDate, endDate },
format: 'JSONEachRow',
});
return result.json();
}
export async function getGscPages(
projectId: string,
startDate: string,
endDate: string,
limit = 100
): Promise<
Array<{
page: string;
clicks: number;
impressions: number;
ctr: number;
position: number;
}>
> {
const result = await originalCh.query({
query: `
SELECT
page,
sum(clicks) as clicks,
sum(impressions) as impressions,
avg(ctr) as ctr,
avg(position) as position
FROM gsc_pages_daily
FINAL
WHERE project_id = {projectId: String}
AND date >= {startDate: String}
AND date <= {endDate: String}
GROUP BY page
ORDER BY clicks DESC
LIMIT {limit: UInt32}
`,
query_params: { projectId, startDate, endDate, limit },
format: 'JSONEachRow',
});
return result.json();
}
export async function getGscQueries(
projectId: string,
startDate: string,
endDate: string,
limit = 100
): Promise<
Array<{
query: string;
clicks: number;
impressions: number;
ctr: number;
position: number;
}>
> {
const result = await originalCh.query({
query: `
SELECT
query,
sum(clicks) as clicks,
sum(impressions) as impressions,
avg(ctr) as ctr,
avg(position) as position
FROM gsc_queries_daily
FINAL
WHERE project_id = {projectId: String}
AND date >= {startDate: String}
AND date <= {endDate: String}
GROUP BY query
ORDER BY clicks DESC
LIMIT {limit: UInt32}
`,
query_params: { projectId, startDate, endDate, limit },
format: 'JSONEachRow',
});
return result.json();
}

View File

@@ -126,6 +126,10 @@ export type CronQueuePayloadFlushReplay = {
type: 'flushReplay';
payload: undefined;
};
export type CronQueuePayloadGscSync = {
type: 'gscSync';
payload: undefined;
};
export type CronQueuePayload =
| CronQueuePayloadSalt
| CronQueuePayloadFlushEvents
@@ -136,7 +140,8 @@ export type CronQueuePayload =
| CronQueuePayloadPing
| CronQueuePayloadProject
| CronQueuePayloadInsightsDaily
| CronQueuePayloadOnboarding;
| CronQueuePayloadOnboarding
| CronQueuePayloadGscSync;
export type MiscQueuePayloadTrialEndingSoon = {
type: 'trialEndingSoon';
@@ -268,3 +273,21 @@ export const insightsQueue = new Queue<InsightsQueuePayloadProject>(
},
}
);
export type GscQueuePayloadSync = {
type: 'gscProjectSync';
payload: { projectId: string };
};
export type GscQueuePayloadBackfill = {
type: 'gscProjectBackfill';
payload: { projectId: string };
};
export type GscQueuePayload = GscQueuePayloadSync | GscQueuePayloadBackfill;
export const gscQueue = new Queue<GscQueuePayload>(getQueueName('gsc'), {
connection: getRedisQueue(),
defaultJobOptions: {
removeOnComplete: 50,
removeOnFail: 100,
},
});

View File

@@ -1,4 +1,5 @@
import { authRouter } from './routers/auth';
import { gscRouter } from './routers/gsc';
import { chartRouter } from './routers/chart';
import { chatRouter } from './routers/chat';
import { clientRouter } from './routers/client';
@@ -53,6 +54,7 @@ export const appRouter = createTRPCRouter({
insight: insightRouter,
widget: widgetRouter,
email: emailRouter,
gsc: gscRouter,
});
// export type definition of API

View File

@@ -0,0 +1,201 @@
import { Arctic, googleGsc } from '@openpanel/auth';
import {
db,
getGscOverview,
getGscPages,
getGscQueries,
listGscSites,
} from '@openpanel/db';
import { gscQueue } from '@openpanel/queue';
import { z } from 'zod';
import { getProjectAccess } from '../access';
import { TRPCAccessError, TRPCNotFoundError } from '../errors';
import { createTRPCRouter, protectedProcedure } from '../trpc';
export const gscRouter = createTRPCRouter({
getConnection: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ input, ctx }) => {
const access = await getProjectAccess({
projectId: input.projectId,
userId: ctx.session.userId,
});
if (!access) {
throw TRPCAccessError('You do not have access to this project');
}
return db.gscConnection.findUnique({
where: { projectId: input.projectId },
select: {
id: true,
siteUrl: true,
lastSyncedAt: true,
lastSyncStatus: true,
lastSyncError: true,
backfillStatus: true,
createdAt: true,
updatedAt: true,
},
});
}),
initiateOAuth: protectedProcedure
.input(z.object({ projectId: z.string() }))
.mutation(async ({ input, ctx }) => {
const access = await getProjectAccess({
projectId: input.projectId,
userId: ctx.session.userId,
});
if (!access) {
throw TRPCAccessError('You do not have access to this project');
}
const state = Arctic.generateState();
const codeVerifier = Arctic.generateCodeVerifier();
const url = googleGsc.createAuthorizationURL(state, codeVerifier, [
'https://www.googleapis.com/auth/webmaster.readonly',
]);
url.searchParams.set('access_type', 'offline');
url.searchParams.set('prompt', 'consent');
return {
url: url.toString(),
state,
codeVerifier,
projectId: input.projectId,
};
}),
getSites: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ input, ctx }) => {
const access = await getProjectAccess({
projectId: input.projectId,
userId: ctx.session.userId,
});
if (!access) {
throw TRPCAccessError('You do not have access to this project');
}
return listGscSites(input.projectId);
}),
selectSite: protectedProcedure
.input(z.object({ projectId: z.string(), siteUrl: z.string() }))
.mutation(async ({ input, ctx }) => {
const access = await getProjectAccess({
projectId: input.projectId,
userId: ctx.session.userId,
});
if (!access) {
throw TRPCAccessError('You do not have access to this project');
}
const conn = await db.gscConnection.findUnique({
where: { projectId: input.projectId },
});
if (!conn) {
throw TRPCNotFoundError('GSC connection not found');
}
await db.gscConnection.update({
where: { projectId: input.projectId },
data: {
siteUrl: input.siteUrl,
backfillStatus: 'pending',
},
});
await gscQueue.add('gscProjectBackfill', {
type: 'gscProjectBackfill',
payload: { projectId: input.projectId },
});
return { ok: true };
}),
disconnect: protectedProcedure
.input(z.object({ projectId: z.string() }))
.mutation(async ({ input, ctx }) => {
const access = await getProjectAccess({
projectId: input.projectId,
userId: ctx.session.userId,
});
if (!access) {
throw TRPCAccessError('You do not have access to this project');
}
await db.gscConnection.deleteMany({
where: { projectId: input.projectId },
});
return { ok: true };
}),
getOverview: protectedProcedure
.input(
z.object({
projectId: z.string(),
startDate: z.string(),
endDate: z.string(),
})
)
.query(async ({ input, ctx }) => {
const access = await getProjectAccess({
projectId: input.projectId,
userId: ctx.session.userId,
});
if (!access) {
throw TRPCAccessError('You do not have access to this project');
}
return getGscOverview(input.projectId, input.startDate, input.endDate);
}),
getPages: protectedProcedure
.input(
z.object({
projectId: z.string(),
startDate: z.string(),
endDate: z.string(),
limit: z.number().min(1).max(1000).optional().default(100),
})
)
.query(async ({ input, ctx }) => {
const access = await getProjectAccess({
projectId: input.projectId,
userId: ctx.session.userId,
});
if (!access) {
throw TRPCAccessError('You do not have access to this project');
}
return getGscPages(
input.projectId,
input.startDate,
input.endDate,
input.limit
);
}),
getQueries: protectedProcedure
.input(
z.object({
projectId: z.string(),
startDate: z.string(),
endDate: z.string(),
limit: z.number().min(1).max(1000).optional().default(100),
})
)
.query(async ({ input, ctx }) => {
const access = await getProjectAccess({
projectId: input.projectId,
userId: ctx.session.userId,
});
if (!access) {
throw TRPCAccessError('You do not have access to this project');
}
return getGscQueries(
input.projectId,
input.startDate,
input.endDate,
input.limit
);
}),
});