feat(geo): make geo a package instead of service (#161)

.github/workflows/docker-build.yml

@@ -2,7 +2,7 @@ name: Docker Build and Push
 
 on:
   push:
-    branches: [ "main" ]
+    # branches: [ "main" ]
     paths:
       - 'apps/api/**'
       - 'apps/worker/**'

.gitignore

@@ -8,6 +8,7 @@ dump-*
 .sql
 tmp
 docker/data*
+*.mmdb
 
 # Logs
 

apps/api/Dockerfile

@@ -28,6 +28,7 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY apps/api/package.json ./apps/api/
 # Packages
 COPY packages/db/package.json packages/db/
+COPY packages/geo/package.json packages/geo/
 COPY packages/trpc/package.json packages/trpc/
 COPY packages/auth/package.json packages/auth/
 COPY packages/json/package.json packages/json/
@@ -59,7 +60,7 @@ COPY apps/api ./apps/api
 COPY packages ./packages
 COPY tooling ./tooling
 
-RUN pnpm db:codegen && \
+RUN pnpm codegen && \
     pnpm --filter api run build
 
 # PROD
@@ -91,6 +92,7 @@ COPY --from=build /app/apps/api ./apps/api
 
 # Packages
 COPY --from=build /app/packages/db ./packages/db
+COPY --from=build /app/packages/geo ./packages/geo
 COPY --from=build /app/packages/auth ./packages/auth
 COPY --from=build /app/packages/trpc ./packages/trpc
 COPY --from=build /app/packages/json ./packages/json

apps/api/package.json

@@ -23,6 +23,7 @@
     "@openpanel/common": "workspace:*",
     "@openpanel/constants": "workspace:*",
     "@openpanel/db": "workspace:*",
+    "@openpanel/geo": "workspace:*",
     "@openpanel/integrations": "workspace:^",
     "@openpanel/json": "workspace:*",
     "@openpanel/logger": "workspace:*",

apps/api/src/controllers/event.controller.ts

@@ -1,4 +1,4 @@
-import { getClientIp, parseIp } from '@/utils/parse-ip';
+import { getClientIp } from '@/utils/get-client-ip';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 
 import { generateDeviceId } from '@openpanel/common/server';
@@ -8,6 +8,7 @@ import { getLock } from '@openpanel/redis';
 import type { PostEventPayload } from '@openpanel/sdk';
 
 import { checkDuplicatedEvent } from '@/utils/deduplicate';
+import { getGeoLocation } from '@openpanel/geo';
 import { getStringHeaders, getTimestamp } from './track.controller';
 
 export async function postEvent(
@@ -26,7 +27,7 @@ export async function postEvent(
     return;
   }
 
-  const [salts, geo] = await Promise.all([getSalts(), parseIp(ip)]);
+  const [salts, geo] = await Promise.all([getSalts(), getGeoLocation(ip)]);
   const currentDeviceId = generateDeviceId({
     salt: salts.current,
     origin: projectId,

apps/api/src/controllers/misc.controller.ts

@@ -4,8 +4,10 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
 import icoToPng from 'ico-to-png';
 import sharp from 'sharp';
 
+import { getClientIp } from '@/utils/get-client-ip';
 import { createHash } from '@openpanel/common/server';
 import { TABLE_NAMES, ch, chQuery, formatClickhouseDate } from '@openpanel/db';
+import { getGeoLocation } from '@openpanel/geo';
 import { cacheable, getCache, getRedisCache } from '@openpanel/redis';
 
 interface GetFaviconParams {
@@ -170,3 +172,12 @@ export async function stats(request: FastifyRequest, reply: FastifyReply) {
     eventsLast24hCount: res.last24hCount,
   });
 }
+
+export async function getGeo(request: FastifyRequest, reply: FastifyReply) {
+  const ip = getClientIp(request);
+  if (!ip) {
+    return reply.status(400).send('Bad Request');
+  }
+  const geo = await getGeoLocation(ip);
+  return reply.status(200).send(geo);
+}

apps/api/src/controllers/profile.controller.ts

@@ -1,10 +1,11 @@
-import { getClientIp, parseIp } from '@/utils/parse-ip';
+import { getClientIp } from '@/utils/get-client-ip';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { assocPath, pathOr } from 'ramda';
 
 import { checkDuplicatedEvent, isDuplicatedEvent } from '@/utils/deduplicate';
 import { parseUserAgent } from '@openpanel/common/server';
 import { getProfileById, upsertProfile } from '@openpanel/db';
+import { getGeoLocation } from '@openpanel/geo';
 import type {
   IncrementProfilePayload,
   UpdateProfilePayload,
@@ -24,7 +25,7 @@ export async function updateProfile(
   const ip = getClientIp(request)!;
   const ua = request.headers['user-agent']!;
   const uaInfo = parseUserAgent(ua, properties);
-  const geo = await parseIp(ip);
+  const geo = await getGeoLocation(ip);
 
   if (
     await checkDuplicatedEvent({

apps/api/src/controllers/track.controller.ts

@@ -1,11 +1,11 @@
-import type { GeoLocation } from '@/utils/parse-ip';
+import { getClientIp } from '@/utils/get-client-ip';
-import { getClientIp, parseIp } from '@/utils/parse-ip';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { path, assocPath, pathOr, pick } from 'ramda';
 
-import { checkDuplicatedEvent, isDuplicatedEvent } from '@/utils/deduplicate';
+import { checkDuplicatedEvent } from '@/utils/deduplicate';
 import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
 import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
+import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
 import { eventsQueue } from '@openpanel/queue';
 import { getLock } from '@openpanel/redis';
 import type {
@@ -114,7 +114,7 @@ export async function handler(
 
   switch (request.body.type) {
     case 'track': {
-      const [salts, geo] = await Promise.all([getSalts(), parseIp(ip)]);
+      const [salts, geo] = await Promise.all([getSalts(), getGeoLocation(ip)]);
       const currentDeviceId = ua
         ? generateDeviceId({
             salt: salts.current,
@@ -190,7 +190,7 @@ export async function handler(
         return;
       }
 
-      const geo = await parseIp(ip);
+      const geo = await getGeoLocation(ip);
       await identify({
         payload: request.body.payload,
         projectId,

apps/api/src/hooks/ip.hook.ts

@@ -1,4 +1,4 @@
-import { getClientIp } from '@/utils/parse-ip';
+import { getClientIp } from '@/utils/get-client-ip';
 import type {
   FastifyReply,
   FastifyRequest,

apps/api/src/index.ts

@@ -239,5 +239,4 @@ const startServer = async () => {
   }
 };
 
-// start
 startServer();

apps/api/src/routes/misc.router.ts

@@ -25,6 +25,12 @@ const miscRouter: FastifyPluginCallback = async (fastify) => {
     url: '/favicon/clear',
     handler: controller.clearFavicons,
   });
+
+  fastify.route({
+    method: 'GET',
+    url: '/geo',
+    handler: controller.getGeo,
+  });
 };
 
 export default miscRouter;

apps/api/src/utils/get-client-ip.ts

@@ -0,0 +1,8 @@
+import type { FastifyRequest } from 'fastify';
+import requestIp from 'request-ip';
+
+const ignore = ['127.0.0.1', '::1'];
+
+export function getClientIp(req: FastifyRequest) {
+  return requestIp.getClientIp(req);
+}

apps/api/src/utils/parse-ip.ts

@@ -1,85 +0,0 @@
-import crypto from 'node:crypto';
-import { getRedisCache } from '@openpanel/redis';
-import type { FastifyRequest } from 'fastify';
-import requestIp from 'request-ip';
-import { logger } from './logger';
-
-interface RemoteIpLookupResponse {
-  country: string | undefined;
-  city: string | undefined;
-  stateprov: string | undefined;
-  longitude: number | undefined;
-  latitude: number | undefined;
-}
-
-export interface GeoLocation {
-  country: string | undefined;
-  city: string | undefined;
-  region: string | undefined;
-  longitude: number | undefined;
-  latitude: number | undefined;
-}
-
-const DEFAULT_GEO: GeoLocation = {
-  country: undefined,
-  city: undefined,
-  region: undefined,
-  longitude: undefined,
-  latitude: undefined,
-};
-
-const ignore = ['127.0.0.1', '::1'];
-
-export function getClientIp(req: FastifyRequest) {
-  return requestIp.getClientIp(req);
-}
-
-export async function parseIp(ip?: string): Promise<GeoLocation> {
-  if (!ip || ignore.includes(ip)) {
-    return DEFAULT_GEO;
-  }
-
-  const hash = crypto.createHash('sha256').update(ip).digest('hex');
-  const cached = await getRedisCache()
-    .get(`geo:${hash}`)
-    .catch(() => {
-      logger.warn('Failed to get geo location from cache', { hash });
-      return null;
-    });
-
-  if (cached) {
-    return JSON.parse(cached);
-  }
-
-  try {
-    const res = await fetch(`${process.env.GEO_IP_HOST}/${ip}`, {
-      signal: AbortSignal.timeout(4000),
-    });
-
-    if (!res.ok) {
-      return DEFAULT_GEO;
-    }
-
-    const json = (await res.json()) as RemoteIpLookupResponse;
-
-    const geo = {
-      country: json.country,
-      city: json.city,
-      region: json.stateprov,
-      longitude: json.longitude,
-      latitude: json.latitude,
-    };
-
-    await getRedisCache().set(
-      `geo:${hash}`,
-      JSON.stringify(geo),
-      'EX',
-      60 * 60 * 24,
-    );
-
-    return geo;
-  } catch (error) {
-    logger.error('Failed to fetch geo location for ip', { error });
-    return DEFAULT_GEO;
-  }
-}

apps/worker/Dockerfile

@@ -27,6 +27,7 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY apps/worker/package.json ./apps/worker/
 # Packages
 COPY packages/db/package.json ./packages/db/
+COPY packages/geo/package.json ./packages/geo/
 COPY packages/json/package.json ./packages/json/
 COPY packages/email/package.json ./packages/email/
 COPY packages/redis/package.json ./packages/redis/
@@ -48,7 +49,7 @@ COPY apps/worker ./apps/worker
 COPY packages ./packages
 COPY tooling ./tooling
 
-RUN pnpm db:codegen && \
+RUN pnpm codegen && \
     pnpm --filter worker run build
 
 # PROD
@@ -73,6 +74,7 @@ COPY --from=build /app/apps/worker ./apps/worker
 
 # Packages
 COPY --from=build /app/packages/db ./packages/db
+COPY --from=build /app/packages/geo ./packages/geo
 COPY --from=build /app/packages/json ./packages/json
 COPY --from=build /app/packages/email ./packages/email
 COPY --from=build /app/packages/redis ./packages/redis

docker-compose.yml

@@ -21,12 +21,6 @@ services:
     ports:
       - 6379:6379
 
-  op-geo:
-    image: observabilitystack/geoip-api:latest
-    restart: always
-    ports:
-      - 8080:8080
-
   op-ch:
     image: clickhouse/clickhouse-server:24.12.2.29-alpine
     restart: always

package.json

@@ -11,7 +11,7 @@
     "dock:ch": "docker compose exec -it op-ch clickhouse-client -d openpanel",
     "dock:redis": "docker compose exec -it op-kv redis-cli",
     "db:codegen": "pnpm -r --filter db run codegen",
-    "codegen": "pnpm db:codegen",
+    "codegen": "pnpm -r --filter db --filter geo run codegen",
     "migrate": "pnpm -r --filter db run migrate",
     "migrate:deploy": "pnpm -r --filter db run migrate:deploy",
     "dev": "pnpm -r --parallel testing",

packages/geo/index.ts

@@ -0,0 +1 @@
+export * from './src/geo';

packages/geo/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@openpanel/geo",
+  "version": "0.0.1",
+  "main": "index.ts",
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "codegen": "jiti scripts/download.ts"
+  },
+  "dependencies": {
+    "@maxmind/geoip2-node": "^6.1.0"
+  },
+  "devDependencies": {
+    "@openpanel/tsconfig": "workspace:*",
+    "@types/node": "20.14.8",
+    "fast-extract": "^1.4.3",
+    "tar": "^7.4.3",
+    "typescript": "^5.2.2",
+    "jiti": "^2.4.1"
+  }
+}

packages/geo/scripts/download.ts

@@ -0,0 +1,66 @@
+import fs from 'node:fs';
+import https from 'node:https';
+import path from 'node:path';
+import zlib from 'node:zlib';
+import * as tar from 'tar';
+import type { Parser } from 'tar';
+
+const db = 'GeoLite2-City';
+
+const download = async (url: string): Promise<Parser> => {
+  return new Promise((resolve) => {
+    https.get(url, (res) => {
+      const gunzip = zlib.createGunzip();
+      const parser = tar.t();
+
+      res.pipe(gunzip).pipe(parser as any);
+      resolve(parser);
+    });
+  });
+};
+
+async function main(): Promise<void> {
+  let url = `https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/${db}.tar.gz`;
+
+  if (process.env.MAXMIND_LICENSE_KEY) {
+    url = [
+      'https://download.maxmind.com/app/geoip_download',
+      `?edition_id=${db}&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`,
+    ].join('');
+  }
+
+  const dest = path.resolve(__dirname, '../');
+
+  if (!fs.existsSync(dest)) {
+    console.log('Geo database not found');
+    process.exit(1);
+  }
+
+  try {
+    const res = await download(url);
+
+    await new Promise<void>((resolve, reject) => {
+      res.on('entry', (entry) => {
+        if (entry.path.endsWith('.mmdb')) {
+          const filename = path.join(dest, path.basename(entry.path));
+          entry.pipe(fs.createWriteStream(filename));
+
+          console.log('Saved geo database:', filename);
+        }
+      });
+
+      res.on('error', (e) => {
+        reject(e);
+      });
+
+      res.on('finish', () => {
+        resolve();
+      });
+    });
+  } catch (error) {
+    console.error('Error downloading geo database:', error);
+    process.exit(1);
+  }
+}
+
+main();

packages/geo/src/geo.ts

@@ -0,0 +1,69 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import type { ReaderModel } from '@maxmind/geoip2-node';
+import { Reader } from '@maxmind/geoip2-node';
+
+const filename = 'GeoLite2-City.mmdb';
+// From api or worker package
+const dbPath = path.join(__dirname, `../../../packages/geo/${filename}`);
+// From local package
+const dbPathLocal = path.join(__dirname, `../${filename}`);
+
+let reader: ReaderModel | null = null;
+
+async function loadDatabase(dbPath: string) {
+  try {
+    const dbBuffer = await readFile(dbPath);
+    reader = Reader.openBuffer(dbBuffer);
+    console.log('GeoLite2-City.mmdb loaded (dist)');
+  } catch (error) {
+    try {
+      const dbBuffer = await readFile(dbPathLocal);
+      reader = Reader.openBuffer(dbBuffer);
+      console.log('GeoLite2-City.mmdb loaded (local)');
+    } catch (error) {
+      console.error('GeoLite2-City.mmdb not found');
+    }
+  }
+}
+
+export interface GeoLocation {
+  country: string | undefined;
+  city: string | undefined;
+  region: string | undefined;
+  longitude: number | undefined;
+  latitude: number | undefined;
+}
+
+const DEFAULT_GEO: GeoLocation = {
+  country: undefined,
+  city: undefined,
+  region: undefined,
+  longitude: undefined,
+  latitude: undefined,
+};
+
+const ignore = ['127.0.0.1', '::1'];
+
+export async function getGeoLocation(ip?: string): Promise<GeoLocation> {
+  if (!ip || ignore.includes(ip)) {
+    return DEFAULT_GEO;
+  }
+
+  if (!reader) {
+    await loadDatabase(dbPath);
+  }
+
+  try {
+    const response = await reader?.city(ip);
+    return {
+      city: response?.city?.names.en,
+      country: response?.country?.isoCode,
+      region: response?.subdivisions?.[0]?.names.en,
+      longitude: response?.location?.longitude,
+      latitude: response?.location?.latitude,
+    };
+  } catch (error) {
+    return DEFAULT_GEO;
+  }
+}

packages/geo/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "@openpanel/tsconfig/base.json",
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./src/*"]
+    },
+    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+  },
+  "include": [".", "*.mmdb"],
+  "exclude": ["node_modules"]
+}

self-hosting/docker-compose.template.yml

@@ -42,10 +42,6 @@ services:
     # ports:
     #   - 6379:6379
 
-  op-geo:
-    image: observabilitystack/geoip-api:latest
-    restart: always
-
   op-ch:
     image: clickhouse/clickhouse-server:24.3.2-alpine
     restart: always
@@ -91,7 +87,6 @@ services:
       - op-db
       - op-ch
       - op-kv
-      - op-geo
     env_file:
       - .env