diff --git a/apps/api/src/controllers/profile.controller.ts b/apps/api/src/controllers/profile.controller.ts index d1f5628d..1be194d5 100644 --- a/apps/api/src/controllers/profile.controller.ts +++ b/apps/api/src/controllers/profile.controller.ts @@ -1,8 +1,8 @@ import { getClientIp, parseIp } from '@/utils/parseIp'; -import { parseUserAgent } from '@/utils/parseUserAgent'; import type { FastifyReply, FastifyRequest } from 'fastify'; import { assocPath, pathOr } from 'ramda'; +import { parseUserAgent } from '@openpanel/common/server'; import { getProfileById, upsertProfile } from '@openpanel/db'; import type { IncrementProfilePayload, diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts index dca57a45..b98e5711 100644 --- a/apps/api/src/controllers/track.controller.ts +++ b/apps/api/src/controllers/track.controller.ts @@ -1,10 +1,9 @@ import type { GeoLocation } from '@/utils/parseIp'; import { getClientIp, parseIp } from '@/utils/parseIp'; -import { parseUserAgent } from '@/utils/parseUserAgent'; import type { FastifyReply, FastifyRequest } from 'fastify'; import { path, assocPath, pathOr, pick } from 'ramda'; -import { generateDeviceId } from '@openpanel/common/server'; +import { generateDeviceId, parseUserAgent } from '@openpanel/common/server'; import { createProfileAlias, getProfileById, diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.ts index b5320ee4..23c456e1 100644 --- a/apps/worker/src/jobs/events.incoming-event.ts +++ b/apps/worker/src/jobs/events.incoming-event.ts @@ -1,11 +1,11 @@ import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer'; -import { parseUserAgent } from '@/utils/parse-user-agent'; import type { Job } from 'bullmq'; import { omit } from 'ramda'; import { v4 as uuid } from 'uuid'; import { logger } from '@/utils/logger'; import { getTime, isSameDomain, parsePath } from '@openpanel/common'; +import { parseUserAgent } from '@openpanel/common/server'; import type { IServiceCreateEventPayload } from '@openpanel/db'; import { checkNotificationRulesForEvent, createEvent } from '@openpanel/db'; import { getLastScreenViewFromProfileId } from '@openpanel/db/src/services/event.service'; diff --git a/apps/worker/src/utils/parse-user-agent.ts b/apps/worker/src/utils/parse-user-agent.ts deleted file mode 100644 index 42ddc8bb..00000000 --- a/apps/worker/src/utils/parse-user-agent.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { UAParser } from 'ua-parser-js'; - -const parsedServerUa = { - isServer: true, - device: 'server', -} as const; - -export function parseUserAgent(ua?: string | null) { - if (!ua) return parsedServerUa; - const res = new UAParser(ua).getResult(); - - if (isServer(ua)) { - return parsedServerUa; - } - - return { - os: res.os.name, - osVersion: res.os.version, - browser: res.browser.name, - browserVersion: res.browser.version, - device: res.device.type ?? getDevice(ua), - brand: res.device.vendor, - model: res.device.model, - isServer: false, - } as const; -} - -const userAgentServerList = [ - // Node.js libraries - 'node', - 'node-fetch', - 'axios', - 'request', - 'superagent', - 'undici', - - // Python libraries - 'python-requests', - 'python-urllib', - 'aiohttp', - 'python', - - // Ruby libraries - 'Faraday', - 'Ruby', - 'http.rb', - - // Go libraries - 'Go-http-client', - 'Go-http-client', - - // Java libraries - 'Apache-HttpClient', - 'okhttp', - 'okhowtp', - - // PHP libraries - 'GuzzleHttp', - 'PHP-cURL', - - // Other - 'Dart', - 'RestSharp', // Popular .NET HTTP client library - 'HttpClientFactory', // .NET's typed client factory - 'Ktor', // A client for Kotlin - 'Ning', // Async HTTP client for Java - 'grpc-csharp', // gRPC for C# - 'Volley', // HTTP library used in Android apps for making network requests - 'Spring', - 'vert.x', - 'grpc-', -]; - -function isServer(userAgent: string) { - const match = userAgentServerList.some((server) => - userAgent.toLowerCase().includes(server.toLowerCase()), - ); - if (match) { - return true; - } - - // Matches user agents like "Go-http-client/1.0" or "Go Http Client/1.0" - // It should just match the first name (with optional spaces) and version - return !!userAgent.match(/^[^\/]+\/[\d.]+$/); -} - -export function getDevice(ua: string) { - const mobile1 = - /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test( - ua, - ); - const mobile2 = - /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test( - ua.slice(0, 4), - ); - const tablet = - /tablet|ipad|android(?!.*mobile)|xoom|sch-i800|kindle|silk|playbook/i.test( - ua, - ); - - if (mobile1 || mobile2) { - return 'mobile'; - } - - if (tablet) { - return 'tablet'; - } - - return 'desktop'; -} diff --git a/packages/common/package.json b/packages/common/package.json index 9b6d4e70..c3e0c276 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -13,6 +13,7 @@ "ramda": "^0.29.1", "slugify": "^1.6.6", "superjson": "^1.13.3", + "ua-parser-js": "^1.0.37", "unique-names-generator": "^4.7.1" }, "devDependencies": { @@ -20,6 +21,7 @@ "@openpanel/validation": "workspace:*", "@types/node": "^18.16.0", "@types/ramda": "^0.29.6", + "@types/ua-parser-js": "^0.7.39", "prisma": "^5.1.1", "typescript": "^5.2.2" } diff --git a/packages/common/server/index.ts b/packages/common/server/index.ts index e5fd0d20..3bb5bb37 100644 --- a/packages/common/server/index.ts +++ b/packages/common/server/index.ts @@ -1,2 +1,3 @@ export * from './crypto'; export * from './profileId'; +export * from './parser-user-agent'; diff --git a/apps/api/src/utils/parseUserAgent.ts b/packages/common/server/parser-user-agent.ts similarity index 87% rename from apps/api/src/utils/parseUserAgent.ts rename to packages/common/server/parser-user-agent.ts index 42ddc8bb..954d9e98 100644 --- a/apps/api/src/utils/parseUserAgent.ts +++ b/packages/common/server/parser-user-agent.ts @@ -9,7 +9,7 @@ export function parseUserAgent(ua?: string | null) { if (!ua) return parsedServerUa; const res = new UAParser(ua).getResult(); - if (isServer(ua)) { + if (isServer(ua, res)) { return parsedServerUa; } @@ -71,17 +71,28 @@ const userAgentServerList = [ 'grpc-', ]; -function isServer(userAgent: string) { - const match = userAgentServerList.some((server) => +function isServer(userAgent: string, res: UAParser.IResult) { + const isInServerList = userAgentServerList.some((server) => userAgent.toLowerCase().includes(server.toLowerCase()), ); - if (match) { + if (isInServerList) { return true; } // Matches user agents like "Go-http-client/1.0" or "Go Http Client/1.0" // It should just match the first name (with optional spaces) and version - return !!userAgent.match(/^[^\/]+\/[\d.]+$/); + const isSingleNameWithVersion = !!userAgent.match(/^[^\/]+\/[\d.]+$/); + if (isSingleNameWithVersion) { + return true; + } + + // If all of these are undefined, we can consider it a server + return ( + res.os.name === undefined && + res.browser.name === undefined && + res.device.vendor === undefined && + res.device.model === undefined + ); } export function getDevice(ua: string) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 10c30f65..c76df094 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -102,9 +102,6 @@ importers: svix: specifier: ^1.24.0 version: 1.24.0 - ua-parser-js: - specifier: ^1.0.37 - version: 1.0.37 url-metadata: specifier: ^4.1.0 version: 4.1.0 @@ -136,9 +133,6 @@ importers: '@types/sqlstring': specifier: ^2.3.2 version: 2.3.2 - '@types/ua-parser-js': - specifier: ^0.7.39 - version: 0.7.39 '@types/uuid': specifier: ^9.0.8 version: 9.0.8 @@ -709,9 +703,6 @@ importers: sqlstring: specifier: ^2.3.3 version: 2.3.3 - ua-parser-js: - specifier: ^1.0.37 - version: 1.0.37 uuid: specifier: ^9.0.1 version: 9.0.1 @@ -728,9 +719,6 @@ importers: '@types/sqlstring': specifier: ^2.3.2 version: 2.3.2 - '@types/ua-parser-js': - specifier: ^0.7.39 - version: 0.7.39 '@types/uuid': specifier: ^9.0.8 version: 9.0.8 @@ -816,6 +804,9 @@ importers: superjson: specifier: ^1.13.3 version: 1.13.3 + ua-parser-js: + specifier: ^1.0.37 + version: 1.0.37 unique-names-generator: specifier: ^4.7.1 version: 4.7.1 @@ -832,6 +823,9 @@ importers: '@types/ramda': specifier: ^0.29.6 version: 0.29.10 + '@types/ua-parser-js': + specifier: ^0.7.39 + version: 0.7.39 prisma: specifier: ^5.1.1 version: 5.9.1 @@ -11700,6 +11694,7 @@ packages: /eslint@8.56.0: resolution: {integrity: sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true dependencies: '@eslint-community/eslint-utils': 4.4.0(eslint@8.56.0)