From 1883ec2170f827bb23a4af8acf1bc4a87ac33828 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carl-Gerhard=20Lindesva=CC=88rd?= <lindesvard@gmail.com>
Date: Thu, 19 Dec 2024 13:44:38 +0100
Subject: [PATCH] fix(clickhouse): remove invalid surrogate pairs from
 properties

---
 packages/common/src/object.ts | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/packages/common/src/object.ts b/packages/common/src/object.ts
index 405156cd..374342b8 100644
--- a/packages/common/src/object.ts
+++ b/packages/common/src/object.ts
@@ -5,6 +5,13 @@ export function toDots(
   obj: Record<string, unknown>,
   path = '',
 ): Record<string, string> {
+  // Clickhouse breaks on insert if a property contains invalid surrogate pairs
+  function removeInvalidSurrogates(value: string): string {
+    const validSurrogatePairRegex =
+      /[\uD800-\uDBFF][\uDC00-\uDFFF]|[^\uD800-\uDFFF]/g;
+    return value.match(validSurrogatePairRegex)?.join('') || '';
+  }
+
   return Object.entries(obj).reduce((acc, [key, value]) => {
     if (typeof value === 'object' && value !== null) {
       return {
@@ -17,10 +24,14 @@ export function toDots(
       return acc;
     }
 
+    const cleanedValue =
+      typeof value === 'string'
+        ? removeInvalidSurrogates(value).trim()
+        : String(value);
+
     return {
       ...acc,
-      [`${path}${key}`]:
-        typeof value === 'string' ? value.trim() : String(value),
+      [`${path}${key}`]: cleanedValue,
     };
   }, {});
 }