diff --git a/packages/common/src/object.ts b/packages/common/src/object.ts index 405156cd..374342b8 100644 --- a/packages/common/src/object.ts +++ b/packages/common/src/object.ts @@ -5,6 +5,13 @@ export function toDots( obj: Record, path = '', ): Record { + // Clickhouse breaks on insert if a property contains invalid surrogate pairs + function removeInvalidSurrogates(value: string): string { + const validSurrogatePairRegex = + /[\uD800-\uDBFF][\uDC00-\uDFFF]|[^\uD800-\uDFFF]/g; + return value.match(validSurrogatePairRegex)?.join('') || ''; + } + return Object.entries(obj).reduce((acc, [key, value]) => { if (typeof value === 'object' && value !== null) { return { @@ -17,10 +24,14 @@ export function toDots( return acc; } + const cleanedValue = + typeof value === 'string' + ? removeInvalidSurrogates(value).trim() + : String(value); + return { ...acc, - [`${path}${key}`]: - typeof value === 'string' ? value.trim() : String(value), + [`${path}${key}`]: cleanedValue, }; }, {}); }