zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feature(dashboard,api): add timezone support

Browse Source 
* feat(dashboard): add support for today, yesterday etc (timezones)

* fix(db): escape js dates

* fix(dashboard): ensure we support default timezone

* final fixes

* remove complete series and add sql with fill instead
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-05-23 11:26:44 +02:00
committed by GitHub
parent 46bfeee131
commit 680727355b
48 changed files with 1817 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/common/index.ts
View File

@@ -1,10 +1,11 @@
export * from './src/date';
export * from './src/timezones';
export * from './src/object';
export * from './src/names';
export * from './src/string';
export * from './src/math';
export * from './src/slug';
export * from './src/fill-series';
export * from './src/url';
export * from './src/id';
export * from './src/get-previous-metric';
export * from './src/group-by-labels';

2
packages/common/package.json
View File

@@ -8,6 +8,7 @@
"dependencies": {
"@openpanel/constants": "workspace:*",
"date-fns": "^3.3.1",
"luxon": "^3.6.1",
"mathjs": "^12.3.2",
"nanoid": "^5.0.7",
"ramda": "^0.29.1",
@@ -19,6 +20,7 @@
"devDependencies": {
"@openpanel/tsconfig": "workspace:*",
"@openpanel/validation": "workspace:*",
"@types/luxon": "^3.6.2",
"@types/node": "20.14.8",
"@types/ramda": "^0.29.6",
"@types/ua-parser-js": "^0.7.39",

59
packages/common/src/date.ts
View File

@@ -1,58 +1,7 @@
import { DateTime } from 'luxon';
export { DateTime };
export function getTime(date: string | number | Date) {
return new Date(date).getTime();
}
export function toISOString(date: string | number | Date) {
return new Date(date).toISOString();
}
export function getTimezoneFromDateString(_date: string) {
const mapper: Record<string, string> = {
'+00:00': 'UTC',
'+01:00': 'Europe/Paris',
'+02:00': 'Europe/Stockholm',
'+03:00': 'Europe/Moscow',
'+04:00': 'Asia/Dubai',
'+05:00': 'Asia/Karachi',
'+06:00': 'Asia/Dhaka',
'+07:00': 'Asia/Bangkok',
'+08:00': 'Asia/Shanghai',
'+09:00': 'Asia/Tokyo',
'+10:00': 'Australia/Sydney',
'+11:00': 'Pacific/Noumea',
'+12:00': 'Pacific/Fiji',
'-02:00': 'America/Noronha',
'-03:00': 'America/Sao_Paulo',
'-04:00': 'America/Santiago',
'-05:00': 'America/Bogota',
'-06:00': 'America/Mexico_City',
'-07:00': 'America/Phoenix',
'-08:00': 'America/Los_Angeles',
'-09:00': 'America/Anchorage',
'-10:00': 'Pacific/Honolulu',
'-11:00': 'Pacific/Midway',
'-12:00': 'Pacific/Tarawa',
// Additional time zones
'+05:30': 'Asia/Kolkata',
'+05:45': 'Asia/Kathmandu',
'+08:45': 'Australia/Eucla',
'+09:30': 'Australia/Darwin',
'+10:30': 'Australia/Adelaide',
'+12:45': 'Pacific/Chatham',
'+13:00': 'Pacific/Apia',
'+14:00': 'Pacific/Kiritimati',
'-02:30': 'America/St_Johns',
'-03:30': 'America/St_Johns',
'-04:30': 'America/Caracas',
'-09:30': 'Pacific/Marquesas',
};
const defaultTimezone = 'UTC';
const match = _date.match(/([+-][0-9]{2}):([0-9]{2})$/)?.[0];
if (match) {
return mapper[match] ?? defaultTimezone;
}
return defaultTimezone;
}

143
packages/common/src/fill-series.ts
View File

@@ -1,143 +0,0 @@
import {
addDays,
addHours,
addMinutes,
addMonths,
addWeeks,
format,
parseISO,
startOfDay,
startOfHour,
startOfMinute,
startOfMonth,
startOfWeek,
} from 'date-fns';
import { NOT_SET_VALUE } from '@openpanel/constants';
import type { IInterval } from '@openpanel/validation';
// Define the data structure
export interface ISerieDataItem {
label_0: string | null | undefined;
label_1?: string | null | undefined;
label_2?: string | null | undefined;
label_3?: string | null | undefined;
count: number;
date: string;
}
export interface ISerieDataItemComplete {
labels: string[];
count: number;
date: string;
}
// Function to round down the date to the nearest interval
function roundDate(date: Date, interval: IInterval): Date {
switch (interval) {
case 'minute':
return startOfMinute(date);
case 'hour':
return startOfHour(date);
case 'day':
return startOfDay(date);
case 'week':
return startOfWeek(date);
case 'month':
return startOfMonth(date);
default:
return startOfMinute(date);
}
}
function filterFalsyAfterTruthy(array: (string | undefined | null)[]) {
let foundTruthy = false;
const filtered = array.filter((item) => {
if (foundTruthy) {
// After a truthy, filter out falsy values
return !!item;
}
if (item) {
// Mark when the first truthy is encountered
foundTruthy = true;
}
// Return all elements until the first truthy is found
return true;
});
if (filtered.some((item) => !!item)) {
return filtered;
}
return [null];
}
function concatLabels(entry: ISerieDataItem): string {
return filterFalsyAfterTruthy([
entry.label_0,
entry.label_1,
entry.label_2,
entry.label_3,
])
.map((label) => label || NOT_SET_VALUE)
.join(':::');
}
// Function to complete the timeline for each label
export function completeSerie(
data: ISerieDataItem[],
_startDate: string,
_endDate: string,
interval: IInterval,
) {
const startDate = parseISO(_startDate);
const endDate = parseISO(_endDate);
// Group data by label
const labelsMap = new Map<string, Map<string, number>>();
data.forEach((entry) => {
const roundedDate = roundDate(parseISO(entry.date), interval);
const dateKey = format(roundedDate, 'yyyy-MM-dd HH:mm:ss');
const label = concatLabels(entry) || NOT_SET_VALUE;
if (!labelsMap.has(label)) {
labelsMap.set(label, new Map());
}
const labelData = labelsMap.get(label)!;
labelData.set(dateKey, (labelData.get(dateKey) || 0) + (entry.count || 0));
});
// Complete the timeline for each label
const result: Record<string, ISerieDataItemComplete[]> = {};
labelsMap.forEach((counts, label) => {
let currentDate = roundDate(startDate, interval);
result[label] = [];
while (currentDate <= endDate) {
const dateKey = format(currentDate, 'yyyy-MM-dd HH:mm:ss');
result[label]!.push({
labels: label.split(':::'),
date: dateKey,
count: counts.get(dateKey) || 0,
});
// Increment the current date based on the interval
switch (interval) {
case 'minute':
currentDate = addMinutes(currentDate, 1);
break;
case 'hour':
currentDate = addHours(currentDate, 1);
break;
case 'day':
currentDate = addDays(currentDate, 1);
break;
case 'week':
currentDate = addWeeks(currentDate, 1);
break;
case 'month':
currentDate = addMonths(currentDate, 1);
break;
}
}
});
return result;
}

70
packages/common/src/group-by-labels.ts Normal file
View File

@@ -0,0 +1,70 @@
export interface ISerieDataItem {
label_0: string | null | undefined;
label_1?: string | null | undefined;
label_2?: string | null | undefined;
label_3?: string | null | undefined;
count: number;
date: string;
}
interface GroupedDataPoint {
date: string;
count: number;
}
interface GroupedResult {
name: string[]; // [label_0, label_1, label_2, label_3]
data: GroupedDataPoint[];
}
export function groupByLabels(data: ISerieDataItem[]): GroupedResult[] {
const groupedMap = new Map<string, GroupedResult>();
const timestamps = new Set<string>();
data.forEach((row) => {
timestamps.add(row.date);
const labels = Object.keys(row)
.filter((key) => key.startsWith('label_'))
.sort((a, b) => {
const numA = Number.parseInt(a.replace('label_', ''));
const numB = Number.parseInt(b.replace('label_', ''));
return numA - numB;
})
.map((key) => (row as any)[key])
.filter((label): label is string => !!label);
const labelKey = labels.join(':::');
if (!groupedMap.has(labelKey)) {
groupedMap.set(labelKey, {
name: labels,
data: [],
});
}
const group = groupedMap.get(labelKey)!;
group.data.push({
date: row.date,
count: row.count,
});
});
const result = Array.from(groupedMap.values()).map((group) => ({
...group,
data: group.data.sort(
(a, b) => new Date(a.date).getTime() - new Date(b.date).getTime(),
),
}));
return result
.filter((group) => group.name.length > 0)
.map((group) => {
return {
...group,
// This will ensure that all dates are present in the data array
data: Array.from(timestamps).map((date) => {
const dataPoint = group.data.find((dp) => dp.date === date);
return dataPoint || { date, count: 0 };
}),
};
});
}

600
packages/common/src/timezones.ts Normal file
View File

@@ -0,0 +1,600 @@
// List of available timezones from Clickhouse `select time_zone from system.time_zones`
export const TIMEZONES = [
'Africa/Abidjan',
'Africa/Accra',
'Africa/Addis_Ababa',
'Africa/Algiers',
'Africa/Asmara',
'Africa/Asmera',
'Africa/Bamako',
'Africa/Bangui',
'Africa/Banjul',
'Africa/Bissau',
'Africa/Blantyre',
'Africa/Brazzaville',
'Africa/Bujumbura',
'Africa/Cairo',
'Africa/Casablanca',
'Africa/Ceuta',
'Africa/Conakry',
'Africa/Dakar',
'Africa/Dar_es_Salaam',
'Africa/Djibouti',
'Africa/Douala',
'Africa/El_Aaiun',
'Africa/Freetown',
'Africa/Gaborone',
'Africa/Harare',
'Africa/Johannesburg',
'Africa/Juba',
'Africa/Kampala',
'Africa/Khartoum',
'Africa/Kigali',
'Africa/Kinshasa',
'Africa/Lagos',
'Africa/Libreville',
'Africa/Lome',
'Africa/Luanda',
'Africa/Lubumbashi',
'Africa/Lusaka',
'Africa/Malabo',
'Africa/Maputo',
'Africa/Maseru',
'Africa/Mbabane',
'Africa/Mogadishu',
'Africa/Monrovia',
'Africa/Nairobi',
'Africa/Ndjamena',
'Africa/Niamey',
'Africa/Nouakchott',
'Africa/Ouagadougou',
'Africa/Porto-Novo',
'Africa/Sao_Tome',
'Africa/Timbuktu',
'Africa/Tripoli',
'Africa/Tunis',
'Africa/Windhoek',
'America/Adak',
'America/Anchorage',
'America/Anguilla',
'America/Antigua',
'America/Araguaina',
'America/Argentina/Buenos_Aires',
'America/Argentina/Catamarca',
'America/Argentina/ComodRivadavia',
'America/Argentina/Cordoba',
'America/Argentina/Jujuy',
'America/Argentina/La_Rioja',
'America/Argentina/Mendoza',
'America/Argentina/Rio_Gallegos',
'America/Argentina/Salta',
'America/Argentina/San_Juan',
'America/Argentina/San_Luis',
'America/Argentina/Tucuman',
'America/Argentina/Ushuaia',
'America/Aruba',
'America/Asuncion',
'America/Atikokan',
'America/Atka',
'America/Bahia',
'America/Bahia_Banderas',
'America/Barbados',
'America/Belem',
'America/Belize',
'America/Blanc-Sablon',
'America/Boa_Vista',
'America/Bogota',
'America/Boise',
'America/Buenos_Aires',
'America/Cambridge_Bay',
'America/Campo_Grande',
'America/Cancun',
'America/Caracas',
'America/Catamarca',
'America/Cayenne',
'America/Cayman',
'America/Chicago',
'America/Chihuahua',
'America/Ciudad_Juarez',
'America/Coral_Harbour',
'America/Cordoba',
'America/Costa_Rica',
'America/Creston',
'America/Cuiaba',
'America/Curacao',
'America/Danmarkshavn',
'America/Dawson',
'America/Dawson_Creek',
'America/Denver',
'America/Detroit',
'America/Dominica',
'America/Edmonton',
'America/Eirunepe',
'America/El_Salvador',
'America/Ensenada',
'America/Fort_Nelson',
'America/Fort_Wayne',
'America/Fortaleza',
'America/Glace_Bay',
'America/Godthab',
'America/Goose_Bay',
'America/Grand_Turk',
'America/Grenada',
'America/Guadeloupe',
'America/Guatemala',
'America/Guayaquil',
'America/Guyana',
'America/Halifax',
'America/Havana',
'America/Hermosillo',
'America/Indiana/Indianapolis',
'America/Indiana/Knox',
'America/Indiana/Marengo',
'America/Indiana/Petersburg',
'America/Indiana/Tell_City',
'America/Indiana/Vevay',
'America/Indiana/Vincennes',
'America/Indiana/Winamac',
'America/Indianapolis',
'America/Inuvik',
'America/Iqaluit',
'America/Jamaica',
'America/Jujuy',
'America/Juneau',
'America/Kentucky/Louisville',
'America/Kentucky/Monticello',
'America/Knox_IN',
'America/Kralendijk',
'America/La_Paz',
'America/Lima',
'America/Los_Angeles',
'America/Louisville',
'America/Lower_Princes',
'America/Maceio',
'America/Managua',
'America/Manaus',
'America/Marigot',
'America/Martinique',
'America/Matamoros',
'America/Mazatlan',
'America/Mendoza',
'America/Menominee',
'America/Merida',
'America/Metlakatla',
'America/Mexico_City',
'America/Miquelon',
'America/Moncton',
'America/Monterrey',
'America/Montevideo',
'America/Montreal',
'America/Montserrat',
'America/Nassau',
'America/New_York',
'America/Nipigon',
'America/Nome',
'America/Noronha',
'America/North_Dakota/Beulah',
'America/North_Dakota/Center',
'America/North_Dakota/New_Salem',
'America/Nuuk',
'America/Ojinaga',
'America/Panama',
'America/Pangnirtung',
'America/Paramaribo',
'America/Phoenix',
'America/Port-au-Prince',
'America/Port_of_Spain',
'America/Porto_Acre',
'America/Porto_Velho',
'America/Puerto_Rico',
'America/Punta_Arenas',
'America/Rainy_River',
'America/Rankin_Inlet',
'America/Recife',
'America/Regina',
'America/Resolute',
'America/Rio_Branco',
'America/Rosario',
'America/Santa_Isabel',
'America/Santarem',
'America/Santiago',
'America/Santo_Domingo',
'America/Sao_Paulo',
'America/Scoresbysund',
'America/Shiprock',
'America/Sitka',
'America/St_Barthelemy',
'America/St_Johns',
'America/St_Kitts',
'America/St_Lucia',
'America/St_Thomas',
'America/St_Vincent',
'America/Swift_Current',
'America/Tegucigalpa',
'America/Thule',
'America/Thunder_Bay',
'America/Tijuana',
'America/Toronto',
'America/Tortola',
'America/Vancouver',
'America/Virgin',
'America/Whitehorse',
'America/Winnipeg',
'America/Yakutat',
'America/Yellowknife',
'Antarctica/Casey',
'Antarctica/Davis',
'Antarctica/DumontDUrville',
'Antarctica/Macquarie',
'Antarctica/Mawson',
'Antarctica/McMurdo',
'Antarctica/Palmer',
'Antarctica/Rothera',
'Antarctica/South_Pole',
'Antarctica/Syowa',
'Antarctica/Troll',
'Antarctica/Vostok',
'Arctic/Longyearbyen',
'Asia/Aden',
'Asia/Almaty',
'Asia/Amman',
'Asia/Anadyr',
'Asia/Aqtau',
'Asia/Aqtobe',
'Asia/Ashgabat',
'Asia/Ashkhabad',
'Asia/Atyrau',
'Asia/Baghdad',
'Asia/Bahrain',
'Asia/Baku',
'Asia/Bangkok',
'Asia/Barnaul',
'Asia/Beirut',
'Asia/Bishkek',
'Asia/Brunei',
'Asia/Calcutta',
'Asia/Chita',
'Asia/Choibalsan',
'Asia/Chongqing',
'Asia/Chungking',
'Asia/Colombo',
'Asia/Dacca',
'Asia/Damascus',
'Asia/Dhaka',
'Asia/Dili',
'Asia/Dubai',
'Asia/Dushanbe',
'Asia/Famagusta',
'Asia/Gaza',
'Asia/Harbin',
'Asia/Hebron',
'Asia/Ho_Chi_Minh',
'Asia/Hong_Kong',
'Asia/Hovd',
'Asia/Irkutsk',
'Asia/Istanbul',
'Asia/Jakarta',
'Asia/Jayapura',
'Asia/Jerusalem',
'Asia/Kabul',
'Asia/Kamchatka',
'Asia/Karachi',
'Asia/Kashgar',
'Asia/Kathmandu',
'Asia/Katmandu',
'Asia/Khandyga',
'Asia/Kolkata',
'Asia/Krasnoyarsk',
'Asia/Kuala_Lumpur',
'Asia/Kuching',
'Asia/Kuwait',
'Asia/Macao',
'Asia/Macau',
'Asia/Magadan',
'Asia/Makassar',
'Asia/Manila',
'Asia/Muscat',
'Asia/Nicosia',
'Asia/Novokuznetsk',
'Asia/Novosibirsk',
'Asia/Omsk',
'Asia/Oral',
'Asia/Phnom_Penh',
'Asia/Pontianak',
'Asia/Pyongyang',
'Asia/Qatar',
'Asia/Qostanay',
'Asia/Qyzylorda',
'Asia/Rangoon',
'Asia/Riyadh',
'Asia/Saigon',
'Asia/Sakhalin',
'Asia/Samarkand',
'Asia/Seoul',
'Asia/Shanghai',
'Asia/Singapore',
'Asia/Srednekolymsk',
'Asia/Taipei',
'Asia/Tashkent',
'Asia/Tbilisi',
'Asia/Tehran',
'Asia/Tel_Aviv',
'Asia/Thimbu',
'Asia/Thimphu',
'Asia/Tokyo',
'Asia/Tomsk',
'Asia/Ujung_Pandang',
'Asia/Ulaanbaatar',
'Asia/Ulan_Bator',
'Asia/Urumqi',
'Asia/Ust-Nera',
'Asia/Vientiane',
'Asia/Vladivostok',
'Asia/Yakutsk',
'Asia/Yangon',
'Asia/Yekaterinburg',
'Asia/Yerevan',
'Atlantic/Azores',
'Atlantic/Bermuda',
'Atlantic/Canary',
'Atlantic/Cape_Verde',
'Atlantic/Faeroe',
'Atlantic/Faroe',
'Atlantic/Jan_Mayen',
'Atlantic/Madeira',
'Atlantic/Reykjavik',
'Atlantic/South_Georgia',
'Atlantic/St_Helena',
'Atlantic/Stanley',
'Australia/ACT',
'Australia/Adelaide',
'Australia/Brisbane',
'Australia/Broken_Hill',
'Australia/Canberra',
'Australia/Currie',
'Australia/Darwin',
'Australia/Eucla',
'Australia/Hobart',
'Australia/LHI',
'Australia/Lindeman',
'Australia/Lord_Howe',
'Australia/Melbourne',
'Australia/NSW',
'Australia/North',
'Australia/Perth',
'Australia/Queensland',
'Australia/South',
'Australia/Sydney',
'Australia/Tasmania',
'Australia/Victoria',
'Australia/West',
'Australia/Yancowinna',
'Brazil/Acre',
'Brazil/DeNoronha',
'Brazil/East',
'Brazil/West',
'CET',
'CST6CDT',
'Canada/Atlantic',
'Canada/Central',
'Canada/Eastern',
'Canada/Mountain',
'Canada/Newfoundland',
'Canada/Pacific',
'Canada/Saskatchewan',
'Canada/Yukon',
'Chile/Continental',
'Chile/EasterIsland',
'Cuba',
'EET',
'EST',
'EST5EDT',
'Egypt',
'Eire',
'Etc/GMT',
'Etc/GMT+0',
'Etc/GMT+1',
'Etc/GMT+10',
'Etc/GMT+11',
'Etc/GMT+12',
'Etc/GMT+2',
'Etc/GMT+3',
'Etc/GMT+4',
'Etc/GMT+5',
'Etc/GMT+6',
'Etc/GMT+7',
'Etc/GMT+8',
'Etc/GMT+9',
'Etc/GMT-0',
'Etc/GMT-1',
'Etc/GMT-10',
'Etc/GMT-11',
'Etc/GMT-12',
'Etc/GMT-13',
'Etc/GMT-14',
'Etc/GMT-2',
'Etc/GMT-3',
'Etc/GMT-4',
'Etc/GMT-5',
'Etc/GMT-6',
'Etc/GMT-7',
'Etc/GMT-8',
'Etc/GMT-9',
'Etc/GMT0',
'Etc/Greenwich',
'Etc/UCT',
'Etc/UTC',
'Etc/Universal',
'Etc/Zulu',
'Europe/Amsterdam',
'Europe/Andorra',
'Europe/Astrakhan',
'Europe/Athens',
'Europe/Belfast',
'Europe/Belgrade',
'Europe/Berlin',
'Europe/Bratislava',
'Europe/Brussels',
'Europe/Bucharest',
'Europe/Budapest',
'Europe/Busingen',
'Europe/Chisinau',
'Europe/Copenhagen',
'Europe/Dublin',
'Europe/Gibraltar',
'Europe/Guernsey',
'Europe/Helsinki',
'Europe/Isle_of_Man',
'Europe/Istanbul',
'Europe/Jersey',
'Europe/Kaliningrad',
'Europe/Kiev',
'Europe/Kirov',
'Europe/Kyiv',
'Europe/Lisbon',
'Europe/Ljubljana',
'Europe/London',
'Europe/Luxembourg',
'Europe/Madrid',
'Europe/Malta',
'Europe/Mariehamn',
'Europe/Minsk',
'Europe/Monaco',
'Europe/Moscow',
'Europe/Nicosia',
'Europe/Oslo',
'Europe/Paris',
'Europe/Podgorica',
'Europe/Prague',
'Europe/Riga',
'Europe/Rome',
'Europe/Samara',
'Europe/San_Marino',
'Europe/Sarajevo',
'Europe/Saratov',
'Europe/Simferopol',
'Europe/Skopje',
'Europe/Sofia',
'Europe/Stockholm',
'Europe/Tallinn',
'Europe/Tirane',
'Europe/Tiraspol',
'Europe/Ulyanovsk',
'Europe/Uzhgorod',
'Europe/Vaduz',
'Europe/Vatican',
'Europe/Vienna',
'Europe/Vilnius',
'Europe/Volgograd',
'Europe/Warsaw',
'Europe/Zagreb',
'Europe/Zaporozhye',
'Europe/Zurich',
'Factory',
'GB',
'GB-Eire',
'GMT',
'GMT+0',
'GMT-0',
'GMT0',
'Greenwich',
'HST',
'Hongkong',
'Iceland',
'Indian/Antananarivo',
'Indian/Chagos',
'Indian/Christmas',
'Indian/Cocos',
'Indian/Comoro',
'Indian/Kerguelen',
'Indian/Mahe',
'Indian/Maldives',
'Indian/Mauritius',
'Indian/Mayotte',
'Indian/Reunion',
'Iran',
'Israel',
'Jamaica',
'Japan',
'Kwajalein',
'Libya',
'MET',
'MST',
'MST7MDT',
'Mexico/BajaNorte',
'Mexico/BajaSur',
'Mexico/General',
'NZ',
'NZ-CHAT',
'Navajo',
'PRC',
'PST8PDT',
'Pacific/Apia',
'Pacific/Auckland',
'Pacific/Bougainville',
'Pacific/Chatham',
'Pacific/Chuuk',
'Pacific/Easter',
'Pacific/Efate',
'Pacific/Enderbury',
'Pacific/Fakaofo',
'Pacific/Fiji',
'Pacific/Funafuti',
'Pacific/Galapagos',
'Pacific/Gambier',
'Pacific/Guadalcanal',
'Pacific/Guam',
'Pacific/Honolulu',
'Pacific/Johnston',
'Pacific/Kanton',
'Pacific/Kiritimati',
'Pacific/Kosrae',
'Pacific/Kwajalein',
'Pacific/Majuro',
'Pacific/Marquesas',
'Pacific/Midway',
'Pacific/Nauru',
'Pacific/Niue',
'Pacific/Norfolk',
'Pacific/Noumea',
'Pacific/Pago_Pago',
'Pacific/Palau',
'Pacific/Pitcairn',
'Pacific/Pohnpei',
'Pacific/Ponape',
'Pacific/Port_Moresby',
'Pacific/Rarotonga',
'Pacific/Saipan',
'Pacific/Samoa',
'Pacific/Tahiti',
'Pacific/Tarawa',
'Pacific/Tongatapu',
'Pacific/Truk',
'Pacific/Wake',
'Pacific/Wallis',
'Pacific/Yap',
'Poland',
'Portugal',
'ROC',
'ROK',
'Singapore',
'Turkey',
'UCT',
'US/Alaska',
'US/Aleutian',
'US/Arizona',
'US/Central',
'US/East-Indiana',
'US/Eastern',
'US/Hawaii',
'US/Indiana-Starke',
'US/Michigan',
'US/Mountain',
'US/Pacific',
'US/Samoa',
'UTC',
'Universal',
'W-SU',
'WET',
'Zulu',
];

24
packages/constants/index.ts
View File

@@ -16,9 +16,14 @@ export const timeWindows = {
},
today: {
key: 'today',
label: '24 hours',
label: 'Today',
shortcut: 'D',
},
yesterday: {
key: 'yesterday',
label: 'Yesterday',
shortcut: 'E',
},
'7d': {
key: '7d',
label: 'Last 7 days',
@@ -29,6 +34,16 @@ export const timeWindows = {
label: 'Last 30 days',
shortcut: 'T',
},
'6m': {
key: '6m',
label: 'Last 6 months',
shortcut: '6',
},
'12m': {
key: '12m',
label: 'Last 12 months',
shortcut: '0',
},
monthToDate: {
key: 'monthToDate',
label: 'Month to Date',
@@ -167,7 +182,10 @@ export function isMinuteIntervalEnabledByRange(
export function isHourIntervalEnabledByRange(range: keyof typeof timeWindows) {
return (
isMinuteIntervalEnabledByRange(range) || range === 'today' || range === '7d'
isMinuteIntervalEnabledByRange(range) ||
range === 'today' ||
range === 'yesterday' ||
range === '7d'
);
}
@@ -177,7 +195,7 @@ export function getDefaultIntervalByRange(
if (range === '30min' || range === 'lastHour') {
return 'minute';
}
if (range === 'today') {
if (range === 'today' || range === 'yesterday') {
return 'hour';
}
if (

2
packages/db/prisma/migrations/20250518190347_add_timezone/migration.sql Normal file
View File

@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "organizations" ADD COLUMN "timezone" TEXT;

1
packages/db/prisma/schema.prisma
View File

@@ -54,6 +54,7 @@ model Organization {
ShareOverview ShareOverview[]
integrations Integration[]
invites Invite[]
timezone String?
// Subscription
subscriptionId String?

14
packages/db/src/clickhouse/client.ts
View File

@@ -1,4 +1,4 @@
import type { ResponseJSON } from '@clickhouse/client';
import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client';
import { ClickHouseLogLevel, createClient } from '@clickhouse/client';
import { escape } from 'sqlstring';
@@ -11,7 +11,6 @@ export { createClient };
const logger = createLogger({ name: 'clickhouse' });
import type { Logger } from '@clickhouse/client';
import { getTimezoneFromDateString } from '@openpanel/common';
// All three LogParams types are exported by the client
interface LogParams {
@@ -142,10 +141,12 @@ export const ch = new Proxy(originalCh, {
export async function chQueryWithMeta<T extends Record<string, any>>(
query: string,
clickhouseSettings?: ClickHouseSettings,
): Promise<ResponseJSON<T>> {
const start = Date.now();
const res = await ch.query({
query,
clickhouse_settings: clickhouseSettings,
});
const json = await res.json<T>();
const keys = Object.keys(json.data[0] || {});
@@ -170,6 +171,7 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
rows: json.rows,
stats: response.statistics,
elapsed: Date.now() - start,
clickhouseSettings,
});
return response;
@@ -177,8 +179,9 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
export async function chQuery<T extends Record<string, any>>(
query: string,
clickhouseSettings?: ClickHouseSettings,
): Promise<T[]> {
return (await chQueryWithMeta<T>(query)).data;
return (await chQueryWithMeta<T>(query, clickhouseSettings)).data;
}
export function formatClickhouseDate(
@@ -188,7 +191,10 @@ export function formatClickhouseDate(
if (skipTime) {
return new Date(date).toISOString().split('T')[0]!;
}
return new Date(date).toISOString().replace('T', ' ').replace(/Z+$/, '');
return new Date(date)
.toISOString()
.replace('T', ' ')
.replace(/(\.\d{3})?Z+$/, '');
}
export function toDate(str: string, interval?: IInterval) {

72
packages/db/src/clickhouse/query-builder.ts
View File

@@ -73,7 +73,11 @@ export class Query<T = any> {
};
private _transform?: Record<string, (item: T) => any>;
private _union?: Query;
constructor(private client: ClickHouseClient) {}
private _dateRegex = /\d{4}-\d{2}-\d{2}([\s\:\d\.]+)?/g;
constructor(
private client: ClickHouseClient,
private timezone: string,
) {}
// Select methods
select<U>(
@@ -121,9 +125,14 @@ export class Query<T = any> {
if (Array.isArray(value)) {
return `(${value.map((v) => this.escapeValue(v)).join(', ')})`;
}
if (value instanceof Date) {
return escape(clix.datetime(value));
if (
(typeof value === 'string' && this._dateRegex.test(value)) ||
value instanceof Date
) {
return this.escapeDate(value);
}
return escape(value);
}
@@ -249,10 +258,10 @@ export class Query<T = any> {
private escapeDate(value: string | Date): string {
if (value instanceof Date) {
return clix.datetime(value);
return escape(clix.datetime(value));
}
return value.replaceAll(/\d{4}-\d{2}-\d{2}([\s\:\d\.]+)?/g, (match) => {
return value.replaceAll(this._dateRegex, (match) => {
return escape(match);
});
}
@@ -348,7 +357,10 @@ export class Query<T = any> {
// SELECT
if (this._select.length > 0) {
parts.push('SELECT', this._select.map(this.escapeDate).join(', '));
parts.push(
'SELECT',
this._select.map((col) => this.escapeDate(col)).join(', '),
);
} else {
parts.push('SELECT *');
}
@@ -483,26 +495,16 @@ export class Query<T = any> {
// Execution methods
async execute(): Promise<T[]> {
const query = this.buildQuery();
console.log('TEST QUERY ----->');
console.log(query);
console.log('<----------');
const perf = performance.now();
try {
const result = await this.client.query({
query,
});
const json = await result.json<T>();
const perf2 = performance.now();
console.log(`PERF: ${perf2 - perf}ms`);
return this.transformJson(json).data;
} catch (error) {
console.log('ERROR ----->');
console.log(error);
console.log('<----------');
console.log(query);
console.log('<----------');
throw error;
}
console.log('query', query);
const result = await this.client.query({
query,
clickhouse_settings: {
session_timezone: this.timezone,
},
});
const json = await result.json<T>();
return this.transformJson(json).data;
}
// Debug methods
@@ -535,7 +537,7 @@ export class Query<T = any> {
}
clone(): Query<T> {
return new Query(this.client).merge(this);
return new Query(this.client, this.timezone).merge(this);
}
// Add merge method
@@ -629,12 +631,8 @@ export class WhereGroupBuilder {
}
// Helper function to create a new query
export function createQuery(client: ClickHouseClient): Query {
return new Query(client);
}
export function clix(client: ClickHouseClient): Query {
return new Query(client);
export function clix(client: ClickHouseClient, timezone?: string): Query {
return new Query(client, timezone ?? 'UTC');
}
clix.exp = (expr: string | Query<any>) =>
@@ -654,7 +652,7 @@ clix.dynamicDatetime = (date: string | Date, interval: IInterval) => {
return clix.datetime(date);
};
clix.toStartOf = (node: string, interval: IInterval) => {
clix.toStartOf = (node: string, interval: IInterval, timezone?: string) => {
switch (interval) {
case 'minute': {
return `toStartOfMinute(${node})`;
@@ -666,10 +664,12 @@ clix.toStartOf = (node: string, interval: IInterval) => {
return `toStartOfDay(${node})`;
}
case 'week': {
return `toStartOfWeek(${node})`;
// Does not respect timezone settings (session_timezone) so we need to pass it manually
return `toStartOfWeek(${node}${timezone ? `, 1, '${timezone}'` : ''})`;
}
case 'month': {
return `toStartOfMonth(${node})`;
// Does not respect timezone settings (session_timezone) so we need to pass it manually
return `toStartOfMonth(${node}${timezone ? `, '${timezone}'` : ''})`;
}
}
};

52
packages/db/src/services/chart.service.ts
View File

@@ -1,19 +1,12 @@
import { escape } from 'sqlstring';
import {
getTimezoneFromDateString,
stripLeadingAndTrailingSlashes,
} from '@openpanel/common';
import { stripLeadingAndTrailingSlashes } from '@openpanel/common';
import type {
IChartEventFilter,
IGetChartDataInput,
} from '@openpanel/validation';
import {
TABLE_NAMES,
formatClickhouseDate,
toDate,
} from '../clickhouse/client';
import { TABLE_NAMES, formatClickhouseDate } from '../clickhouse/client';
import { createSqlBuilder } from '../sql-builder';
export function transformPropertyKey(property: string) {
@@ -61,9 +54,9 @@ export function getChartSql({
startDate,
endDate,
projectId,
chartType,
limit,
}: IGetChartDataInput) {
timezone,
}: IGetChartDataInput & { timezone: string }) {
const {
sb,
join,
@@ -73,6 +66,7 @@ export function getChartSql({
getSelect,
getOrderBy,
getGroupBy,
getFill,
} = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
@@ -99,34 +93,40 @@ export function getChartSql({
sb.select.count = 'count(*) as count';
switch (interval) {
case 'minute': {
sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfMinute(toDateTime('${startDate}')) TO toStartOfMinute(toDateTime('${endDate}')) STEP toIntervalMinute(1)`;
sb.select.date = 'toStartOfMinute(created_at) as date';
break;
}
case 'hour': {
sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfHour(toDateTime('${startDate}')) TO toStartOfHour(toDateTime('${endDate}')) STEP toIntervalHour(1)`;
sb.select.date = 'toStartOfHour(created_at) as date';
break;
}
case 'day': {
sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfDay(toDateTime('${startDate}')) TO toStartOfDay(toDateTime('${endDate}')) STEP toIntervalDay(1)`;
sb.select.date = 'toStartOfDay(created_at) as date';
break;
}
case 'week': {
sb.select.date = `toStartOfWeek(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfWeek(toDateTime('${startDate}'), 1, '${timezone}') TO toStartOfWeek(toDateTime('${endDate}'), 1, '${timezone}') STEP toIntervalWeek(1)`;
sb.select.date = `toStartOfWeek(created_at, 1, '${timezone}') as date`;
break;
}
case 'month': {
sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfMonth(toDateTime('${startDate}'), '${timezone}') TO toStartOfMonth(toDateTime('${endDate}'), '${timezone}') STEP toIntervalMonth(1)`;
sb.select.date = `toStartOfMonth(created_at, '${timezone}') as date`;
break;
}
}
sb.groupBy.date = 'date';
sb.orderBy.date = 'date ASC';
if (startDate) {
sb.where.startDate = `${toDate('created_at', interval)} >= ${toDate(formatClickhouseDate(startDate), interval)}`;
sb.where.startDate = `created_at >= toDateTime('${formatClickhouseDate(startDate)}')`;
}
if (endDate) {
sb.where.endDate = `${toDate('created_at', interval)} <= ${toDate(formatClickhouseDate(endDate), interval)}`;
sb.where.endDate = `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;
}
if (breakdowns.length > 0 && limit) {
@@ -179,18 +179,14 @@ export function getChartSql({
ORDER BY profile_id, created_at DESC
) as subQuery`;
console.log(
`${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
);
return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
console.log('CHART SQL', sql);
return sql;
}
console.log(
`${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
);
return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
console.log('CHART SQL', sql);
return sql;
}
export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {

22
packages/db/src/services/insights.service.ts
View File

@@ -1,5 +1,5 @@
import type { ClickHouseClient } from '@clickhouse/client';
import { Query, createQuery } from '../clickhouse/query-builder';
import { clix } from '../clickhouse/query-builder';
export interface Insight {
type: string;
@@ -73,7 +73,7 @@ export class InsightsService {
constructor(private client: ClickHouseClient) {}
private async getTrafficSpikes(projectId: string): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'referrer_name',
'toDate(created_at) as date',
@@ -100,7 +100,7 @@ export class InsightsService {
}
private async getEventSurges(projectId: string): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toDate(created_at) as date',
'COUNT(*) as event_count',
@@ -126,7 +126,7 @@ export class InsightsService {
}
private async getNewVisitorTrends(projectId: string): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toMonth(created_at) as month',
'COUNT(DISTINCT device_id) as new_visitors',
@@ -155,7 +155,7 @@ export class InsightsService {
private async getReferralSourceHighlights(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'referrer_name',
'COUNT(*) as count',
@@ -179,7 +179,7 @@ export class InsightsService {
private async getSessionDurationChanges(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toWeek(created_at) as week',
'avg(duration) as avg_duration',
@@ -205,7 +205,7 @@ export class InsightsService {
}
private async getTopPerformingContent(projectId: string): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'path',
'COUNT(*) as view_count',
@@ -233,7 +233,7 @@ export class InsightsService {
private async getBounceRateImprovements(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toMonth(created_at) as month',
'sum(is_bounce) / COUNT(*) as bounce_rate',
@@ -261,7 +261,7 @@ export class InsightsService {
private async getReturningVisitorTrends(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toQuarter(created_at) as quarter',
'COUNT(DISTINCT device_id) as returning_visitors',
@@ -290,7 +290,7 @@ export class InsightsService {
private async getGeographicInterestShifts(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'country',
'COUNT(*) as visitor_count',
@@ -318,7 +318,7 @@ export class InsightsService {
private async getEventCompletionChanges(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'event_name',
'toMonth(created_at) as month',

39
packages/db/src/services/organization.service.ts
View File

@@ -14,10 +14,6 @@ export type IServiceMember = Prisma.MemberGetPayload<{
}> & { access: ProjectAccess[] };
export type IServiceProjectAccess = ProjectAccess;
export function transformOrganization<T>(org: T) {
return org;
}
export async function getOrganizations(userId: string | null) {
if (!userId) return [];
@@ -34,10 +30,10 @@ export async function getOrganizations(userId: string | null) {
},
});
return organizations.map(transformOrganization);
return organizations;
}
export function getOrganizationBySlug(slug: string) {
export function getOrganizationById(slug: string) {
return db.organization.findUniqueOrThrow({
where: {
id: slug,
@@ -59,7 +55,7 @@ export async function getOrganizationByProjectId(projectId: string) {
return null;
}
return transformOrganization(project.organization);
return project.organization;
}
export const getOrganizationByProjectIdCached = cacheable(
@@ -258,3 +254,32 @@ export async function getOrganizationSubscriptionChartEndDate(
return endDate;
}
const DEFAULT_TIMEZONE = 'UTC';
export async function getSettingsForOrganization(organizationId: string) {
const organization = await db.organization.findUniqueOrThrow({
where: {
id: organizationId,
},
});
return {
timezone: organization.timezone || DEFAULT_TIMEZONE,
};
}
export async function getSettingsForProject(projectId: string) {
const project = await db.project.findUniqueOrThrow({
where: {
id: projectId,
},
include: {
organization: true,
},
});
return {
timezone: project.organization.timezone || DEFAULT_TIMEZONE,
};
}

133
packages/db/src/services/overview.service.ts
View File

@@ -15,7 +15,9 @@ export const zGetMetricsInput = z.object({
interval: zTimeInterval,
});
export type IGetMetricsInput = z.infer<typeof zGetMetricsInput>;
export type IGetMetricsInput = z.infer<typeof zGetMetricsInput> & {
timezone: string;
};
export const zGetTopPagesInput = z.object({
projectId: z.string(),
@@ -27,7 +29,9 @@ export const zGetTopPagesInput = z.object({
limit: z.number().optional(),
});
export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput>;
export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput> & {
timezone: string;
};
export const zGetTopEntryExitInput = z.object({
projectId: z.string(),
@@ -40,7 +44,9 @@ export const zGetTopEntryExitInput = z.object({
limit: z.number().optional(),
});
export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput>;
export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput> & {
timezone: string;
};
export const zGetTopGenericInput = z.object({
projectId: z.string(),
@@ -75,7 +81,9 @@ export const zGetTopGenericInput = z.object({
limit: z.number().optional(),
});
export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput>;
export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput> & {
timezone: string;
};
export class OverviewService {
private pendingQueries: Map<string, Promise<number | null>> = new Map();
@@ -91,11 +99,13 @@ export class OverviewService {
startDate,
endDate,
filters,
timezone,
}: {
projectId: string;
startDate: string;
endDate: string;
filters: IChartEventFilter[];
timezone: string;
}) {
const where = this.getRawWhereClause('sessions', filters);
const key = `total_sessions_${projectId}_${startDate}_${endDate}_${JSON.stringify(filters)}`;
@@ -109,15 +119,15 @@ export class OverviewService {
// Create new query promise and store it
const queryPromise = getCache(key, 15, async () => {
try {
const result = await clix(this.client)
const result = await clix(this.client, timezone)
.select<{
total_sessions: number;
}>(['sum(sign) as total_sessions'])
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(where)
.having('sum(sign)', '>', 0)
@@ -138,6 +148,7 @@ export class OverviewService {
startDate,
endDate,
interval,
timezone,
}: IGetMetricsInput): Promise<{
metrics: {
bounce_rate: number;
@@ -160,17 +171,17 @@ export class OverviewService {
const where = this.getRawWhereClause('sessions', filters);
if (this.isPageFilter(filters)) {
// Session aggregation with bounce rates
const sessionAggQuery = clix(this.client)
const sessionAggQuery = clix(this.client, timezone)
.select([
`${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
`${clix.toStartOf('created_at', interval, timezone)} AS date`,
'round((countIf(is_bounce = 1 AND sign = 1) * 100.) / countIf(sign = 1), 2) AS bounce_rate',
])
.from(TABLE_NAMES.sessions, true)
.where('sign', '=', 1)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(where)
.groupBy(['date'])
@@ -178,7 +189,7 @@ export class OverviewService {
.orderBy('date', 'ASC');
// Overall unique visitors
const overallUniqueVisitorsQuery = clix(this.client)
const overallUniqueVisitorsQuery = clix(this.client, timezone)
.select([
'uniq(profile_id) AS unique_visitors',
'uniq(session_id) AS total_sessions',
@@ -187,23 +198,23 @@ export class OverviewService {
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters));
return clix(this.client)
return clix(this.client, timezone)
.with('session_agg', sessionAggQuery)
.with(
'overall_bounce_rate',
clix(this.client)
clix(this.client, timezone)
.select(['bounce_rate'])
.from('session_agg')
.where('date', '=', clix.exp("'1970-01-01 00:00:00'")),
)
.with(
'daily_stats',
clix(this.client)
clix(this.client, timezone)
.select(['date', 'bounce_rate'])
.from('session_agg')
.where('date', '!=', clix.exp("'1970-01-01 00:00:00'")),
@@ -221,7 +232,7 @@ export class OverviewService {
overall_total_sessions: number;
overall_bounce_rate: number;
}>([
`${clix.toStartOfInterval('e.created_at', interval, startDate)} AS date`,
`${clix.toInterval('e.created_at', interval)} AS date`,
'ds.bounce_rate as bounce_rate',
'uniq(e.profile_id) AS unique_visitors',
'uniq(e.session_id) AS total_sessions',
@@ -236,20 +247,29 @@ export class OverviewService {
.from(`${TABLE_NAMES.events} AS e`)
.leftJoin(
'daily_stats AS ds',
`${clix.toStartOfInterval('e.created_at', interval, startDate)} = ds.date`,
`${clix.toInterval('e.created_at', interval)} = ds.date`,
)
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
.where('e.created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters))
.groupBy(['date', 'ds.bounce_rate'])
.orderBy('date', 'ASC')
.fill(
clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
clix.toStartOf(
clix.datetime(
startDate,
['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
),
interval,
),
clix.datetime(
endDate,
['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
),
clix.toInterval('1', interval),
)
.transform({
@@ -289,7 +309,7 @@ export class OverviewService {
});
}
const query = clix(this.client)
const query = clix(this.client, timezone)
.select<{
date: string;
bounce_rate: number;
@@ -299,7 +319,7 @@ export class OverviewService {
total_screen_views: number;
views_per_session: number;
}>([
`${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
`${clix.toStartOf('created_at', interval, timezone)} AS date`,
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
'uniqIf(profile_id, sign > 0) AS unique_visitors',
'sum(sign) AS total_sessions',
@@ -310,8 +330,8 @@ export class OverviewService {
])
.from('sessions')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.where('project_id', '=', projectId)
.rawWhere(where)
@@ -320,8 +340,17 @@ export class OverviewService {
.rollup()
.orderBy('date', 'ASC')
.fill(
clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
clix.toStartOf(
clix.datetime(
startDate,
['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
),
interval,
),
clix.datetime(
endDate,
['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
),
clix.toInterval('1', interval),
)
.transform({
@@ -384,8 +413,9 @@ export class OverviewService {
endDate,
cursor = 1,
limit = 10,
timezone,
}: IGetTopPagesInput) {
const pageStatsQuery = clix(this.client)
const pageStatsQuery = clix(this.client, timezone)
.select([
'origin',
'path',
@@ -398,15 +428,15 @@ export class OverviewService {
.where('name', '=', 'screen_view')
.where('path', '!=', '')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.groupBy(['origin', 'path'])
.orderBy('count', 'DESC')
.limit(limit)
.offset((cursor - 1) * limit);
const bounceStatsQuery = clix(this.client)
const bounceStatsQuery = clix(this.client, timezone)
.select([
'entry_path',
'entry_origin',
@@ -416,15 +446,15 @@ export class OverviewService {
.where('sign', '=', 1)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.groupBy(['entry_path', 'entry_origin']);
pageStatsQuery.rawWhere(this.getRawWhereClause('events', filters));
bounceStatsQuery.rawWhere(this.getRawWhereClause('sessions', filters));
const mainQuery = clix(this.client)
const mainQuery = clix(this.client, timezone)
.with('page_stats', pageStatsQuery)
.with('bounce_stats', bounceStatsQuery)
.select<{
@@ -455,6 +485,7 @@ export class OverviewService {
startDate,
endDate,
filters,
timezone,
});
return mainQuery.execute();
@@ -468,6 +499,7 @@ export class OverviewService {
mode,
cursor = 1,
limit = 10,
timezone,
}: IGetTopEntryExitInput) {
const where = this.getRawWhereClause('sessions', filters);
@@ -476,11 +508,12 @@ export class OverviewService {
filters,
startDate,
endDate,
timezone,
});
const offset = (cursor - 1) * limit;
const query = clix(this.client)
const query = clix(this.client, timezone)
.select<{
origin: string;
path: string;
@@ -497,8 +530,8 @@ export class OverviewService {
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(where)
.groupBy([`${mode}_origin`, `${mode}_path`])
@@ -510,7 +543,7 @@ export class OverviewService {
let mainQuery = query;
if (this.isPageFilter(filters)) {
mainQuery = clix(this.client)
mainQuery = clix(this.client, timezone)
.with('distinct_sessions', distinctSessionQuery)
.merge(query)
.where(
@@ -525,6 +558,7 @@ export class OverviewService {
startDate,
endDate,
filters,
timezone,
});
return mainQuery.execute();
@@ -535,19 +569,21 @@ export class OverviewService {
filters,
startDate,
endDate,
timezone,
}: {
projectId: string;
filters: IChartEventFilter[];
startDate: string;
endDate: string;
timezone: string;
}) {
return clix(this.client)
return clix(this.client, timezone)
.select(['DISTINCT session_id'])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters));
}
@@ -560,12 +596,14 @@ export class OverviewService {
column,
cursor = 1,
limit = 10,
timezone,
}: IGetTopGenericInput) {
const distinctSessionQuery = this.getDistinctSessions({
projectId,
filters,
startDate,
endDate,
timezone,
});
const prefixColumn = (() => {
@@ -584,7 +622,7 @@ export class OverviewService {
const offset = (cursor - 1) * limit;
const query = clix(this.client)
const query = clix(this.client, timezone)
.select<{
prefix?: string;
name: string;
@@ -601,8 +639,8 @@ export class OverviewService {
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.groupBy([prefixColumn, column].filter(Boolean))
.having('sum(sign)', '>', 0)
@@ -613,7 +651,7 @@ export class OverviewService {
let mainQuery = query;
if (this.isPageFilter(filters)) {
mainQuery = clix(this.client)
mainQuery = clix(this.client, timezone)
.with('distinct_sessions', distinctSessionQuery)
.merge(query)
.where(
@@ -632,6 +670,7 @@ export class OverviewService {
startDate,
endDate,
filters,
timezone,
}),
]);

5
packages/db/src/sql-builder.ts
View File

@@ -10,6 +10,7 @@ export interface SqlBuilderObject {
joins: Record<string, string>;
limit: number | undefined;
offset: number | undefined;
fill: string | undefined;
}
export function createSqlBuilder() {
@@ -26,6 +27,7 @@ export function createSqlBuilder() {
joins: {},
limit: undefined,
offset: undefined,
fill: undefined,
};
const getWhere = () =>
@@ -43,6 +45,7 @@ export function createSqlBuilder() {
const getOffset = () => (sb.offset ? `OFFSET ${sb.offset}` : '');
const getJoins = () =>
Object.keys(sb.joins).length ? join(sb.joins, ' ') : '';
const getFill = () => (sb.fill ? `WITH FILL ${sb.fill}` : '');
return {
sb,
@@ -54,6 +57,7 @@ export function createSqlBuilder() {
getOrderBy,
getHaving,
getJoins,
getFill,
getSql: () => {
const sql = [
getSelect(),
@@ -65,6 +69,7 @@ export function createSqlBuilder() {
getOrderBy(),
getLimit(),
getOffset(),
getFill(),
]
.filter(Boolean)
.join(' ');

397
packages/trpc/src/routers/chart.helpers.ts
View File

@@ -1,45 +1,29 @@
import {
differenceInMilliseconds,
endOfMonth,
endOfYear,
formatISO,
startOfDay,
startOfMonth,
startOfYear,
subDays,
subMilliseconds,
subMinutes,
subMonths,
subYears,
} from 'date-fns';
import * as mathjs from 'mathjs';
import { last, pluck, repeat, reverse, uniq } from 'ramda';
import { last, pluck, reverse, uniq } from 'ramda';
import { escape } from 'sqlstring';
import type { ISerieDataItem } from '@openpanel/common';
import {
DateTime,
average,
completeSerie,
getPreviousMetric,
groupByLabels,
max,
min,
round,
slug,
sum,
} from '@openpanel/common';
import type { ISerieDataItem } from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import {
TABLE_NAMES,
chQuery,
createSqlBuilder,
db,
formatClickhouseDate,
getChartSql,
getEventFiltersWhereClause,
getOrganizationByProjectId,
getOrganizationByProjectIdCached,
getOrganizationSubscriptionChartEndDate,
getProfiles,
getSettingsForProject,
} from '@openpanel/db';
import type {
FinalChart,
@@ -48,9 +32,7 @@ import type {
IChartInputWithDates,
IChartRange,
IGetChartDataInput,
IInterval,
} from '@openpanel/validation';
import { TRPCNotFoundError } from '../errors';
function getEventLegend(event: IChartEvent) {
return event.displayName || event.name;
@@ -134,115 +116,190 @@ export function withFormula(
];
}
const toDynamicISODateWithTZ = (
date: string,
blueprint: string,
interval: IInterval,
) => {
// If we have a space in the date we know it's a date with time
if (date.includes(' ')) {
// If interval is minutes we need to convert the timezone to what timezone is used (either on client or the server)
// - We use timezone from server if its a predefined range (yearToDate, lastYear, etc.)
// - We use timezone from client if its a custom range
if (interval === 'minute' || interval === 'hour') {
return (
date.replace(' ', 'T') +
// Only append timezone if it's not UTC (Z)
(blueprint.match(/[+-]\d{2}:\d{2}/) ? blueprint.slice(-6) : 'Z')
);
}
// Otherwise we just return without the timezone
// It will be converted to the correct timezone on the client
return date.replace(' ', 'T');
}
return `${date}T00:00:00Z`;
};
export function getDatesFromRange(range: IChartRange) {
export function getDatesFromRange(range: IChartRange, timezone: string) {
if (range === '30min' || range === 'lastHour') {
const minutes = range === '30min' ? 30 : 60;
const startDate = formatISO(subMinutes(new Date(), minutes));
const endDate = formatISO(new Date());
const startDate = DateTime.now()
.minus({ minute: minutes })
.startOf('minute')
.setZone(timezone)
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.setZone(timezone)
.endOf('minute')
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate,
endDate,
startDate: startDate,
endDate: endDate,
};
}
if (range === 'today') {
// This is last 24 hours instead
// Makes it easier to handle timezones
// const startDate = startOfDay(new Date());
// const endDate = endOfDay(new Date());
const startDate = subDays(new Date(), 1);
const endDate = new Date();
const startDate = DateTime.now()
.setZone(timezone)
.startOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.setZone(timezone)
.endOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: formatISO(startDate),
endDate: formatISO(endDate),
startDate: startDate,
endDate: endDate,
};
}
if (range === 'yesterday') {
const startDate = DateTime.now()
.minus({ day: 1 })
.setZone(timezone)
.startOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.minus({ day: 1 })
.setZone(timezone)
.endOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
};
}
if (range === '7d') {
const startDate = formatISO(startOfDay(subDays(new Date(), 7)));
const endDate = formatISO(new Date());
const startDate = DateTime.now()
.minus({ day: 7 })
.setZone(timezone)
.startOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.setZone(timezone)
.endOf('day')
.plus({ millisecond: 1 })
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate,
endDate,
startDate: startDate,
endDate: endDate,
};
}
if (range === '6m') {
const startDate = DateTime.now()
.minus({ month: 6 })
.setZone(timezone)
.startOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.setZone(timezone)
.endOf('day')
.plus({ millisecond: 1 })
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
};
}
if (range === '12m') {
const startDate = DateTime.now()
.minus({ month: 12 })
.setZone(timezone)
.startOf('month')
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.setZone(timezone)
.endOf('month')
.plus({ millisecond: 1 })
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
};
}
if (range === 'monthToDate') {
const startDate = formatISO(startOfMonth(new Date()));
const endDate = formatISO(new Date());
const startDate = DateTime.now()
.setZone(timezone)
.startOf('month')
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.setZone(timezone)
.endOf('day')
.plus({ millisecond: 1 })
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate,
endDate,
startDate: startDate,
endDate: endDate,
};
}
if (range === 'lastMonth') {
const month = subMonths(new Date(), 1);
const startDate = formatISO(startOfMonth(month));
const endDate = formatISO(endOfMonth(month));
const month = DateTime.now()
.minus({ month: 1 })
.setZone(timezone)
.startOf('month');
const startDate = month.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = month
.endOf('month')
.plus({ millisecond: 1 })
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate,
endDate,
startDate: startDate,
endDate: endDate,
};
}
if (range === 'yearToDate') {
const startDate = formatISO(startOfYear(new Date()));
const endDate = formatISO(new Date());
const startDate = DateTime.now()
.setZone(timezone)
.startOf('year')
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.setZone(timezone)
.endOf('day')
.plus({ millisecond: 1 })
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate,
endDate,
startDate: startDate,
endDate: endDate,
};
}
if (range === 'lastYear') {
const year = subYears(new Date(), 1);
const startDate = formatISO(startOfYear(year));
const endDate = formatISO(endOfYear(year));
const year = DateTime.now().minus({ year: 1 }).setZone(timezone);
const startDate = year.startOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = year.endOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate,
endDate,
startDate: startDate,
endDate: endDate,
};
}
// range === '30d'
const startDate = formatISO(startOfDay(subDays(new Date(), 30)));
const endDate = formatISO(new Date());
const startDate = DateTime.now()
.minus({ day: 30 })
.setZone(timezone)
.startOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
const endDate = DateTime.now()
.setZone(timezone)
.endOf('day')
.plus({ millisecond: 1 })
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate,
endDate,
startDate: startDate,
endDate: endDate,
};
}
@@ -268,12 +325,15 @@ function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
return filled.reverse();
}
export function getChartStartEndDate({
startDate,
endDate,
range,
}: Pick<IChartInput, 'endDate' | 'startDate' | 'range'>) {
const ranges = getDatesFromRange(range);
export function getChartStartEndDate(
{
startDate,
endDate,
range,
}: Pick<IChartInput, 'endDate' | 'startDate' | 'range'>,
timezone: string,
) {
const ranges = getDatesFromRange(range, timezone);
if (startDate && endDate) {
return { startDate: startDate, endDate: endDate };
@@ -293,10 +353,25 @@ export function getChartPrevStartEndDate({
startDate: string;
endDate: string;
}) {
const diff = differenceInMilliseconds(new Date(endDate), new Date(startDate));
let diff = DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss').diff(
DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss'),
);
// this will make sure our start and end date's are correct
// otherwise if a day ends with 23:59:59.999 and starts with 00:00:00.000
// the diff will be 23:59:59.999 and that will make the start date wrong
// so we add 1 millisecond to the diff
if ((diff.milliseconds / 1000) % 2 !== 0) {
diff = diff.plus({ millisecond: 1 });
}
return {
startDate: formatISO(subMilliseconds(new Date(startDate), diff + 1000)),
endDate: formatISO(subMilliseconds(new Date(endDate), diff + 1000)),
startDate: DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss')
.minus({ millisecond: diff.milliseconds })
.toFormat('yyyy-MM-dd HH:mm:ss'),
endDate: DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss')
.minus({ millisecond: diff.milliseconds })
.toFormat('yyyy-MM-dd HH:mm:ss'),
};
}
@@ -386,118 +461,60 @@ export async function getFunnelData({
};
}
export async function getFunnelStep({
projectId,
startDate,
endDate,
step,
...payload
}: IChartInput & {
step: number;
}) {
throw new Error('not implemented');
// if (!startDate || !endDate) {
// throw new Error('startDate and endDate are required');
// }
// if (payload.events.length === 0) {
// throw new Error('no events selected');
// }
// const funnels = payload.events.map((event) => {
// const { sb, getWhere } = createSqlBuilder();
// sb.where = getEventFiltersWhereClause(event.filters);
// sb.where.name = `name = ${escape(event.name)}`;
// return getWhere().replace('WHERE ', '');
// });
// const innerSql = `SELECT
// session_id,
// windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
// FROM ${TABLE_NAMES.events}
// WHERE
// project_id = ${escape(projectId)} AND
// created_at >= '${formatClickhouseDate(startDate)}' AND
// created_at <= '${formatClickhouseDate(endDate)}' AND
// name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
// GROUP BY session_id`;
// const profileIdsQuery = `WITH sessions AS (${innerSql})
// SELECT
// DISTINCT e.profile_id as id
// FROM sessions s
// JOIN ${TABLE_NAMES.events} e ON s.session_id = e.session_id
// WHERE
// s.level = ${step} AND
// e.project_id = ${escape(projectId)} AND
// e.created_at >= '${formatClickhouseDate(startDate)}' AND
// e.created_at <= '${formatClickhouseDate(endDate)}' AND
// name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
// ORDER BY e.created_at DESC
// LIMIT 500
// `;
// const res = await chQuery<{
// id: string;
// }>(profileIdsQuery);
// return getProfiles(
// res.map((r) => r.id),
// projectId,
// );
}
export async function getChartSerie(payload: IGetChartDataInput) {
export async function getChartSerie(
payload: IGetChartDataInput,
timezone: string,
) {
async function getSeries() {
const result = await chQuery<ISerieDataItem>(getChartSql(payload));
const result = await chQuery<ISerieDataItem>(
getChartSql({ ...payload, timezone }),
{
session_timezone: timezone,
},
);
if (result.length === 0 && payload.breakdowns.length > 0) {
return await chQuery<ISerieDataItem>(
getChartSql({
...payload,
breakdowns: [],
timezone,
}),
{
session_timezone: timezone,
},
);
}
return result;
}
return getSeries()
.then((data) =>
completeSerie(data, payload.startDate, payload.endDate, payload.interval),
)
.then(groupByLabels)
.then((series) => {
return Object.keys(series).map((key) => {
const firstDataItem = series[key]![0]!;
const isBreakdown =
payload.breakdowns.length && firstDataItem.labels.length;
const serieLabel = isBreakdown
? firstDataItem.labels
: [getEventLegend(payload.event)];
return series.map((serie) => {
return {
name: serieLabel,
...serie,
event: payload.event,
data: series[key]!.map((item) => ({
...item,
date: toDynamicISODateWithTZ(
item.date,
payload.startDate,
payload.interval,
),
})),
};
});
});
}
export type IGetChartSerie = Awaited<ReturnType<typeof getChartSeries>>[number];
export async function getChartSeries(input: IChartInputWithDates) {
export async function getChartSeries(
input: IChartInputWithDates,
timezone: string,
) {
const series = (
await Promise.all(
input.events.map(async (event) =>
getChartSerie({
...input,
event,
}),
getChartSerie(
{
...input,
event,
},
timezone,
),
),
)
).flat();
@@ -510,7 +527,8 @@ export async function getChartSeries(input: IChartInputWithDates) {
}
export async function getChart(input: IChartInput) {
const currentPeriod = getChartStartEndDate(input);
const { timezone } = await getSettingsForProject(input.projectId);
const currentPeriod = getChartStartEndDate(input, timezone);
const previousPeriod = getChartPrevStartEndDate(currentPeriod);
const endDate = await getOrganizationSubscriptionChartEndDate(
@@ -522,14 +540,17 @@ export async function getChart(input: IChartInput) {
currentPeriod.endDate = endDate;
}
const promises = [getChartSeries({ ...input, ...currentPeriod })];
const promises = [getChartSeries({ ...input, ...currentPeriod }, timezone)];
if (input.previous) {
promises.push(
getChartSeries({
...input,
...previousPeriod,
}),
getChartSeries(
{
...input,
...previousPeriod,
},
timezone,
),
);
}

14
packages/trpc/src/routers/chart.ts
View File

@@ -13,6 +13,7 @@ import {
db,
funnelService,
getSelectPropertyKey,
getSettingsForProject,
toDate,
} from '@openpanel/db';
import {
@@ -80,7 +81,7 @@ export const chartRouter = createTRPCRouter({
}),
)
.query(async ({ input: { projectId, event } }) => {
const profiles = await clix(ch)
const profiles = await clix(ch, 'UTC')
.select<Pick<IServiceProfile, 'properties'>>(['properties'])
.from(TABLE_NAMES.profiles)
.where('project_id', '=', projectId)
@@ -214,7 +215,8 @@ export const chartRouter = createTRPCRouter({
}),
funnel: protectedProcedure.input(zChartInput).query(async ({ input }) => {
const currentPeriod = getChartStartEndDate(input);
const { timezone } = await getSettingsForProject(input.projectId);
const currentPeriod = getChartStartEndDate(input, timezone);
const previousPeriod = getChartPrevStartEndDate(currentPeriod);
const [current, previous] = await Promise.all([
@@ -231,7 +233,8 @@ export const chartRouter = createTRPCRouter({
}),
conversion: protectedProcedure.input(zChartInput).query(async ({ input }) => {
const currentPeriod = getChartStartEndDate(input);
const { timezone } = await getSettingsForProject(input.projectId);
const currentPeriod = getChartStartEndDate(input, timezone);
const previousPeriod = getChartPrevStartEndDate(currentPeriod);
const [current, previous] = await Promise.all([
@@ -254,7 +257,7 @@ export const chartRouter = createTRPCRouter({
}),
chart: publicProcedure
.use(cacher)
// .use(cacher)
.input(zChartInput)
.query(async ({ input, ctx }) => {
if (ctx.session.userId) {
@@ -301,8 +304,9 @@ export const chartRouter = createTRPCRouter({
}),
)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const { projectId, firstEvent, secondEvent } = input;
const dates = getChartStartEndDate(input);
const dates = getChartStartEndDate(input, timezone);
const diffInterval = {
minute: () => differenceInDays(dates.endDate, dates.startDate),
hour: () => differenceInDays(dates.endDate, dates.startDate),

5
packages/trpc/src/routers/event.ts
View File

@@ -12,6 +12,7 @@ import {
formatClickhouseDate,
getEventList,
getEvents,
getSettingsForProject,
overviewService,
sessionService,
} from '@openpanel/db';
@@ -275,7 +276,8 @@ export const eventRouter = createTRPCRouter({
}),
)
.query(async ({ input }) => {
const { startDate, endDate } = getChartStartEndDate(input);
const { timezone } = await getSettingsForProject(input.projectId);
const { startDate, endDate } = getChartStartEndDate(input, timezone);
if (input.search) {
input.filters.push({
id: 'path',
@@ -292,6 +294,7 @@ export const eventRouter = createTRPCRouter({
interval: input.interval,
cursor: input.cursor || 1,
limit: input.take,
timezone,
});
}),

5
packages/trpc/src/routers/onboarding.ts
View File

@@ -2,7 +2,7 @@ import crypto from 'node:crypto';
import type { z } from 'zod';
import { stripTrailingSlash } from '@openpanel/common';
import { db, getId, getOrganizationBySlug, getUserById } from '@openpanel/db';
import { db, getId, getOrganizationById, getUserById } from '@openpanel/db';
import type { IServiceUser, ProjectType } from '@openpanel/db';
import { zOnboardingProject } from '@openpanel/validation';
@@ -16,7 +16,7 @@ async function createOrGetOrganization(
user: IServiceUser,
) {
if (input.organizationId) {
return await getOrganizationBySlug(input.organizationId);
return await getOrganizationById(input.organizationId);
}
const TRIAL_DURATION_IN_DAYS = 30;
@@ -29,6 +29,7 @@ async function createOrGetOrganization(
createdByUserId: user.id,
subscriptionEndsAt: addDays(new Date(), TRIAL_DURATION_IN_DAYS),
subscriptionStatus: 'trialing',
timezone: input.timezone,
},
});

10
packages/trpc/src/routers/organization.ts
View File

@@ -1,7 +1,7 @@
import { z } from 'zod';
import { connectUserToOrganization, db } from '@openpanel/db';
import { zInviteUser } from '@openpanel/validation';
import { zEditOrganization, zInviteUser } from '@openpanel/validation';
import { generateSecureId } from '@openpanel/common/server/id';
import { sendEmail } from '@openpanel/email';
@@ -12,12 +12,7 @@ import { createTRPCRouter, protectedProcedure } from '../trpc';
export const organizationRouter = createTRPCRouter({
update: protectedProcedure
.input(
z.object({
id: z.string(),
name: z.string(),
}),
)
.input(zEditOrganization)
.mutation(async ({ input, ctx }) => {
const access = await getOrganizationAccess({
userId: ctx.session.userId,
@@ -34,6 +29,7 @@ export const organizationRouter = createTRPCRouter({
},
data: {
name: input.name,
timezone: input.timezone,
},
});
}),

24
packages/trpc/src/routers/overview.ts
View File

@@ -1,13 +1,13 @@
import {
getOrganizationByProjectIdCached,
getOrganizationSubscriptionChartEndDate,
getSettingsForProject,
overviewService,
zGetMetricsInput,
zGetTopGenericInput,
zGetTopPagesInput,
} from '@openpanel/db';
import { type IChartRange, zRange } from '@openpanel/validation';
import { TRPCError } from '@trpc/server';
import { format } from 'date-fns';
import { z } from 'zod';
import { cacheMiddleware, createTRPCRouter, publicProcedure } from '../trpc';
import {
@@ -34,8 +34,8 @@ function getCurrentAndPrevious<
range: IChartRange;
projectId: string;
},
>(input: T, fetchPrevious = false) {
const current = getChartStartEndDate(input);
>(input: T, fetchPrevious: boolean, timezone: string) {
const current = getChartStartEndDate(input, timezone);
const previous = getChartPrevStartEndDate(current);
return async <R>(
@@ -88,9 +88,11 @@ export const overviewRouter = createTRPCRouter({
)
.use(cacher)
.query(async ({ ctx, input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const { current, previous } = await getCurrentAndPrevious(
input,
{ ...input, timezone },
true,
timezone,
)(overviewService.getMetrics.bind(overviewService));
return {
metrics: {
@@ -107,6 +109,7 @@ export const overviewRouter = createTRPCRouter({
const prev = previous?.series[index];
return {
...item,
date: format(item.date, 'yyyy-MM-dd HH:mm:ss'),
prev_bounce_rate: prev?.bounce_rate,
prev_unique_visitors: prev?.unique_visitors,
prev_total_screen_views: prev?.total_screen_views,
@@ -129,12 +132,14 @@ export const overviewRouter = createTRPCRouter({
)
.use(cacher)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const { current } = await getCurrentAndPrevious(
input,
{ ...input },
false,
timezone,
)(async (input) => {
if (input.mode === 'page') {
return overviewService.getTopPages(input);
return overviewService.getTopPages({ ...input, timezone });
}
if (input.mode === 'bot') {
@@ -144,6 +149,7 @@ export const overviewRouter = createTRPCRouter({
return overviewService.getTopEntryExit({
...input,
mode: input.mode,
timezone,
});
});
@@ -160,9 +166,11 @@ export const overviewRouter = createTRPCRouter({
)
.use(cacher)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const { current } = await getCurrentAndPrevious(
input,
{ ...input, timezone },
false,
timezone,
)(overviewService.getTopGeneric.bind(overviewService));
return current;

7
packages/trpc/src/routers/reference.ts
View File

@@ -1,6 +1,6 @@
import { z } from 'zod';
import { db, getReferences } from '@openpanel/db';
import { db, getReferences, getSettingsForProject } from '@openpanel/db';
import { zCreateReference, zRange } from '@openpanel/validation';
import { getProjectAccess } from '../access';
@@ -56,8 +56,9 @@ export const referenceRouter = createTRPCRouter({
range: zRange,
}),
)
.query(({ input: { projectId, ...input } }) => {
const { startDate, endDate } = getChartStartEndDate(input);
.query(async ({ input: { projectId, ...input } }) => {
const { timezone } = await getSettingsForProject(projectId);
const { startDate, endDate } = getChartStartEndDate(input, timezone);
return getReferences({
where: {
projectId,

8
packages/trpc/src/routers/subscription.ts
View File

@@ -1,7 +1,7 @@
import {
db,
getOrganizationBillingEventsCountSerieCached,
getOrganizationBySlug,
getOrganizationById,
} from '@openpanel/db';
import {
cancelSubscription,
@@ -24,7 +24,7 @@ export const subscriptionRouter = createTRPCRouter({
getCurrent: protectedProcedure
.input(z.object({ organizationId: z.string() }))
.query(async ({ input }) => {
const organization = await getOrganizationBySlug(input.organizationId);
const organization = await getOrganizationById(input.organizationId);
if (!organization.subscriptionProductId) {
return null;
@@ -150,7 +150,7 @@ export const subscriptionRouter = createTRPCRouter({
cancelSubscription: protectedProcedure
.input(z.object({ organizationId: z.string() }))
.mutation(async ({ input }) => {
const organization = await getOrganizationBySlug(input.organizationId);
const organization = await getOrganizationById(input.organizationId);
if (!organization.subscriptionId) {
throw TRPCBadRequestError('Organization has no subscription');
}
@@ -163,7 +163,7 @@ export const subscriptionRouter = createTRPCRouter({
portal: protectedProcedure
.input(z.object({ organizationId: z.string() }))
.mutation(async ({ input }) => {
const organization = await getOrganizationBySlug(input.organizationId);
const organization = await getOrganizationById(input.organizationId);
if (!organization.subscriptionCustomerId) {
throw TRPCBadRequestError('Organization has no subscription');
}

7
packages/validation/src/index.ts
View File

@@ -203,6 +203,7 @@ export const zOnboardingProject = z
website: z.boolean(),
app: z.boolean(),
backend: z.boolean(),
timezone: z.string().optional(),
})
.superRefine((data, ctx) => {
if (!data.organization && !data.organizationId) {
@@ -434,3 +435,9 @@ export const zCheckout = z.object({
productId: z.string(),
});
export type ICheckout = z.infer<typeof zCheckout>;
export const zEditOrganization = z.object({
id: z.string().min(2),
name: z.string().min(2),
timezone: z.string().min(1),
});