From de8f5c4413dddeaabac072f92a93168a275c4dd8 Mon Sep 17 00:00:00 2001 From: Shrutesh Sharma <102956391+shrutesh1@users.noreply.github.com> Date: Thu, 20 Feb 2025 11:11:27 +0530 Subject: [PATCH 01/14] Added batching --- src/pages/api/batch.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/pages/api/batch.ts diff --git a/src/pages/api/batch.ts b/src/pages/api/batch.ts new file mode 100644 index 0000000000..93632d114a --- /dev/null +++ b/src/pages/api/batch.ts @@ -0,0 +1,35 @@ +import sendHandler from './send'; + +export default async function handler(req, res) { + if (req.method !== 'POST') { + res.setHeader('Allow', ['POST']); + return res.status(405).end(`Method ${req.method} Not Allowed`); + } + + const events = req.body; + + if (!Array.isArray(events)) { + return res.status(400).json({ error: 'Invalid payload, expected an array.' }); + } + + try { + for (const event of events) { + const mockReq = { + ...req, + body: event, + headers: { ...req.headers, origin: req.headers.origin || 'http://localhost:3000' }, + }; + + const mockRes = { + ...res, + end: () => {}, // Prevent premature response closure + }; + + await sendHandler(mockReq, mockRes); + } + + return res.status(200).json({ success: true, message: `${events.length} events processed.` }); + } catch (error) { + return res.status(500).json({ error: 'Internal Server Error' }); + } +} From 75b0b2e67753656ba1a6c266c235fbf294495501 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Sat, 22 Feb 2025 08:09:01 -0800 Subject: [PATCH 02/14] Make user id optional. --- src/app/api/users/route.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/api/users/route.ts b/src/app/api/users/route.ts index f6b32fe7e8..c5896f8929 100644 --- a/src/app/api/users/route.ts +++ b/src/app/api/users/route.ts @@ -8,6 +8,7 @@ import { createUser, getUserByUsername } from '@/queries'; export async function POST(request: Request) { const schema = z.object({ + id: z.string().uuid().optional(), username: z.string().max(255), password: z.string(), role: z.string().regex(/admin|user|view-only/i), @@ -23,7 +24,7 @@ export async function POST(request: Request) { return unauthorized(); } - const { username, password, role } = body; + const { id, username, password, role } = body; const existingUser = await getUserByUsername(username, { showDeleted: true }); @@ -32,7 +33,7 @@ export async function POST(request: Request) { } const user = await createUser({ - id: uuid(), + id: id || uuid(), username, password: hashPassword(password), role: role ?? ROLES.user, From bdeaa9e5c667a30d75ba827faede8538426cb945 Mon Sep 17 00:00:00 2001 From: Harry Oosterveen Date: Tue, 25 Feb 2025 12:33:02 +0100 Subject: [PATCH 03/14] Fix duplicate key errors --- src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx | 3 ++- .../(main)/websites/[websiteId]/sessions/SessionsWeekly.tsx | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx b/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx index bb0225cc77..26c921e43c 100644 --- a/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx +++ b/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx @@ -71,9 +71,10 @@ export function RealtimeLog({ data }: { data: RealtimeData }) { if (__type === TYPE_EVENT) { return formatMessage(messages.eventLog, { - event: {eventName || formatMessage(labels.unknown)}, + event: {eventName || formatMessage(labels.unknown)}, url: ( {format(getDayOfWeekAsDate(index), 'EEE', { locale: dateLocale })} - {day?.map((hour: number) => { + {day?.map((hour: number, j) => { const pct = hour / max; return ( -
+
{hour > 0 && ( Date: Tue, 25 Feb 2025 13:17:53 +0100 Subject: [PATCH 04/14] Add keys for RealTime Session events --- .../(main)/websites/[websiteId]/realtime/RealtimeLog.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx b/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx index 26c921e43c..6a2b3c25c4 100644 --- a/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx +++ b/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx @@ -101,10 +101,10 @@ export function RealtimeLog({ data }: { data: RealtimeData }) { if (__type === TYPE_SESSION) { return formatMessage(messages.visitorLog, { - country: {countryNames[country] || formatMessage(labels.unknown)}, - browser: {BROWSERS[browser]}, - os: {OS_NAMES[os] || os}, - device: {formatMessage(labels[device] || labels.unknown)}, + country: {countryNames[country] || formatMessage(labels.unknown)}, + browser: {BROWSERS[browser]}, + os: {OS_NAMES[os] || os}, + device: {formatMessage(labels[device] || labels.unknown)}, }); } }; From 796f6d448c835e06c993a0a2d2d8f8396fc4a0b9 Mon Sep 17 00:00:00 2001 From: Shrutesh Date: Wed, 26 Feb 2025 11:49:33 +0530 Subject: [PATCH 05/14] Update batch.ts --- src/pages/api/batch.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pages/api/batch.ts b/src/pages/api/batch.ts index 93632d114a..0557d4da3f 100644 --- a/src/pages/api/batch.ts +++ b/src/pages/api/batch.ts @@ -22,7 +22,13 @@ export default async function handler(req, res) { const mockRes = { ...res, - end: () => {}, // Prevent premature response closure + status: (code) => { + res.status(code); + return mockRes; + }, + json: (data) => res.json(data), + setHeader: (key, value) => res.setHeader(key, value), + end: () => {}, }; await sendHandler(mockReq, mockRes); From 4c45285010e4ac9ff0bf2baa11ce6f07871e64c3 Mon Sep 17 00:00:00 2001 From: Harry Oosterveen Date: Thu, 27 Feb 2025 14:42:04 +0100 Subject: [PATCH 06/14] Fix https://github.com/umami-software/umami/issues/3255 --- src/lib/prisma.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index e2f50a6c72..c8286082e4 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -192,7 +192,9 @@ async function parseFilters( options: QueryOptions = {}, ) { const website = await fetchWebsite(websiteId); - const joinSession = Object.keys(filters).find(key => SESSION_COLUMNS.includes(key)); + const joinSession = Object.keys(filters).find(key => + ['referrer', ...SESSION_COLUMNS].includes(key), + ); return { joinSession: From 0d153a27dcab298fa5ec586543288d1b0b9ac6cb Mon Sep 17 00:00:00 2001 From: Louis Vallat Date: Sat, 1 Mar 2025 00:00:50 +0100 Subject: [PATCH 07/14] feat: add CORS headers to any value of COLLECT_API_ENDPOINT in addition to /api/* endpoints Signed-off-by: Louis Vallat --- next.config.js | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/next.config.js b/next.config.js index 7a65c47273..590d7121b5 100644 --- a/next.config.js +++ b/next.config.js @@ -59,15 +59,29 @@ const trackerHeaders = [ }, ]; +const apiHeaders = [ + { + key: 'Access-Control-Allow-Origin', + value: '*' + }, + { + key: 'Access-Control-Allow-Headers', + value: '*' + }, + { + key: 'Access-Control-Allow-Methods', + value: 'GET, DELETE, POST, PUT' + }, + { + key: 'Access-Control-Max-Age', + value: corsMaxAge || '86400' + }, +]; + const headers = [ { source: '/api/:path*', - headers: [ - { key: 'Access-Control-Allow-Origin', value: '*' }, - { key: 'Access-Control-Allow-Headers', value: '*' }, - { key: 'Access-Control-Allow-Methods', value: 'GET, DELETE, POST, PUT' }, - { key: 'Access-Control-Max-Age', value: corsMaxAge || '86400' }, - ], + headers: apiHeaders }, { source: '/:path*', @@ -89,6 +103,11 @@ if (trackerScriptURL) { } if (collectApiEndpoint) { + headers.push({ + source: collectApiEndpoint, + headers: apiHeaders, + }); + rewrites.push({ source: collectApiEndpoint, destination: '/api/send', From a8835f385e69fcb208feb8e0741697430cf84329 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 28 Feb 2025 16:58:57 -0800 Subject: [PATCH 08/14] Refactored batch route. --- src/app/api/batch/route.ts | 39 ++++++++++++++++++++++++++++++++++++ src/app/api/send/route.ts | 2 +- src/lib/request.ts | 4 ++-- src/pages/api/batch.ts | 41 -------------------------------------- 4 files changed, 42 insertions(+), 44 deletions(-) create mode 100644 src/app/api/batch/route.ts delete mode 100644 src/pages/api/batch.ts diff --git a/src/app/api/batch/route.ts b/src/app/api/batch/route.ts new file mode 100644 index 0000000000..87e04110d3 --- /dev/null +++ b/src/app/api/batch/route.ts @@ -0,0 +1,39 @@ +import { z } from 'zod'; +import * as send from '@/app/api/send/route'; +import { parseRequest } from '@/lib/request'; +import { json, serverError } from '@/lib/response'; + +const schema = z.array(z.object({}).passthrough()); + +export async function POST(request: Request) { + try { + const { body, error } = await parseRequest(request, schema, { skipAuth: true }); + + if (error) { + return error(); + } + + const errors = []; + + let index = 0; + for (const data of body) { + const newRequest = new Request(request, { body: JSON.stringify(data) }); + const response = await send.POST(newRequest); + + if (!response.ok) { + errors.push({ index, response: await response.json() }); + } + + index++; + } + + return json({ + size: body.length, + processed: body.length - errors.length, + errors: errors.length, + details: errors, + }); + } catch (e) { + return serverError(e); + } +} diff --git a/src/app/api/send/route.ts b/src/app/api/send/route.ts index 933ef78e22..80db8f96d5 100644 --- a/src/app/api/send/route.ts +++ b/src/app/api/send/route.ts @@ -21,7 +21,7 @@ const schema = z.object({ referrer: urlOrPathParam.optional(), screen: z.string().max(11).optional(), title: z.string().optional(), - url: urlOrPathParam, + url: urlOrPathParam.optional(), name: z.string().max(50).optional(), tag: z.string().max(50).optional(), ip: z.string().ip().optional(), diff --git a/src/lib/request.ts b/src/lib/request.ts index 9d32f89b37..0c71537ae8 100644 --- a/src/lib/request.ts +++ b/src/lib/request.ts @@ -1,4 +1,4 @@ -import { ZodObject } from 'zod'; +import { ZodSchema } from 'zod'; import { FILTER_COLUMNS } from '@/lib/constants'; import { badRequest, unauthorized } from '@/lib/response'; import { getAllowedUnits, getMinimumUnit } from '@/lib/date'; @@ -15,7 +15,7 @@ export async function getJsonBody(request: Request) { export async function parseRequest( request: Request, - schema?: ZodObject, + schema?: ZodSchema, options?: { skipAuth: boolean }, ): Promise { const url = new URL(request.url); diff --git a/src/pages/api/batch.ts b/src/pages/api/batch.ts deleted file mode 100644 index 0557d4da3f..0000000000 --- a/src/pages/api/batch.ts +++ /dev/null @@ -1,41 +0,0 @@ -import sendHandler from './send'; - -export default async function handler(req, res) { - if (req.method !== 'POST') { - res.setHeader('Allow', ['POST']); - return res.status(405).end(`Method ${req.method} Not Allowed`); - } - - const events = req.body; - - if (!Array.isArray(events)) { - return res.status(400).json({ error: 'Invalid payload, expected an array.' }); - } - - try { - for (const event of events) { - const mockReq = { - ...req, - body: event, - headers: { ...req.headers, origin: req.headers.origin || 'http://localhost:3000' }, - }; - - const mockRes = { - ...res, - status: (code) => { - res.status(code); - return mockRes; - }, - json: (data) => res.json(data), - setHeader: (key, value) => res.setHeader(key, value), - end: () => {}, - }; - - await sendHandler(mockReq, mockRes); - } - - return res.status(200).json({ success: true, message: `${events.length} events processed.` }); - } catch (error) { - return res.status(500).json({ error: 'Internal Server Error' }); - } -} From 05db1a8ba24b90ba9b3feb257c8e1ba23ec51d67 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 28 Feb 2025 17:37:56 -0800 Subject: [PATCH 09/14] Removed css rule. Fixes #3272 --- .eslintrc.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.json b/.eslintrc.json index 82f6a122d1..324e291cd7 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -33,6 +33,7 @@ "react/prop-types": "off", "import/no-anonymous-default-export": "off", "import/no-named-as-default": "off", + "css-modules/no-unused-class": "off", "@next/next/no-img-element": "off", "@typescript-eslint/no-empty-function": "off", "@typescript-eslint/no-explicit-any": "off", From 9a87442870dec6314586b9571030464fef2926b3 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 28 Feb 2025 21:04:53 -0800 Subject: [PATCH 10/14] Added SKIP_DB_MIGRATION var. --- scripts/check-db.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/check-db.js b/scripts/check-db.js index cdfeafa327..ca0fca31c9 100644 --- a/scripts/check-db.js +++ b/scripts/check-db.js @@ -82,9 +82,11 @@ async function checkV1Tables() { } async function applyMigration() { - console.log(execSync('prisma migrate deploy').toString()); + if (!process.env.SKIP_DB_MIGRATION) { + console.log(execSync('prisma migrate deploy').toString()); - success('Database is up to date.'); + success('Database is up to date.'); + } } (async () => { From 30b28793cf524ef50d29a4859e4c034c8f5f4b43 Mon Sep 17 00:00:00 2001 From: David Ventura Date: Mon, 6 Jan 2025 13:46:38 +0100 Subject: [PATCH 11/14] Allow populating event's createdAt on the send endpoint --- src/app/api/send/route.ts | 11 +++++++---- src/queries/sql/events/saveEvent.ts | 12 +++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/app/api/send/route.ts b/src/app/api/send/route.ts index 80db8f96d5..4189d7d91e 100644 --- a/src/app/api/send/route.ts +++ b/src/app/api/send/route.ts @@ -26,6 +26,7 @@ const schema = z.object({ tag: z.string().max(50).optional(), ip: z.string().ip().optional(), userAgent: z.string().optional(), + createdAt: yup.number().optional(), }), }); @@ -55,6 +56,7 @@ export async function POST(request: Request) { data, title, tag, + reqCreatedAt, } = payload; // Cache check @@ -119,14 +121,14 @@ export async function POST(request: Request) { } // Visit info - const now = Math.floor(new Date().getTime() / 1000); + const createdAt = Math.floor((reqCreatedAt || new Date()).getTime() / 1000); let visitId = cache?.visitId || uuid(sessionId, visitSalt()); - let iat = cache?.iat || now; + let iat = cache?.iat || createdAt; // Expire visit after 30 minutes - if (now - iat > 1800) { + if (createdAt - iat > 1800) { visitId = uuid(sessionId, visitSalt()); - iat = now; + iat = createdAt; } if (type === COLLECTION_TYPE.event) { @@ -179,6 +181,7 @@ export async function POST(request: Request) { subdivision2, city, tag, + createdAt, }); } diff --git a/src/queries/sql/events/saveEvent.ts b/src/queries/sql/events/saveEvent.ts index 65ee1175b8..3b3f3a99dd 100644 --- a/src/queries/sql/events/saveEvent.ts +++ b/src/queries/sql/events/saveEvent.ts @@ -29,6 +29,7 @@ export async function saveEvent(args: { subdivision2?: string; city?: string; tag?: string; + createdAt?: Date; }) { return runQuery({ [PRISMA]: () => relationalQuery(args), @@ -49,6 +50,7 @@ async function relationalQuery(data: { eventName?: string; eventData?: any; tag?: string; + createdAt?: Date; }) { const { websiteId, @@ -63,6 +65,7 @@ async function relationalQuery(data: { eventData, pageTitle, tag, + createdAt, } = data; const websiteEventId = uuid(); @@ -80,6 +83,7 @@ async function relationalQuery(data: { pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH), eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, + createdAt, tag, }, }); @@ -121,6 +125,7 @@ async function clickhouseQuery(data: { subdivision2?: string; city?: string; tag?: string; + createdAt?: Date; }) { const { websiteId, @@ -139,12 +144,13 @@ async function clickhouseQuery(data: { subdivision2, city, tag, + createdAt, ...args } = data; const { insert, getUTCString } = clickhouse; const { sendMessage } = kafka; const eventId = uuid(); - const createdAt = getUTCString(); + const createdAtUTC = getUTCString(createdAt); const message = { ...args, @@ -170,7 +176,7 @@ async function clickhouseQuery(data: { event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, tag: tag, - created_at: createdAt, + created_at: createdAtUTC, }; if (kafka.enabled) { @@ -187,7 +193,7 @@ async function clickhouseQuery(data: { urlPath: urlPath?.substring(0, URL_LENGTH), eventName: eventName?.substring(0, EVENT_NAME_LENGTH), eventData, - createdAt, + createdAt: createdAtUTC, }); } From 65f18d12ab91bb12870551a8bfbfcdaa467d5236 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Sat, 1 Mar 2025 14:40:37 -0800 Subject: [PATCH 12/14] Added timestamp property to payload. --- next-env.d.ts | 2 +- src/app/api/send/route.ts | 13 +++++++++---- src/lib/schema.ts | 2 ++ src/queries/sql/events/saveEvent.ts | 10 ++++++++-- src/queries/sql/events/saveEventData.ts | 10 ++++++---- src/queries/sql/sessions/saveSessionData.ts | 11 +++++++---- 6 files changed, 33 insertions(+), 15 deletions(-) diff --git a/next-env.d.ts b/next-env.d.ts index 40c3d68096..1b3be0840f 100644 --- a/next-env.d.ts +++ b/next-env.d.ts @@ -2,4 +2,4 @@ /// // NOTE: This file should not be edited -// see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information. +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/src/app/api/send/route.ts b/src/app/api/send/route.ts index 80db8f96d5..cc3b03cb98 100644 --- a/src/app/api/send/route.ts +++ b/src/app/api/send/route.ts @@ -7,16 +7,16 @@ import { badRequest, json, forbidden, serverError } from '@/lib/response'; import { fetchSession, fetchWebsite } from '@/lib/load'; import { getClientInfo, hasBlockedIp } from '@/lib/detect'; import { secret, uuid, visitSalt } from '@/lib/crypto'; -import { COLLECTION_TYPE, DOMAIN_REGEX } from '@/lib/constants'; +import { COLLECTION_TYPE } from '@/lib/constants'; +import { anyObjectParam, urlOrPathParam } from '@/lib/schema'; import { createSession, saveEvent, saveSessionData } from '@/queries'; -import { urlOrPathParam } from '@/lib/schema'; const schema = z.object({ type: z.enum(['event', 'identify']), payload: z.object({ website: z.string().uuid(), - data: z.object({}).passthrough().optional(), - hostname: z.string().regex(DOMAIN_REGEX).max(100).optional(), + data: anyObjectParam.optional(), + hostname: z.string().max(100).optional(), language: z.string().max(35).optional(), referrer: urlOrPathParam.optional(), screen: z.string().max(11).optional(), @@ -26,6 +26,7 @@ const schema = z.object({ tag: z.string().max(50).optional(), ip: z.string().ip().optional(), userAgent: z.string().optional(), + timestamp: z.coerce.number().int().optional(), }), }); @@ -55,6 +56,7 @@ export async function POST(request: Request) { data, title, tag, + timestamp, } = payload; // Cache check @@ -88,6 +90,7 @@ export async function POST(request: Request) { } const sessionId = uuid(websiteId, ip, userAgent); + const createdAt = timestamp ? new Date(timestamp * 1000) : new Date(); // Find session if (!clickhouse.enabled && !cache?.sessionId) { @@ -179,6 +182,7 @@ export async function POST(request: Request) { subdivision2, city, tag, + createdAt, }); } @@ -191,6 +195,7 @@ export async function POST(request: Request) { websiteId, sessionId, sessionData: data, + createdAt, }); } diff --git a/src/lib/schema.ts b/src/lib/schema.ts index 8df7be9fa7..4e2b3e4a34 100644 --- a/src/lib/schema.ts +++ b/src/lib/schema.ts @@ -36,6 +36,8 @@ export const unitParam = z.string().refine(value => UNIT_TYPES.includes(value), export const roleParam = z.enum(['team-member', 'team-view-only', 'team-manager']); +export const anyObjectParam = z.object({}).passthrough(); + export const urlOrPathParam = z.string().refine( value => { try { diff --git a/src/queries/sql/events/saveEvent.ts b/src/queries/sql/events/saveEvent.ts index 65ee1175b8..148b03f33b 100644 --- a/src/queries/sql/events/saveEvent.ts +++ b/src/queries/sql/events/saveEvent.ts @@ -29,6 +29,7 @@ export async function saveEvent(args: { subdivision2?: string; city?: string; tag?: string; + createdAt?: Date; }) { return runQuery({ [PRISMA]: () => relationalQuery(args), @@ -49,6 +50,7 @@ async function relationalQuery(data: { eventName?: string; eventData?: any; tag?: string; + createdAt?: Date; }) { const { websiteId, @@ -63,6 +65,7 @@ async function relationalQuery(data: { eventData, pageTitle, tag, + createdAt, } = data; const websiteEventId = uuid(); @@ -81,6 +84,7 @@ async function relationalQuery(data: { eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, tag, + createdAt, }, }); @@ -92,6 +96,7 @@ async function relationalQuery(data: { urlPath: urlPath?.substring(0, URL_LENGTH), eventName: eventName?.substring(0, EVENT_NAME_LENGTH), eventData, + createdAt, }); } @@ -121,6 +126,7 @@ async function clickhouseQuery(data: { subdivision2?: string; city?: string; tag?: string; + createdAt?: Date; }) { const { websiteId, @@ -139,12 +145,12 @@ async function clickhouseQuery(data: { subdivision2, city, tag, + createdAt, ...args } = data; const { insert, getUTCString } = clickhouse; const { sendMessage } = kafka; const eventId = uuid(); - const createdAt = getUTCString(); const message = { ...args, @@ -170,7 +176,7 @@ async function clickhouseQuery(data: { event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, tag: tag, - created_at: createdAt, + created_at: getUTCString(createdAt), }; if (kafka.enabled) { diff --git a/src/queries/sql/events/saveEventData.ts b/src/queries/sql/events/saveEventData.ts index 7c158da404..16a5cab107 100644 --- a/src/queries/sql/events/saveEventData.ts +++ b/src/queries/sql/events/saveEventData.ts @@ -15,7 +15,7 @@ export async function saveEventData(data: { urlPath?: string; eventName?: string; eventData: DynamicData; - createdAt?: string; + createdAt?: Date; }) { return runQuery({ [PRISMA]: () => relationalQuery(data), @@ -27,8 +27,9 @@ async function relationalQuery(data: { websiteId: string; eventId: string; eventData: DynamicData; + createdAt?: Date; }): Promise { - const { websiteId, eventId, eventData } = data; + const { websiteId, eventId, eventData, createdAt } = data; const jsonKeys = flattenJSON(eventData); @@ -42,6 +43,7 @@ async function relationalQuery(data: { numberValue: a.dataType === DATA_TYPE.number ? a.value : null, dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dataType, + createdAt, })); return prisma.client.eventData.createMany({ @@ -56,7 +58,7 @@ async function clickhouseQuery(data: { urlPath?: string; eventName?: string; eventData: DynamicData; - createdAt?: string; + createdAt?: Date; }) { const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data; @@ -77,7 +79,7 @@ async function clickhouseQuery(data: { string_value: getStringValue(value, dataType), number_value: dataType === DATA_TYPE.number ? value : null, date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null, - created_at: createdAt, + created_at: getUTCString(createdAt), }; }); diff --git a/src/queries/sql/sessions/saveSessionData.ts b/src/queries/sql/sessions/saveSessionData.ts index 35f0c71263..a060e9a848 100644 --- a/src/queries/sql/sessions/saveSessionData.ts +++ b/src/queries/sql/sessions/saveSessionData.ts @@ -11,6 +11,7 @@ export async function saveSessionData(data: { websiteId: string; sessionId: string; sessionData: DynamicData; + createdAt?: Date; }) { return runQuery({ [PRISMA]: () => relationalQuery(data), @@ -22,9 +23,10 @@ export async function relationalQuery(data: { websiteId: string; sessionId: string; sessionData: DynamicData; + createdAt?: Date; }) { const { client } = prisma; - const { websiteId, sessionId, sessionData } = data; + const { websiteId, sessionId, sessionData, createdAt } = data; const jsonKeys = flattenJSON(sessionData); @@ -37,6 +39,7 @@ export async function relationalQuery(data: { numberValue: a.dataType === DATA_TYPE.number ? a.value : null, dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dataType, + createdAt, })); const existing = await client.sessionData.findMany({ @@ -77,12 +80,12 @@ async function clickhouseQuery(data: { websiteId: string; sessionId: string; sessionData: DynamicData; + createdAt?: Date; }) { - const { websiteId, sessionId, sessionData } = data; + const { websiteId, sessionId, sessionData, createdAt } = data; const { insert, getUTCString } = clickhouse; const { sendMessage } = kafka; - const createdAt = getUTCString(); const jsonKeys = flattenJSON(sessionData); @@ -95,7 +98,7 @@ async function clickhouseQuery(data: { string_value: getStringValue(value, dataType), number_value: dataType === DATA_TYPE.number ? value : null, date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null, - created_at: createdAt, + created_at: getUTCString(createdAt), }; }); From 925c7562153a0ef656cf5199e892e8d4e7616804 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Sat, 1 Mar 2025 16:29:35 -0800 Subject: [PATCH 13/14] Updated salt methods. --- next-env.d.ts | 2 +- src/app/api/send/route.ts | 25 +++++++++++++++---------- src/lib/crypto.ts | 15 +-------------- src/queries/sql/events/saveEvent.ts | 3 +-- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/next-env.d.ts b/next-env.d.ts index 1b3be0840f..40c3d68096 100644 --- a/next-env.d.ts +++ b/next-env.d.ts @@ -2,4 +2,4 @@ /// // NOTE: This file should not be edited -// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. +// see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information. diff --git a/src/app/api/send/route.ts b/src/app/api/send/route.ts index b9556ddd6c..8519a73e1b 100644 --- a/src/app/api/send/route.ts +++ b/src/app/api/send/route.ts @@ -1,12 +1,13 @@ import { z } from 'zod'; import { isbot } from 'isbot'; -import { createToken, parseToken } from '@/lib/jwt'; +import { startOfHour, startOfMonth } from 'date-fns'; import clickhouse from '@/lib/clickhouse'; import { parseRequest } from '@/lib/request'; import { badRequest, json, forbidden, serverError } from '@/lib/response'; import { fetchSession, fetchWebsite } from '@/lib/load'; import { getClientInfo, hasBlockedIp } from '@/lib/detect'; -import { secret, uuid, visitSalt } from '@/lib/crypto'; +import { createToken, parseToken } from '@/lib/jwt'; +import { secret, uuid, hash } from '@/lib/crypto'; import { COLLECTION_TYPE } from '@/lib/constants'; import { anyObjectParam, urlOrPathParam } from '@/lib/schema'; import { createSession, saveEvent, saveSessionData } from '@/queries'; @@ -89,8 +90,13 @@ export async function POST(request: Request) { return forbidden(); } - const sessionId = uuid(websiteId, ip, userAgent); const createdAt = timestamp ? new Date(timestamp * 1000) : new Date(); + const now = Math.floor(new Date().getTime() / 1000); + + const sessionSalt = hash(startOfMonth(createdAt).toUTCString()); + const visitSalt = hash(startOfHour(createdAt).toUTCString()); + + const sessionId = uuid(websiteId, ip, userAgent, sessionSalt); // Find session if (!clickhouse.enabled && !cache?.sessionId) { @@ -122,14 +128,13 @@ export async function POST(request: Request) { } // Visit info - const createdAt = Math.floor((reqCreatedAt || new Date()).getTime() / 1000); - let visitId = cache?.visitId || uuid(sessionId, visitSalt()); - let iat = cache?.iat || createdAt; + let visitId = cache?.visitId || uuid(sessionId, visitSalt); + let iat = cache?.iat || now; // Expire visit after 30 minutes - if (createdAt - iat > 1800) { - visitId = uuid(sessionId, visitSalt()); - iat = createdAt; + if (!timestamp && now - iat > 1800) { + visitId = uuid(sessionId, visitSalt); + iat = now; } if (type === COLLECTION_TYPE.event) { @@ -201,7 +206,7 @@ export async function POST(request: Request) { const token = createToken({ websiteId, sessionId, visitId, iat }, secret()); - return json({ cache: token }); + return json({ cache: token, sessionId, visitId }); } catch (e) { return serverError(e); } diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index a4ff3a526f..d22bad091d 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -1,5 +1,4 @@ import crypto from 'crypto'; -import { startOfHour, startOfMonth } from 'date-fns'; import prand from 'pure-rand'; import { v4, v5 } from 'uuid'; @@ -77,20 +76,8 @@ export function secret() { return hash(process.env.APP_SECRET || process.env.DATABASE_URL); } -export function salt() { - const ROTATING_SALT = hash(startOfMonth(new Date()).toUTCString()); - - return hash(secret(), ROTATING_SALT); -} - -export function visitSalt() { - const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString()); - - return hash(secret(), ROTATING_SALT); -} - export function uuid(...args: any) { if (!args.length) return v4(); - return v5(hash(...args, salt()), v5.DNS); + return v5(hash(...args, secret()), v5.DNS); } diff --git a/src/queries/sql/events/saveEvent.ts b/src/queries/sql/events/saveEvent.ts index 5df276e18a..148b03f33b 100644 --- a/src/queries/sql/events/saveEvent.ts +++ b/src/queries/sql/events/saveEvent.ts @@ -83,7 +83,6 @@ async function relationalQuery(data: { pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH), eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, - createdAt, tag, createdAt, }, @@ -194,7 +193,7 @@ async function clickhouseQuery(data: { urlPath: urlPath?.substring(0, URL_LENGTH), eventName: eventName?.substring(0, EVENT_NAME_LENGTH), eventData, - createdAt: createdAtUTC, + createdAt, }); } From cb7eef200cb90a2447319558ecf2dfbeb4f3e718 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Sat, 1 Mar 2025 17:18:46 -0800 Subject: [PATCH 14/14] Added check for do not track to tracker. --- src/tracker/index.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/tracker/index.js b/src/tracker/index.js index dbd47b7c61..c423a66b53 100644 --- a/src/tracker/index.js +++ b/src/tracker/index.js @@ -1,11 +1,12 @@ (window => { const { screen: { width, height }, - navigator: { language }, + navigator: { language, doNotTrack: ndnt, msDoNotTrack: msdnt }, location, document, history, top, + doNotTrack, } = window; const { hostname, href, origin } = location; const { currentScript, referrer } = document; @@ -21,6 +22,7 @@ const hostUrl = attr(_data + 'host-url'); const tag = attr(_data + 'tag'); const autoTrack = attr(_data + 'auto-track') !== _false; + const dnt = attr(_data + 'do-not-track') === _true; const excludeSearch = attr(_data + 'exclude-search') === _true; const excludeHash = attr(_data + 'exclude-hash') === _true; const domain = attr(_data + 'domains') || ''; @@ -46,6 +48,11 @@ tag: tag ? tag : undefined, }); + const hasDoNotTrack = () => { + const dnt = doNotTrack || ndnt || msdnt; + return dnt === 1 || dnt === '1' || dnt === 'yes'; + }; + /* Event handlers */ const handlePush = (state, title, url) => { @@ -182,7 +189,8 @@ disabled || !website || (localStorage && localStorage.getItem('umami.disabled')) || - (domain && !domains.includes(hostname)); + (domain && !domains.includes(hostname)) || + (dnt && hasDoNotTrack()); const send = async (payload, type = 'event') => { if (trackingDisabled()) return;