Merge branch 'dev' of https://github.com/umami-software/umami into dev

.eslintrc.json

@@ -33,6 +33,7 @@
     "react/prop-types": "off",
     "import/no-anonymous-default-export": "off",
     "import/no-named-as-default": "off",
+    "css-modules/no-unused-class": "off",
     "@next/next/no-img-element": "off",
     "@typescript-eslint/no-empty-function": "off",
     "@typescript-eslint/no-explicit-any": "off",

next.config.js

@@ -59,15 +59,29 @@ const trackerHeaders = [
   },
 ];
 
+const apiHeaders = [
+  {
+    key: 'Access-Control-Allow-Origin',
+    value: '*'
+  },
+  {
+    key: 'Access-Control-Allow-Headers',
+    value: '*'
+  },
+  {
+    key: 'Access-Control-Allow-Methods',
+    value: 'GET, DELETE, POST, PUT'
+  },
+  {
+    key: 'Access-Control-Max-Age',
+    value: corsMaxAge || '86400'
+  },
+];
+
 const headers = [
   {
     source: '/api/:path*',
-    headers: [
-      { key: 'Access-Control-Allow-Origin', value: '*' },
-      { key: 'Access-Control-Allow-Headers', value: '*' },
-      { key: 'Access-Control-Allow-Methods', value: 'GET, DELETE, POST, PUT' },
-      { key: 'Access-Control-Max-Age', value: corsMaxAge || '86400' },
-    ],
+    headers: apiHeaders
   },
   {
     source: '/:path*',
@@ -89,6 +103,11 @@ if (trackerScriptURL) {
 }
 
 if (collectApiEndpoint) {
+  headers.push({
+    source: collectApiEndpoint,
+    headers: apiHeaders,
+  });
+
   rewrites.push({
     source: collectApiEndpoint,
     destination: '/api/send',

scripts/check-db.js

@@ -82,9 +82,11 @@ async function checkV1Tables() {
 }
 
 async function applyMigration() {
-  console.log(execSync('prisma migrate deploy').toString());
+  if (!process.env.SKIP_DB_MIGRATION) {
+    console.log(execSync('prisma migrate deploy').toString());
 
-  success('Database is up to date.');
+    success('Database is up to date.');
+  }
 }
 
 (async () => {

src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx

@@ -71,9 +71,10 @@ export function RealtimeLog({ data }: { data: RealtimeData }) {
 
     if (__type === TYPE_EVENT) {
       return formatMessage(messages.eventLog, {
-        event: <b>{eventName || formatMessage(labels.unknown)}</b>,
+        event: <b key="b">{eventName || formatMessage(labels.unknown)}</b>,
         url: (
           <a
+            key="a"
             href={`//${website?.domain}${url}`}
             className={styles.link}
             target="_blank"
@@ -100,10 +101,10 @@ export function RealtimeLog({ data }: { data: RealtimeData }) {
 
     if (__type === TYPE_SESSION) {
       return formatMessage(messages.visitorLog, {
-        country: <b>{countryNames[country] || formatMessage(labels.unknown)}</b>,
-        browser: <b>{BROWSERS[browser]}</b>,
-        os: <b>{OS_NAMES[os] || os}</b>,
-        device: <b>{formatMessage(labels[device] || labels.unknown)}</b>,
+        country: <b key="country">{countryNames[country] || formatMessage(labels.unknown)}</b>,
+        browser: <b key="browser">{BROWSERS[browser]}</b>,
+        os: <b key="os">{OS_NAMES[os] || os}</b>,
+        device: <b key="device">{formatMessage(labels[device] || labels.unknown)}</b>,
       });
     }
   };

src/app/(main)/websites/[websiteId]/sessions/SessionsWeekly.tsx

@@ -62,10 +62,10 @@ export function SessionsWeekly({ websiteId }: { websiteId: string }) {
                 <div className={styles.header}>
                   {format(getDayOfWeekAsDate(index), 'EEE', { locale: dateLocale })}
                 </div>
-                {day?.map((hour: number) => {
+                {day?.map((hour: number, j) => {
                   const pct = hour / max;
                   return (
-                    <div key={hour} className={classNames(styles.cell)}>
+                    <div key={j} className={classNames(styles.cell)}>
                       {hour > 0 && (
                         <TooltipPopup
                           label={`${formatMessage(labels.visitors)}: ${hour}`}

src/app/api/batch/route.ts

@@ -0,0 +1,39 @@
+import { z } from 'zod';
+import * as send from '@/app/api/send/route';
+import { parseRequest } from '@/lib/request';
+import { json, serverError } from '@/lib/response';
+
+const schema = z.array(z.object({}).passthrough());
+
+export async function POST(request: Request) {
+  try {
+    const { body, error } = await parseRequest(request, schema, { skipAuth: true });
+
+    if (error) {
+      return error();
+    }
+
+    const errors = [];
+
+    let index = 0;
+    for (const data of body) {
+      const newRequest = new Request(request, { body: JSON.stringify(data) });
+      const response = await send.POST(newRequest);
+
+      if (!response.ok) {
+        errors.push({ index, response: await response.json() });
+      }
+
+      index++;
+    }
+
+    return json({
+      size: body.length,
+      processed: body.length - errors.length,
+      errors: errors.length,
+      details: errors,
+    });
+  } catch (e) {
+    return serverError(e);
+  }
+}

src/app/api/send/route.ts

@@ -1,31 +1,33 @@
 import { z } from 'zod';
 import { isbot } from 'isbot';
-import { createToken, parseToken } from '@/lib/jwt';
+import { startOfHour, startOfMonth } from 'date-fns';
 import clickhouse from '@/lib/clickhouse';
 import { parseRequest } from '@/lib/request';
 import { badRequest, json, forbidden, serverError } from '@/lib/response';
 import { fetchSession, fetchWebsite } from '@/lib/load';
 import { getClientInfo, hasBlockedIp } from '@/lib/detect';
-import { secret, uuid, visitSalt } from '@/lib/crypto';
-import { COLLECTION_TYPE, DOMAIN_REGEX } from '@/lib/constants';
+import { createToken, parseToken } from '@/lib/jwt';
+import { secret, uuid, hash } from '@/lib/crypto';
+import { COLLECTION_TYPE } from '@/lib/constants';
+import { anyObjectParam, urlOrPathParam } from '@/lib/schema';
 import { createSession, saveEvent, saveSessionData } from '@/queries';
-import { urlOrPathParam } from '@/lib/schema';
 
 const schema = z.object({
   type: z.enum(['event', 'identify']),
   payload: z.object({
     website: z.string().uuid(),
-    data: z.object({}).passthrough().optional(),
-    hostname: z.string().regex(DOMAIN_REGEX).max(100).optional(),
+    data: anyObjectParam.optional(),
+    hostname: z.string().max(100).optional(),
     language: z.string().max(35).optional(),
     referrer: urlOrPathParam.optional(),
     screen: z.string().max(11).optional(),
     title: z.string().optional(),
-    url: urlOrPathParam,
+    url: urlOrPathParam.optional(),
     name: z.string().max(50).optional(),
     tag: z.string().max(50).optional(),
     ip: z.string().ip().optional(),
     userAgent: z.string().optional(),
+    timestamp: z.coerce.number().int().optional(),
   }),
 });
 
@@ -55,6 +57,7 @@ export async function POST(request: Request) {
       data,
       title,
       tag,
+      timestamp,
     } = payload;
 
     // Cache check
@@ -87,7 +90,13 @@ export async function POST(request: Request) {
       return forbidden();
     }
 
-    const sessionId = uuid(websiteId, ip, userAgent);
+    const createdAt = timestamp ? new Date(timestamp * 1000) : new Date();
+    const now = Math.floor(new Date().getTime() / 1000);
+
+    const sessionSalt = hash(startOfMonth(createdAt).toUTCString());
+    const visitSalt = hash(startOfHour(createdAt).toUTCString());
+
+    const sessionId = uuid(websiteId, ip, userAgent, sessionSalt);
 
     // Find session
     if (!clickhouse.enabled && !cache?.sessionId) {
@@ -119,13 +128,12 @@ export async function POST(request: Request) {
     }
 
     // Visit info
-    const now = Math.floor(new Date().getTime() / 1000);
-    let visitId = cache?.visitId || uuid(sessionId, visitSalt());
+    let visitId = cache?.visitId || uuid(sessionId, visitSalt);
     let iat = cache?.iat || now;
 
     // Expire visit after 30 minutes
-    if (now - iat > 1800) {
-      visitId = uuid(sessionId, visitSalt());
+    if (!timestamp && now - iat > 1800) {
+      visitId = uuid(sessionId, visitSalt);
       iat = now;
     }
 
@@ -179,6 +187,7 @@ export async function POST(request: Request) {
         subdivision2,
         city,
         tag,
+        createdAt,
       });
     }
 
@@ -191,12 +200,13 @@ export async function POST(request: Request) {
         websiteId,
         sessionId,
         sessionData: data,
+        createdAt,
       });
     }
 
     const token = createToken({ websiteId, sessionId, visitId, iat }, secret());
 
-    return json({ cache: token });
+    return json({ cache: token, sessionId, visitId });
   } catch (e) {
     return serverError(e);
   }

src/app/api/users/route.ts

@@ -8,6 +8,7 @@ import { createUser, getUserByUsername } from '@/queries';
 
 export async function POST(request: Request) {
   const schema = z.object({
+    id: z.string().uuid().optional(),
     username: z.string().max(255),
     password: z.string(),
     role: z.string().regex(/admin|user|view-only/i),
@@ -23,7 +24,7 @@ export async function POST(request: Request) {
     return unauthorized();
   }
 
-  const { username, password, role } = body;
+  const { id, username, password, role } = body;
 
   const existingUser = await getUserByUsername(username, { showDeleted: true });
 
@@ -32,7 +33,7 @@ export async function POST(request: Request) {
   }
 
   const user = await createUser({
-    id: uuid(),
+    id: id || uuid(),
     username,
     password: hashPassword(password),
     role: role ?? ROLES.user,

src/lib/crypto.ts

@@ -1,5 +1,4 @@
 import crypto from 'crypto';
-import { startOfHour, startOfMonth } from 'date-fns';
 import prand from 'pure-rand';
 import { v4, v5 } from 'uuid';
 
@@ -77,20 +76,8 @@ export function secret() {
   return hash(process.env.APP_SECRET || process.env.DATABASE_URL);
 }
 
-export function salt() {
-  const ROTATING_SALT = hash(startOfMonth(new Date()).toUTCString());
-
-  return hash(secret(), ROTATING_SALT);
-}
-
-export function visitSalt() {
-  const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString());
-
-  return hash(secret(), ROTATING_SALT);
-}
-
 export function uuid(...args: any) {
   if (!args.length) return v4();
 
-  return v5(hash(...args, salt()), v5.DNS);
+  return v5(hash(...args, secret()), v5.DNS);
 }

src/lib/prisma.ts

@@ -192,7 +192,9 @@ async function parseFilters(
   options: QueryOptions = {},
 ) {
   const website = await fetchWebsite(websiteId);
-  const joinSession = Object.keys(filters).find(key => SESSION_COLUMNS.includes(key));
+  const joinSession = Object.keys(filters).find(key =>
+    ['referrer', ...SESSION_COLUMNS].includes(key),
+  );
 
   return {
     joinSession:

src/lib/request.ts

@@ -1,4 +1,4 @@
-import { ZodObject } from 'zod';
+import { ZodSchema } from 'zod';
 import { FILTER_COLUMNS } from '@/lib/constants';
 import { badRequest, unauthorized } from '@/lib/response';
 import { getAllowedUnits, getMinimumUnit } from '@/lib/date';
@@ -15,7 +15,7 @@ export async function getJsonBody(request: Request) {
 
 export async function parseRequest(
   request: Request,
-  schema?: ZodObject<any>,
+  schema?: ZodSchema,
   options?: { skipAuth: boolean },
 ): Promise<any> {
   const url = new URL(request.url);

src/lib/schema.ts

@@ -36,6 +36,8 @@ export const unitParam = z.string().refine(value => UNIT_TYPES.includes(value),
 
 export const roleParam = z.enum(['team-member', 'team-view-only', 'team-manager']);
 
+export const anyObjectParam = z.object({}).passthrough();
+
 export const urlOrPathParam = z.string().refine(
   value => {
     try {