From e7a074fd5a53bfa64d04968ea543279a0b3aa901 Mon Sep 17 00:00:00 2001 From: Diljit Date: Tue, 30 Jul 2024 08:47:21 +0530 Subject: [PATCH] chore: Add page load trace with resource, paint and navigation spans (#34957) --- .../UITelemetry/PageLoadInstrumentation.ts | 361 ++++++++++++++++++ app/client/src/UITelemetry/auto-otel-web.ts | 21 +- app/client/src/UITelemetry/generateTraces.ts | 10 +- app/client/src/index.tsx | 15 +- 4 files changed, 400 insertions(+), 7 deletions(-) create mode 100644 app/client/src/UITelemetry/PageLoadInstrumentation.ts diff --git a/app/client/src/UITelemetry/PageLoadInstrumentation.ts b/app/client/src/UITelemetry/PageLoadInstrumentation.ts new file mode 100644 index 00000000000..917baece3b7 --- /dev/null +++ b/app/client/src/UITelemetry/PageLoadInstrumentation.ts @@ -0,0 +1,361 @@ +import type { Span } from "@opentelemetry/api"; +import { InstrumentationBase } from "@opentelemetry/instrumentation"; +import { startRootSpan, startNestedSpan } from "./generateTraces"; +import { onLCP, onFCP } from "web-vitals/attribution"; +import type { + LCPMetricWithAttribution, + FCPMetricWithAttribution, + NavigationTimingPolyfillEntry, +} from "web-vitals"; + +export class PageLoadInstrumentation extends InstrumentationBase { + // PerformanceObserver to observe resource timings + resourceTimingObserver: PerformanceObserver | null = null; + // Root span for the page load instrumentation + rootSpan: Span; + // List of resource URLs to ignore + ignoreResourceUrls: string[] = []; + // Timestamp when the page was last hidden + pageLastHiddenAt: number = 0; + // Duration the page was hidden for + pageHiddenFor: number = 0; + // Flag to check if navigation entry was pushed + wasNavigationEntryPushed: boolean = false; + // Set to keep track of resource entries + resourceEntriesSet: Set = new Set(); + // Timeout for polling resource entries + resourceEntryPollTimeout: number | null = null; + + constructor({ ignoreResourceUrls = [] }: { ignoreResourceUrls?: string[] }) { + // Initialize the base instrumentation with the name and version + super("appsmith-page-load-instrumentation", "1.0.0", { + enabled: true, + }); + this.ignoreResourceUrls = ignoreResourceUrls; + // Start the root span for the page load + this.rootSpan = startRootSpan("PAGE_LOAD", {}, 0); + } + + init() { + // init method is present in the base class and needs to be implemented + // This is method is never called by the OpenTelemetry SDK + // Leaving it empty as it is done by other OpenTelemetry instrumentation classes + } + + enable(): void { + this.addVisibilityChangeListener(); + + // Listen for LCP and FCP events + // reportAllChanges: true will report all LCP and FCP events + // binding the context to the class to access class properties + onLCP(this.onLCPReport.bind(this), { reportAllChanges: true }); + onFCP(this.onFCPReport.bind(this), { reportAllChanges: true }); + + // Check if PerformanceObserver is available + if (PerformanceObserver) { + this.observeResourceTimings(); + } else { + // If PerformanceObserver is not available, fallback to polling + this.pollResourceTimingEntries(); + } + } + + private addVisibilityChangeListener() { + // Listen for page visibility changes to track time spent on hidden page + document.addEventListener("visibilitychange", () => { + if (document.visibilityState === "hidden") { + this.pageLastHiddenAt = performance.now(); + } else { + const endTime = performance.now(); + this.pageHiddenFor = endTime - this.pageLastHiddenAt; + } + }); + } + + // Handler for LCP report + private onLCPReport(metric: LCPMetricWithAttribution) { + const { + attribution: { lcpEntry }, + } = metric; + + if (lcpEntry) { + this.pushLcpTimingToSpan(lcpEntry); + } + } + + // Handler for FCP report + private onFCPReport(metric: FCPMetricWithAttribution) { + const { + attribution: { fcpEntry, navigationEntry }, + } = metric; + + // Push navigation entry only once + // This is to avoid pushing multiple navigation entries + if (navigationEntry && !this.wasNavigationEntryPushed) { + this.pushNavigationTimingToSpan(navigationEntry); + this.wasNavigationEntryPushed = true; + } + + if (fcpEntry) { + this.pushPaintTimingToSpan(fcpEntry); + } + } + + private getElementName(element?: Element | null, depth = 0): string { + // Limit the depth to 3 to avoid long element names + if (!element || depth > 3) { + return ""; + } + + const elementTestId = element.getAttribute("data-testid"); + const className = element.className + ? "." + element.className.split(" ").join(".") + : ""; + const elementId = element.id ? `#${element.id}` : ""; + + const elementName = `${element.tagName}${elementId}${className}:${elementTestId}`; + + // Recursively get the parent element names + const parentElementName = this.getElementName( + element.parentElement, + depth + 1, + ); + + return `${parentElementName} > ${elementName}`; + } + + // Convert kebab-case to SCREAMING_SNAKE_CASE + private kebabToScreamingSnakeCase(str: string) { + return str.replace(/-/g, "_").toUpperCase(); + } + + // Push paint timing to span + private pushPaintTimingToSpan(entry: PerformanceEntry) { + const paintSpan = startNestedSpan( + this.kebabToScreamingSnakeCase(entry.name), + this.rootSpan, + {}, + 0, + ); + + paintSpan.end(entry.startTime); + } + + // Push LCP timing to span + private pushLcpTimingToSpan(entry: LargestContentfulPaint) { + const { element, entryType, loadTime, renderTime, startTime, url } = entry; + + const lcpSpan = startNestedSpan( + this.kebabToScreamingSnakeCase(entryType), + this.rootSpan, + { + url, + renderTime, + element: this.getElementName(element), + entryType, + loadTime, + pageHiddenFor: this.pageHiddenFor, + }, + 0, + ); + + lcpSpan.end(startTime); + } + + // Push navigation timing to span + private pushNavigationTimingToSpan( + entry: PerformanceNavigationTiming | NavigationTimingPolyfillEntry, + ) { + const { + connectEnd, + connectStart, + domainLookupEnd, + domainLookupStart, + domComplete, + domContentLoadedEventEnd, + domContentLoadedEventStart, + domInteractive, + entryType, + fetchStart, + loadEventEnd, + loadEventStart, + name: url, + redirectEnd, + redirectStart, + requestStart, + responseEnd, + responseStart, + secureConnectionStart, + startTime: navigationStartTime, + type: navigationType, + unloadEventEnd, + unloadEventStart, + workerStart, + } = entry; + + this.rootSpan.setAttributes({ + connectEnd, + connectStart, + decodedBodySize: + (entry as PerformanceNavigationTiming).decodedBodySize || 0, + domComplete, + domContentLoadedEventEnd, + domContentLoadedEventStart, + domInteractive, + domainLookupEnd, + domainLookupStart, + encodedBodySize: + (entry as PerformanceNavigationTiming).encodedBodySize || 0, + entryType, + fetchStart, + initiatorType: + (entry as PerformanceNavigationTiming).initiatorType || "navigation", + loadEventEnd, + loadEventStart, + nextHopProtocol: + (entry as PerformanceNavigationTiming).nextHopProtocol || "", + redirectCount: (entry as PerformanceNavigationTiming).redirectCount || 0, + redirectEnd, + redirectStart, + requestStart, + responseEnd, + responseStart, + secureConnectionStart, + navigationStartTime, + transferSize: (entry as PerformanceNavigationTiming).transferSize || 0, + navigationType, + url, + unloadEventEnd, + unloadEventStart, + workerStart, + }); + + this.rootSpan?.end(entry.domContentLoadedEventEnd); + } + + // Observe resource timings using PerformanceObserver + private observeResourceTimings() { + this.resourceTimingObserver = new PerformanceObserver((list) => { + const entries = list.getEntries() as PerformanceResourceTiming[]; + const resources = this.getResourcesToTrack(entries); + resources.forEach((entry) => { + this.pushResourceTimingToSpan(entry); + }); + }); + + this.resourceTimingObserver.observe({ + type: "resource", + buffered: true, + }); + } + + // Filter out resources to track based on ignoreResourceUrls + private getResourcesToTrack(resources: PerformanceResourceTiming[]) { + return resources.filter(({ name }) => { + return !this.ignoreResourceUrls.some((ignoreUrl) => + name.includes(ignoreUrl), + ); + }); + } + + // Push resource timing to span + private pushResourceTimingToSpan(entry: PerformanceResourceTiming) { + const { + connectEnd, + connectStart, + decodedBodySize, + domainLookupEnd, + domainLookupStart, + duration: resourceDuration, + encodedBodySize, + entryType, + fetchStart, + initiatorType, + name: url, + nextHopProtocol, + redirectEnd, + redirectStart, + requestStart, + responseEnd, + responseStart, + secureConnectionStart, + transferSize, + workerStart, + } = entry; + + const resourceSpan = startNestedSpan( + entry.name, + this.rootSpan, + { + connectEnd, + connectStart, + decodedBodySize, + domainLookupEnd, + domainLookupStart, + encodedBodySize, + entryType, + fetchStart, + firstInterimResponseStart: (entry as any).firstInterimResponseStart, + initiatorType, + nextHopProtocol, + redirectEnd, + redirectStart, + requestStart, + responseEnd, + responseStart, + resourceDuration, + secureConnectionStart, + transferSize, + url, + workerStart, + renderBlockingStatus: (entry as any).renderBlockingStatus, + }, + entry.startTime, + ); + + resourceSpan.end(entry.startTime + entry.responseEnd); + } + + // Get unique key for a resource entry + private getResourceEntryKey(entry: PerformanceResourceTiming) { + return `${entry.name}:${entry.startTime}:${entry.entryType}`; + } + + // Poll resource timing entries periodically + private pollResourceTimingEntries() { + // Clear the previous timeout + if (this.resourceEntryPollTimeout) { + clearInterval(this.resourceEntryPollTimeout); + } + + const resources = performance.getEntriesByType( + "resource", + ) as PerformanceResourceTiming[]; + + const filteredResources = this.getResourcesToTrack(resources); + + filteredResources.forEach((entry) => { + const key = this.getResourceEntryKey(entry); + if (!this.resourceEntriesSet.has(key)) { + this.pushResourceTimingToSpan(entry); + this.resourceEntriesSet.add(key); + } + }); + + // Poll every 5 seconds + this.resourceEntryPollTimeout = setTimeout( + this.pollResourceTimingEntries, + 5000, + ); + } + + disable(): void { + if (this.resourceTimingObserver) { + this.resourceTimingObserver.disconnect(); + } + + if (this.rootSpan) { + this.rootSpan.end(); + } + } +} diff --git a/app/client/src/UITelemetry/auto-otel-web.ts b/app/client/src/UITelemetry/auto-otel-web.ts index 389e5e04fdd..fca494e321d 100644 --- a/app/client/src/UITelemetry/auto-otel-web.ts +++ b/app/client/src/UITelemetry/auto-otel-web.ts @@ -20,14 +20,21 @@ import { } from "@opentelemetry/exporter-metrics-otlp-http"; import type { Context, TextMapSetter } from "@opentelemetry/api"; import { metrics } from "@opentelemetry/api"; +import { registerInstrumentations } from "@opentelemetry/instrumentation"; +import { PageLoadInstrumentation } from "./PageLoadInstrumentation"; enum CompressionAlgorithm { NONE = "none", GZIP = "gzip", } const { newRelic } = getAppsmithConfigs(); -const { applicationId, otlpEndpoint, otlpLicenseKey, otlpServiceName } = - newRelic; +const { + applicationId, + browserAgentEndpoint, + otlpEndpoint, + otlpLicenseKey, + otlpServiceName, +} = newRelic; const tracerProvider = new WebTracerProvider({ resource: new Resource({ @@ -112,3 +119,13 @@ const meterProvider = new MeterProvider({ // Register the MeterProvider globally metrics.setGlobalMeterProvider(meterProvider); + +registerInstrumentations({ + tracerProvider, + meterProvider, + instrumentations: [ + new PageLoadInstrumentation({ + ignoreResourceUrls: [browserAgentEndpoint, otlpEndpoint], + }), + ], +}); diff --git a/app/client/src/UITelemetry/generateTraces.ts b/app/client/src/UITelemetry/generateTraces.ts index 776d132aec2..1f261284890 100644 --- a/app/client/src/UITelemetry/generateTraces.ts +++ b/app/client/src/UITelemetry/generateTraces.ts @@ -7,10 +7,10 @@ import type { import { SpanKind } from "@opentelemetry/api"; import { context } from "@opentelemetry/api"; import { trace } from "@opentelemetry/api"; -import { deviceType } from "react-device-detect"; - +import { deviceType, browserName, browserVersion } from "react-device-detect"; import { APP_MODE } from "entities/App"; import { matchBuilderPath, matchViewerPath } from "constants/routes"; +import nanoid from "nanoid"; import memoizeOne from "memoize-one"; const GENERATOR_TRACE = "generator-tracer"; @@ -18,6 +18,8 @@ const GENERATOR_TRACE = "generator-tracer"; export type OtlpSpan = Span; export type SpanAttributes = Attributes; +const OTLP_SESSION_ID = nanoid(); + const getAppMode = memoizeOne((pathname: string) => { const isEditorUrl = matchBuilderPath(pathname); const isViewerUrl = matchViewerPath(pathname); @@ -29,6 +31,7 @@ const getAppMode = memoizeOne((pathname: string) => { : ""; return appMode; }); + const getCommonTelemetryAttributes = () => { const pathname = window.location.pathname; const appMode = getAppMode(pathname); @@ -36,6 +39,9 @@ const getCommonTelemetryAttributes = () => { return { appMode, deviceType, + browserName, + browserVersion, + otlpSessionId: OTLP_SESSION_ID, }; }; diff --git a/app/client/src/index.tsx b/app/client/src/index.tsx index 30891d5c3fc..28cc0dbbedf 100755 --- a/app/client/src/index.tsx +++ b/app/client/src/index.tsx @@ -24,7 +24,9 @@ import { setAutoFreeze } from "immer"; import AppErrorBoundary from "./AppErrorBoundry"; import log from "loglevel"; import { getAppsmithConfigs } from "@appsmith/configs"; -import { BrowserAgent } from "@newrelic/browser-agent/loaders/browser-agent"; +import { PageViewTiming } from "@newrelic/browser-agent/features/page_view_timing"; +import { PageViewEvent } from "@newrelic/browser-agent/features/page_view_event"; +import { Agent } from "@newrelic/browser-agent/loaders/agent"; const { newRelic } = getAppsmithConfigs(); const { enableNewRelic } = newRelic; @@ -33,7 +35,6 @@ const newRelicBrowserAgentConfig = { init: { distributed_tracing: { enabled: true }, privacy: { cookies_enabled: true }, - ajax: { deny_list: [newRelic.browserAgentEndpoint] }, }, info: { beacon: newRelic.browserAgentEndpoint, @@ -53,7 +54,15 @@ const newRelicBrowserAgentConfig = { // The agent loader code executes immediately on instantiation. if (enableNewRelic) { - new BrowserAgent(newRelicBrowserAgentConfig); + new Agent( + { + ...newRelicBrowserAgentConfig, + features: [PageViewTiming, PageViewEvent], + }, + // The second argument agentIdentifier is not marked as optional in its type definition. + // Passing a null value throws an error as well. So we pass undefined. + undefined, + ); } const shouldAutoFreeze = process.env.NODE_ENV === "development";