O11y Artifact changes

packages/wdio-browserstack-service/README.md

@@ -28,8 +28,8 @@ export const config = {
     key: process.env.BROWSERSTACK_ACCESS_KEY,
     services: [
         ['browserstack', {
-            testObservability: true,
-            testObservabilityOptions: {
+            testReporting: true,
+            testReportingOptions: {
                 projectName: "Your project name goes here",
                 buildName: "The static build job name goes here e.g. Nightly regression"
             },
@@ -44,15 +44,15 @@ export const config = {
 
 In order to authorize to the BrowserStack service your config needs to contain a [`user`](https://webdriver.io/docs/options#user) and [`key`](https://webdriver.io/docs/options#key) option.
 
-### testObservability
+### testReporting
 
 Test Observability is an advanced test reporting tool that gives insights to improve your automation tests and helps you debug faster. It’s enabled by default by setting the `testObservability`​ flag as `true` for all users of browserstack-service. You can disable this by setting the `testObservability`​ flag to `false`.
 
-Once your tests finish running, you can visit [Test Observability](https://observability.browserstack.com/) to debug your builds with additional insights like Unique Error Analysis, Automatic Flaky Test Detection, and more.
+Once your tests finish running, you can visit [Test Reporting and Analytics](https://automation.browserstack.com/) to debug your builds with additional insights like Unique Error Analysis, Automatic Flaky Test Detection, and more.
 
 You can use Test Observability even if you don’t run your tests on the BrowserStack infrastructure. Even if you run your tests on a CI, a local machine, or even on other cloud service providers, Test Observability can still generate intelligent test reports and advanced analytics on your tests.
 
-If you want to use Test Observability without running your tests on BrowserStack infrastructure, you can set your config as follows:
+If you want to use Test Reporting and Analytics without running your tests on BrowserStack infrastructure, you can set your config as follows:
 
 
 ```js
@@ -61,8 +61,8 @@ export const config = {
     // ...
     services: [
         ['browserstack', {
-            testObservability: true,
-            testObservabilityOptions: {
+            testReporting: true,
+            testReportingOptions: {
                 user: process.env.BROWSERSTACK_USERNAME,
                 key: process.env.BROWSERSTACK_ACCESS_KEY,
                 projectName: "Your project name goes here",
@@ -74,7 +74,7 @@ export const config = {
 };
 ```
 
-You can explore all the features of Test Observability in [this sandbox](https://observability-demo.browserstack.com/) or read more about it [here](https://www.browserstack.com/docs/test-observability/overview/what-is-test-observability).
+You can explore all the features of Test Reporting and Analytics in [this sandbox](https://automation.browserstack.com/) or read more about it [here](https://www.browserstack.com/docs/test-reporting-and-analytics/overview/what-is-test-observability).
 
 ### browserstackLocal
 Set this to true to enable routing connections from BrowserStack cloud through your computer.

packages/wdio-browserstack-service/src/cleanup.ts

@@ -43,24 +43,24 @@ export default class BStackCleanup {
         if (!process.env[BROWSERSTACK_TESTHUB_JWT]) {
             return
         }
-        BStackLogger.debug('Executing observability cleanup')
+        BStackLogger.debug('Executing Test Reporting and Analytics cleanup')
         try {
             const result = await stopBuildUpstream()
-            if (process.env[BROWSERSTACK_OBSERVABILITY] && process.env[BROWSERSTACK_TESTHUB_UUID]) {
-                BStackLogger.info(`\nVisit https://observability.browserstack.com/builds/${process.env[BROWSERSTACK_TESTHUB_UUID]} to view build report, insights, and many more debugging information all at one place!\n`)
+            if ((process.env[BROWSERSTACK_OBSERVABILITY]) && process.env[BROWSERSTACK_TESTHUB_UUID]) {
+                BStackLogger.info(`\nVisit https://automation.browserstack.com/builds/${process.env[BROWSERSTACK_TESTHUB_UUID]} to view build report, insights, and many more debugging information all at one place!\n`)
             }
             const status = (result && result.status) || 'failed'
             const message = (result && result.message)
             this.updateO11yStopData(funnelData, status, status === 'failed' ? message : undefined)
         } catch (e: unknown) {
-            BStackLogger.error('Error in stopping Observability build: ' + e)
+            BStackLogger.error('Error in stopping Test Reporting and Analytics build: ' + e)
             this.updateO11yStopData(funnelData, 'failed', e)
         }
     }
 
     static updateO11yStopData(funnelData: FunnelData, status: string, error: unknown = undefined) {
         const toData = funnelData?.event_properties?.productUsage?.testObservability
-        // Return if no O11y data in funnel data
+        // Return if no Test Reporting and Analytics data in funnel data
         if (!toData) {
             return
         }

packages/wdio-browserstack-service/src/constants.ts

@@ -117,9 +117,16 @@ export const BROWSERSTACK_PERCY = 'BROWSERSTACK_PERCY'
 // Whether session is a accessibility session
 export const BROWSERSTACK_ACCESSIBILITY = 'BROWSERSTACK_ACCESSIBILITY'
 
-// Whether session is a observability session
+// Whether session is a test reporting session
 export const BROWSERSTACK_OBSERVABILITY = 'BROWSERSTACK_OBSERVABILITY'
 
+// New Test Reporting and Analytics environment variables
+export const BROWSERSTACK_TEST_REPORTING = 'BROWSERSTACK_TEST_REPORTING'
+export const BROWSERSTACK_TEST_REPORTING_DEBUG = 'BROWSERSTACK_TEST_REPORTING_DEBUG'
+export const TEST_REPORTING_BUILD_TAG = 'TEST_REPORTING_BUILD_TAG'
+export const TEST_REPORTING_PROJECT_NAME = 'TEST_REPORTING_PROJECT_NAME'
+export const TEST_REPORTING_BUILD_NAME = 'TEST_REPORTING_BUILD_NAME'
+
 // Maximum size of VCS info which is allowed
 export const MAX_GIT_META_DATA_SIZE_IN_BYTES = 64 * 1024
 

packages/wdio-browserstack-service/src/crash-reporter.ts

@@ -136,6 +136,8 @@ export default class CrashReporter {
                     for (let idx = 1; idx < serviceArray.length; idx++) {
                         this.deletePIIKeysFromObject(serviceArray[idx])
                         if (serviceArray[idx]) {
+                            // Handle both new testReportingOptions and legacy testObservabilityOptions
+                            this.deletePIIKeysFromObject(serviceArray[idx].testReportingOptions)
                             this.deletePIIKeysFromObject(serviceArray[idx].testObservabilityOptions)
                         }
                     }

packages/wdio-browserstack-service/src/insights-handler.ts

@@ -101,7 +101,7 @@ class _InsightsHandler {
             await (this._browser as WebdriverIO.Browser).executeScript(`browserstack_executor: ${JSON.stringify({
                 action: 'annotate',
                 arguments: {
-                    data: `ObservabilitySync:${Date.now()}`,
+                    data: `TestReportingSync:${Date.now()}`,
                     level: 'debug'
                 }
             })}`, [])
@@ -529,7 +529,7 @@ class _InsightsHandler {
                 this.listener.logCreated([stdLog])
             }
         } catch (error) {
-            BStackLogger.debug(`Exception in uploading log data to Observability with error : ${error}`)
+            BStackLogger.debug(`Exception in uploading log data to Test Reporting and Analytics with error : ${error}`)
         }
     }
 

packages/wdio-browserstack-service/src/launcher.ts

@@ -22,7 +22,9 @@ import {
     BROWSERSTACK_TESTHUB_UUID,
     VALID_APP_EXTENSION,
     BROWSERSTACK_PERCY,
-    BROWSERSTACK_OBSERVABILITY
+    BROWSERSTACK_OBSERVABILITY,
+    BROWSERSTACK_TEST_REPORTING,
+    TEST_REPORTING_PROJECT_NAME
 } from './constants.js'
 import {
     launchTestSession,
@@ -83,6 +85,32 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
         if (!this._config) {
             this._config = _options
         }
+
+        //normalizing testReporting config and env variables
+        if (!isUndefined(_options.testReporting)){
+            _options.testObservability = _options.testReporting
+        }
+
+        if (!isUndefined(_options.testReportingOptions)){
+            _options.testObservabilityOptions = _options.testReportingOptions
+        }
+
+        if (!isUndefined(process.env[BROWSERSTACK_TEST_REPORTING])){
+            process.env[BROWSERSTACK_OBSERVABILITY] = process.env[BROWSERSTACK_TEST_REPORTING]
+        }
+
+        if (!isUndefined(process.env[TEST_REPORTING_PROJECT_NAME])){
+            process.env.TEST_OBSERVABILITY_PROJECT_NAME = process.env[TEST_REPORTING_PROJECT_NAME]
+        }
+
+        if (!isUndefined(process.env.TEST_REPORTING_BUILD_NAME)) {
+            process.env.TEST_OBSERVABILITY_BUILD_NAME = process.env.TEST_REPORTING_BUILD_NAME
+        }
+
+        if (!isUndefined(process.env.TEST_REPORTING_BUILD_TAG)) {
+            process.env.TEST_OBSERVABILITY_BUILD_TAG = process.env.TEST_REPORTING_BUILD_TAG
+        }
+
         this.browserStackConfig = BrowserStackConfig.getInstance(_options, _config)
         if (Array.isArray(capabilities)) {
             capabilities
@@ -113,7 +141,7 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
                             }
                         }
 
-                        // Need this details for sending data to Observability
+                        // Need this details for sending data to Test Reporting and Analytics
                         this._buildIdentifier = capability['browserstack.buildIdentifier']?.toString()
                         // @ts-expect-error ToDo: fix invalid cap
                         this._buildName = capability.build?.toString()
@@ -174,7 +202,7 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
         }
         this._options.accessibility = this._accessibilityAutomation as boolean
 
-        // by default observability will be true unless specified as false
+        // Default is true unless explicitly set to false
         this._options.testObservability = this._options.testObservability !== false
 
         if (this._options.testObservability
@@ -405,8 +433,9 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
 
         BStackLogger.debug('Sending stop launch event')
         await stopBuildUpstream()
-        if (process.env[BROWSERSTACK_OBSERVABILITY] && process.env[BROWSERSTACK_TESTHUB_UUID]) {
-            console.log(`\nVisit https://observability.browserstack.com/builds/${process.env[BROWSERSTACK_TESTHUB_UUID]} to view build report, insights, and many more debugging information all at one place!\n`)
+
+        if ((process.env[BROWSERSTACK_OBSERVABILITY]) && process.env[BROWSERSTACK_TESTHUB_UUID]) {
+            console.log(`\nVisit https://automation.browserstack.com/builds/${process.env[BROWSERSTACK_TESTHUB_UUID]} to view build report, insights, and many more debugging information all at one place!\n`)
         }
         this.browserStackConfig.testObservability.buildStopped = true
 
@@ -441,7 +470,9 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
         }
 
         if (this._options.forcedStop) {
-            return process.kill(this.browserstackLocal.pid as number)
+            const pid = this.browserstackLocal.pid as number
+            process.kill(pid)
+            return pid
         }
 
         let timer: NodeJS.Timeout

packages/wdio-browserstack-service/src/request-handler.ts

@@ -29,7 +29,7 @@ export default class RequestQueueHandler {
 
     add (event: UploadType) {
         if (!process.env[TESTOPS_BUILD_COMPLETED_ENV]) {
-            throw new Error('Observability build start not completed yet.')
+            throw new Error('Test Reporting and Analytics build start not completed yet.')
         }
 
         this.queue.push(event)

packages/wdio-browserstack-service/src/testHub/utils.ts

@@ -38,7 +38,7 @@ export const shouldProcessEventForTesthub = (eventType: string): boolean => {
 
 export const handleErrorForObservability = (error: Errors | null): void => {
     process.env[BROWSERSTACK_OBSERVABILITY] = 'false'
-    logBuildError(error, 'observability')
+    logBuildError(error, 'Test Reporting and Analytics')
 }
 
 export const handleErrorForAccessibility = (error: Errors | null): void => {

packages/wdio-browserstack-service/src/types.ts

@@ -36,6 +36,15 @@ export interface TestObservabilityOptions {
     ignoreHooksStatus?: boolean
 }
 
+// New interface for Test Reporting and Analytics (same structure as TestObservabilityOptions for backward compatibility)
+export interface TestReportingOptions {
+    buildName?: string,
+    projectName?: string,
+    buildTag?: string[],
+    user?: string,
+    key?: string
+}
+
 export interface BrowserstackOptions extends Options.Testrunner {
     selfHeal?: boolean;
 }
@@ -49,17 +58,31 @@ export interface BrowserstackConfig {
      */
     buildIdentifier?: string;
     /**
-     * Set this to true to enable BrowserStack Test Observability which will collect test related data
+     * Set this to true to enable BrowserStack Test Reporting and Analytics which will collect test related data
      * (name, hierarchy, status, error stack trace, file name and hierarchy), test commands, etc.
-     * and show all the data in a meaningful manner in BrowserStack Test Observability dashboards for faster test debugging and better insights.
+     * and show all the data in a meaningful manner in BrowserStack Test Reporting and Analytics dashboards for faster test debugging and better insights.
      * @default true
+     * @deprecated Use testReporting instead
      */
     testObservability?: boolean;
     /**
-     * Set the Test Observability related config options under this key.
+     * Set this to true to enable BrowserStack Test Reporting and Analytics which will collect test related data
+     * (name, hierarchy, status, error stack trace, file name and hierarchy), test commands, etc.
+     * and show all the data in a meaningful manner in BrowserStack Test Reporting and Analytics dashboards for faster test debugging and better insights.
+     * @default true
+     */
+    testReporting?: boolean;
+    /**
+     * Set the Test Reporting and Analytics related config options under this key.
      * For e.g. buildName, projectName, BrowserStack access credentials, etc.
+     * @deprecated Use testReportingOptions instead
      */
     testObservabilityOptions?: TestObservabilityOptions;
+    /**
+     * Set the Test Reporting and Analytics related config options under this key.
+     * For e.g. buildName, projectName, BrowserStack access credentials, etc.
+     */
+    testReportingOptions?: TestReportingOptions;
     /**
      * Set this to true to enable BrowserStack Percy which will take screenshots
      * and snapshots for your tests run on Browserstack
@@ -169,7 +192,7 @@ export interface BrowserstackConfig {
 }
 
 /**
- * Observability types
+ * Test Reporting and Analytics types
  */
 export interface PlatformMeta {
     sessionId?: string,

packages/wdio-browserstack-service/src/util.ts

@@ -41,7 +41,7 @@ import {
     GIT_META_DATA_TRUNCATED,
     APP_ALLY_ENDPOINT,
     APP_ALLY_ISSUES_SUMMARY_ENDPOINT,
-    APP_ALLY_ISSUES_ENDPOINT,
+    APP_ALLY_ISSUES_ENDPOINT
 } from './constants.js'
 import CrashReporter from './crash-reporter.js'
 import { BStackLogger } from './bstackLogger.js'
@@ -1211,7 +1211,7 @@ export async function batchAndPostEvents (eventUrl: string, kind: string, data:
         })
         BStackLogger.debug(`[${kind}] Success response: ${JSON.stringify(await response.json())}`)
     } catch (error) {
-        BStackLogger.debug(`[${kind}] EXCEPTION IN ${kind} REQUEST TO TEST OBSERVABILITY : ${error}`)
+        BStackLogger.debug(`[${kind}] EXCEPTION IN ${kind} REQUEST TO TEST REPORTING AND ANALYTICS : ${error}`)
         throw new Error('Exception in request ' + error)
     }
 }

packages/wdio-browserstack-service/tests/cleanup.test.ts

@@ -41,7 +41,7 @@ describe('BStackCleanup', () => {
     })
 
     describe('startCleanup', () => {
-        it('executes observability cleanup if --observability is present in argv', async () => {
+        it('executes test reporting cleanup if --observability is present in argv', async () => {
             process.argv.push('--observability', '--funnelData')
             process.env[BROWSERSTACK_TESTHUB_JWT] = 'some jwt'
 
@@ -66,13 +66,13 @@ describe('BStackCleanup', () => {
         })
     })
 
-    describe('executeObservabilityCleanup', () => {
-        it('does not invoke stop call for observability when jwt is not set', async () => {
+    describe('executeTestReportingCleanup (legacy executeObservabilityCleanup)', () => {
+        it('does not invoke stop call for test reporting when jwt is not set', async () => {
             await BStackCleanup.executeObservabilityCleanup({} as any)
             expect(stopBuildUpstream).toBeCalledTimes(0)
         })
 
-        it('invoke stop call for observability when jwt is set', async () => {
+        it('invoke stop call for test reporting when jwt is set', async () => {
             process.env[BROWSERSTACK_TESTHUB_JWT] = 'jwtToken'
             await BStackCleanup.executeObservabilityCleanup({} as any)
             expect(stopBuildUpstream).toBeCalledTimes(1)

packages/wdio-browserstack-service/tests/crash-reporter.test.ts

@@ -83,7 +83,43 @@ describe('CrashReporter', () => {
             })
         })
 
-        it('should delete user key from testObservabilityOptions', () => {
+        it('should delete user key from testReportingOptions and legacy testObservabilityOptions', () => {
+            const userConfig = {
+                'framework': 'some framework',
+                'user': 'some user',
+                'key': 'key',
+                capabilities: {},
+                services: [
+                    ['browserstack', {
+                        testObservabilityOptions: {
+                            user: 'username',
+                            key: 'access-key',
+                        },
+                        testReportingOptions: {
+                            user: 'username',
+                            key: 'access-key',
+                        },
+                    }]
+                ]
+            }
+
+            // @ts-ignore
+            const filteredConfig = CrashReporter.filterPII(userConfig)
+            expect(filteredConfig).toEqual({
+                'framework': 'some framework',
+                capabilities: {},
+                services: [
+                    ['browserstack', {
+                        testObservabilityOptions: {
+                        },
+                        testReportingOptions: {
+                        },
+                    }]
+                ]
+            })
+        })
+
+        it('should delete user key from legacy testObservabilityOptions only', () => {
             const userConfig = {
                 'framework': 'some framework',
                 'user': 'some user',

packages/wdio-browserstack-service/tests/launcher.test.ts

@@ -645,6 +645,11 @@ describe('onComplete', () => {
     it('should reject with an error, if local.stop throws an error', () => {
         const service = new BrowserstackLauncher({} as any, [{ browserName: '' }] as any, {} as any)
         service.browserstackLocal = new Local()
+
+        // Ensure isRunning returns true for this instance
+        const BrowserstackLocalIsRunningSpy = vi.spyOn(service.browserstackLocal, 'isRunning')
+        BrowserstackLocalIsRunningSpy.mockImplementationOnce(() => true)
+
         const BrowserstackLocalStopSpy = vi.spyOn(service.browserstackLocal, 'stop')
         BrowserstackLocalStopSpy.mockImplementationOnce((cb) => cb(error))
         return expect(service.onComplete()).rejects.toThrow(error)

packages/wdio-browserstack-service/tests/request-handler.test.ts

@@ -28,7 +28,7 @@ describe('RequestQueueHandler', () => {
     describe('add', () => {
         it('throw error if BS_TESTOPS_BUILD_COMPLETED not present', () => {
             delete process.env[TESTOPS_BUILD_COMPLETED_ENV]
-            expect(() => requestQueueHandler.add({ event_type: 'there' })).toThrowError(/Observability build start not completed yet/)
+            expect(() => requestQueueHandler.add({ event_type: 'there' })).toThrowError(/Test Reporting and Analytics build start not completed yet/)
         })
 
         it('if event_type in BATCH_EVENT_TYPES', () => {