Release build 5.3.0 [ci release]

Author: shakyShane

Sources/ContentScopeScripts/dist/contentScope.js

@@ -1,4 +1,9 @@
-import { getElement, getElementMatches, getElements } from '../utils.js' // Assuming you have imported the address comparison function
+import {
+    cleanArray,
+    getElement,
+    getElementMatches,
+    getElements
+} from '../utils.js' // Assuming you have imported the address comparison function
 import { ErrorResponse, ProfileResult, SuccessResponse } from '../types.js'
 import { isSameAge } from '../comparisons/is-same-age.js'
 import { isSameName } from '../comparisons/is-same-name.js'
@@ -72,7 +77,12 @@ export function extractProfiles (action, userData, root = document) {
 
     return {
         results: profilesElementList.map((element) => {
-            const scrapedData = createProfile(element, action.profile)
+            const elementFactory = (key, value) => {
+                return value?.findElements
+                    ? cleanArray(getElements(element, value.selector))
+                    : cleanArray(getElement(element, value.selector) || getElementMatches(element, value.selector))
+            }
+            const scrapedData = createProfile(elementFactory, action.profile)
             const { result, score, matchedFields } = scrapedDataMatchesUserData(userData, scrapedData)
             return new ProfileResult({
                 scrapedData,
@@ -105,37 +115,40 @@ export function extractProfiles (action, userData, root = document) {
  *   "profileUrl": "https://example.com/1234"
  * }
  *
- * @param {HTMLElement} profileElement
+ * @param {(key: string, value: ExtractProfileProperty) => {innerText: string}[]} elementFactory
+ *   a function that produces elements for a given key + ExtractProfileProperty
  * @param {Record<string, ExtractProfileProperty>} extractData
  * @return {Record<string, any>}
  */
-function createProfile (profileElement, extractData) {
+export function createProfile (elementFactory, extractData) {
     const output = {}
     for (const [key, value] of Object.entries(extractData)) {
         if (!value?.selector) {
             output[key] = null
         } else {
+            const elements = elementFactory(key, value)
             const evaluatedValue = value?.findElements
-                ? findFromElements(profileElement, key, value)
-                : findFromElement(profileElement, key, value)
+                ? findFromElements(elements, key, value)
+                : findFromElement(elements[0], key, value)
 
             // Note: This can return a string, string[], or null
             const extractedValue = extractValue(key, value, evaluatedValue)
 
-            // try to use the extracted value first, then the originally evaluated, falling back to null
-            output[key] = extractedValue || evaluatedValue || null
+            // try to use the extracted value, or fall back to null
+            // this allows 'extractValue' to return null|undefined
+            output[key] = extractedValue || null
         }
     }
     return output
 }
 
 /**
- * @param {HTMLElement} profileElement
+ * @param {{innerText: string}[]} elements
  * @param {string} key
  * @param {ExtractProfileProperty} extractField
+ * @return {string[]}
  */
-function findFromElements (profileElement, key, extractField) {
-    const elements = getElements(profileElement, extractField.selector) || null
+function findFromElements (elements, key, extractField) {
     const elementValues = []
     if (elements) {
         for (const element of elements) {
@@ -153,15 +166,12 @@ function findFromElements (profileElement, key, extractField) {
 }
 
 /**
- * @param {HTMLElement} profileElement
+ * @param {{innerText: string}} element
  * @param {string} dataKey - such as 'name', 'age' etc
  * @param {ExtractProfileProperty} extractField
  * @return {string}
  */
-function findFromElement (profileElement, dataKey, extractField) {
-    const element = getElement(profileElement, extractField.selector) ||
-        getElementMatches(profileElement, extractField.selector)
-
+function findFromElement (element, dataKey, extractField) {
     // todo: should we use textContent here?
     let elementValue = rules[dataKey]?.(element) ?? element?.innerText ?? null
 
@@ -282,7 +292,7 @@ export function aggregateFields (profile) {
  * @param {string} key
  * @param {ExtractProfileProperty} value
  * @param {string | string[]} elementValue
- * @return {string|string[]|null}
+ * @return {string|string[]|null|undefined}
  */
 export function extractValue (key, value, elementValue) {
     if (!elementValue) return null

Sources/ContentScopeScripts/dist/contentScopeIsolated.js

@@ -161,3 +161,13 @@ function safeQuerySelectorAllXpath (element, selector) {
 export function generateRandomInt (min, max) {
     return Math.floor(Math.random() * (max - min + 1) + min)
 }
+
+/**
+ * Flatten create an array of any input, removing nulls, undefined and empty strings
+ * @template T
+ * @param {T | T[] | null | undefined} input
+ * @return {NonNullable<T>[]}
+ */
+export function cleanArray (input) {
+    return [].concat(/** @type {any} */(input)).flat().filter(Boolean)
+}

build/android/contentScope.js

@@ -0,0 +1,102 @@
+import { createProfile } from '../src/features/broker-protection/actions/extract.js'
+
+describe('create profiles from extracted data', () => {
+    it('handles combined, single strings', () => {
+        const selectors = {
+            name: {
+                selector: '.name',
+                beforeText: ','
+            },
+            age: {
+                selector: '.name',
+                afterText: ','
+            }
+        }
+
+        const elementExamples = [
+            { text: 'John smith, 39', expected: { name: 'John smith', age: '39' } },
+            { text: 'John smith  , 39', expected: { name: 'John smith', age: '39' } },
+            { text: 'John\nsmith  , 39', expected: { name: 'John smith', age: '39' } },
+            { text: 'John\nsmith  , 39 anything', expected: { name: 'John smith', age: '39' } },
+            { text: '   John smith, 39', expected: { name: 'John smith', age: '39' } },
+            { text: '   John smith,39', expected: { name: 'John smith', age: '39' } },
+            { text: '   John smith, ~39', expected: { name: 'John smith', age: '39' } },
+            { text: 'Jane Doe, 45', expected: { name: 'Jane Doe', age: '45' } },
+            { text: '   Robert Johnson,22', expected: { name: 'Robert Johnson', age: '22' } },
+            { text: 'Emily Wilson , 29 ', expected: { name: 'Emily Wilson', age: '29' } },
+            { text: 'Daniel Miller\n\n, 56', expected: { name: 'Daniel Miller', age: '56' } },
+            { text: 'Henry White   ,46', expected: { name: 'Henry White', age: '46' } },
+            { text: 'Linda Brown ~ , 35', expected: { name: 'Linda Brown ~', age: '35' } },
+            { text: 'Emma Green, 30 any', expected: { name: 'Emma Green', age: '30' } },
+            { text: '  Sophia Blue, 26', expected: { name: 'Sophia Blue', age: '26' } },
+
+            // examples where age is absent, so the result should be null
+            { text: 'John smith', expected: { name: 'John smith', age: null } },
+            { text: 'John smith   ,   ', expected: { name: 'John smith', age: null } },
+            { text: 'John smith   \n,\n   ', expected: { name: 'John smith', age: null } }
+        ]
+
+        for (const elementExample of elementExamples) {
+            const elementFactory = () => {
+                return [{ innerText: elementExample.text }]
+            }
+            const profile = createProfile(elementFactory, selectors)
+            expect(profile).toEqual(elementExample.expected)
+        }
+    })
+    it('handles multiple strings', () => {
+        const elementExamples = [
+            {
+                selectors: {
+                    alternativeNamesList: {
+                        selector: 'example',
+                        afterText: 'Also Known as:',
+                        separator: ','
+                    }
+                },
+                elements: [{ innerText: 'Also Known as: John Smith, Jon Smith' }],
+                expected: {
+                    alternativeNamesList: ['John Smith', 'Jon Smith']
+                }
+            },
+            {
+                selectors: {
+                    alternativeNamesList: {
+                        selector: 'example',
+                        findElements: true,
+                        afterText: 'Also Known as:'
+                    }
+                },
+                elements: [
+                    { innerText: 'Also Known as: John Smith' },
+                    { innerText: 'Jon Smith' }
+                ],
+                expected: {
+                    alternativeNamesList: ['John Smith', 'Jon Smith']
+                }
+            },
+            {
+                selectors: {
+                    alternativeNamesList: {
+                        selector: 'example',
+                        findElements: true,
+                        beforeText: ', '
+                    }
+                },
+                elements: [
+                    { innerText: 'John Smith, 89' },
+                    { innerText: 'Jon Smith, 78' }
+                ],
+                expected: {
+                    alternativeNamesList: ['John Smith', 'Jon Smith']
+                }
+            }
+        ]
+
+        for (const elementExample of elementExamples) {
+            const elementFactory = () => elementExample.elements
+            const profile = createProfile(elementFactory, elementExample.selectors)
+            expect(profile).toEqual(elementExample.expected)
+        }
+    })
+})