Merge branch 'main' into chronic-leech

HTTPArchive · Feb 16, 2025 · fbe02da · fbe02da
2 parents 561e5bd + 1caf76d
commit fbe02da
Show file tree

Hide file tree

Showing 7 changed files with 80 additions and 12 deletions.
diff --git a/definitions/declarations/chrome-ux-report.js b/definitions/declarations/chrome-ux-report.js
@@ -30,9 +30,3 @@ FROM ${ctx.ref(database, 'materialized', 'device_summary')}
 |> WHERE cnt_devices != 3 OR cnt_ranks != 10
 |> SELECT "Table data doesn't match 3 unique devices and 10 ranks" AS error_message
 `)
-
-declare({
-  database,
-  schema: 'experimental',
-  name: 'global'
-})
diff --git a/definitions/declarations/httparchive.js b/definitions/declarations/httparchive.js
@@ -40,3 +40,21 @@ ORDER BY cnt_pages DESC
     name: table
   })
 )
+
+operate('create_reservation_assignment')
+  .tags(['crawl_complete'])
+  .queries(ctx => `
+CREATE ASSIGNMENT
+\`httparchive.region-us.retrospective-reprocessing.pipeline\`
+OPTIONS (
+  assignee = 'projects/httparchive',
+  job_type = 'QUERY')
+`)
+
+operate('drop_reservation_assignment')
+  .dependencies(['requests_10k'])
+  .tags(['crawl_complete'])
+  .queries(ctx => `
+DROP ASSIGNMENT IF EXISTS
+\`httparchive.region-us.retrospective-reprocessing.pipeline\`
+`)
diff --git a/definitions/output/core_web_vitals/technology_matrix.js b/definitions/output/core_web_vitals/technology_matrix.js
@@ -0,0 +1,55 @@
+publish('technology_matrix', {
+  type: 'table',
+  schema: 'core_web_vitals',
+  bigquery: {
+    partitionBy: 'date',
+    clusterBy: ['client', 't1']
+  },
+  description: 'Used in: https://lookerstudio.google.com/u/2/reporting/0ad64c8f-644a-40f9-93e4-0ccd5f72b33d',
+  tags: ['crawl_complete']
+}).query(ctx => `
+WITH a AS (
+  SELECT
+    date,
+    client,
+    root_page,
+    technology AS t1,
+    ARRAY_TO_STRING(ARRAY(SELECT category FROM UNNEST(categories) AS category ORDER BY category), ', ') AS c1
+  FROM ${ctx.ref('crawl', 'pages')},
+    UNNEST(technologies)
+  WHERE
+    date = '${constants.currentMonth}' AND
+    is_root_page
+), b AS (
+  SELECT
+    client,
+    root_page,
+    technology AS t2,
+    ARRAY_TO_STRING(ARRAY(SELECT category FROM UNNEST(categories) AS category ORDER BY category), ', ') AS c2
+  FROM ${ctx.ref('crawl', 'pages')},
+    UNNEST(technologies)
+  WHERE
+    date = '${constants.currentMonth}' AND
+    is_root_page
+)
+
+
+SELECT
+  date,
+  client,
+  t1,
+  c1,
+  t2,
+  c2,
+  COUNT(DISTINCT root_page) AS pages
+FROM a
+LEFT JOIN b
+USING (client, root_page)
+GROUP BY
+  date,
+  client,
+  t1,
+  c1,
+  t2,
+  c2
+`)
diff --git a/definitions/output/crawl/pages.js b/definitions/output/crawl/pages.js
@@ -47,7 +47,8 @@ publish('pages', {
     technologies: 'Technologies detected at runtime (see https://www.wappalyzer.com/)',
     metadata: 'Additional metadata about the test'
   },
-  tags: ['crawl_complete']
+  tags: ['crawl_complete'],
+  dependencies: ['create_reservation_assignment']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}' AND

diff --git a/definitions/output/crawl/parsed_css.js b/definitions/output/crawl/parsed_css.js
@@ -7,7 +7,8 @@ publish('parsed_css', {
     clusterBy: ['client', 'is_root_page', 'rank', 'page'],
     requirePartitionFilter: true
   },
-  tags: ['crawl_complete']
+  tags: ['crawl_complete'],
+  dependencies: ['create_reservation_assignment']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}'

diff --git a/definitions/output/crawl/requests.js b/definitions/output/crawl/requests.js
@@ -36,7 +36,8 @@ publish('requests', {
     },
     response_body: 'Text-based response body'
   },
-  tags: ['crawl_complete']
+  tags: ['crawl_complete'],
+  dependencies: ['create_reservation_assignment']
 }).preOps(ctx => `
 FOR client_value IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS client) DO
   FOR is_root_page_value IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS is_root_page) DO

diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js
@@ -15,8 +15,7 @@ WITH pages AS (
   INNER JOIN pages.technologies AS tech
   WHERE
     date = '${pastMonth}'
-    ${constants.devRankFilter} AND
-    tech.technology IS NOT NULL
+    ${constants.devRankFilter}
 ),
 
 tech_origins AS (
@@ -78,7 +77,6 @@ SELECT
   NULL AS description,
   NULL AS category,
   NULL AS category_obj,
-  NULL AS similar_technologies,
   STRUCT(
     MAX(IF(client = 'desktop', origins, 0)) AS desktop,
     MAX(IF(client = 'mobile', origins, 0)) AS mobile