Merge pull request #98 from richardadonnell/1.58

Merge version 1.58 into main

.gitignore

@@ -46,3 +46,4 @@ releases/
 *.Identifier
 .specstory/
 browser-logs/
+logs/

PLANNING.md

@@ -0,0 +1,102 @@
+DO NOT BREAK OR REMOVE ANY CURRENT FUNCTIONALITY. MAINTAIN ALL CURRENT CSS STYLES.
+
+DO NOT MAKE ASSUMPTIONS ABOUT WHAT I NEED. ASK ME FOR ANYTHING YOU NEED.
+
+ONLY WORK ON ONE THING AT A TIME. BE VERY PRECISE AND THOUGHTFUL ABOUT EACH STEP. DO NOT HALLUCINATE.
+
+---
+
+# Upwork Job Scraper Enhancement: Multiple Search-Webhook Pairs
+
+## Overview
+Enhance the Chrome extension to support multiple pairs of search URLs and webhook URLs, allowing users to monitor different job searches and send results to different webhooks.
+
+## Requirements
+
+### Core Features
+- Allow unlimited search URL and webhook URL pairs
+- Each pair should have:
+  - Name for identification
+  - Search URL
+  - Webhook URL
+  - Enable/disable toggle
+- Combined job list in extension UI
+- Include search source information in webhook payload
+- Fully migrate from single URL system to new multi-pair system
+- Share global notification settings across all pairs
+
+### Data Structure
+```javascript
+// New storage structure
+{
+  "searchWebhookPairs": [
+    {
+      id: "unique-id",
+      name: "Pair Name",
+      searchUrl: "https://www.upwork.com/nx/search/jobs/?...",
+      webhookUrl: "https://...",
+      enabled: true
+    }
+    // ... more pairs
+  ]
+}
+
+// Enhanced webhook payload
+{
+  // ... existing job data ...
+  "source": {
+    "name": "Pair Name",
+    "searchUrl": "https://..."
+  }
+}
+```
+
+## Implementation Phases
+
+### Phase 1: Data Structure and Storage
+- [ ] Create new storage schema for search-webhook pairs
+- [ ] Implement migration from old single-URL system
+- [ ] Add utility functions for managing pairs
+- [ ] Update background.js to handle new storage format
+
+### Phase 2: UI Updates
+- [ ] Design new UI section for managing pairs
+- [ ] Add UI components:
+  - [ ] Pair list view
+  - [ ] Add/remove pair buttons
+  - [ ] Fields for name, URLs, and toggle
+  - [ ] Test webhook button per pair
+  - [ ] Open search URL button per pair
+- [ ] Update settings.js for new UI interactions
+
+### Phase 3: Job Processing
+- [ ] Update job scraping to handle multiple search URLs
+- [ ] Modify webhook payload to include source information
+- [ ] Update job processing to handle multiple webhooks
+- [ ] Enhance error handling for multiple pairs
+
+### Phase 4: Testing and Refinement
+- [ ] Test migration from old to new system
+- [ ] Test multiple search URLs
+- [ ] Test webhook functionality
+- [ ] Test enable/disable functionality
+- [ ] Performance testing with multiple pairs
+
+## Decisions Made
+
+1. **Number of Pairs**: No limit on the number of pairs a user can create
+2. **Job List Display**: Combined list in UI, with source tracking in webhook payload
+3. **Migration Strategy**: Full migration to new system (no simple mode)
+4. **Notification Settings**: Shared global settings across all pairs
+5. **Duplicate Jobs**: No special handling needed for jobs found by multiple searches
+
+## Next Steps
+1. Begin with Phase 1: Data Structure and Storage
+2. Create detailed technical specifications for each phase
+3. Implement changes incrementally with testing at each step
+
+## Questions/Concerns to Address
+- Impact on performance with multiple search URLs
+- Storage limits consideration for Chrome storage
+- Error handling strategy for multiple webhook failures
+- UI/UX for managing many pairs 

upwork-job-scraper/background.js

@@ -187,17 +187,19 @@ try {
       // Clear any existing alarms first
       await chrome.alarms.clear("checkJobs");
 
+      // Migrate old storage to new format
+      await migrateStorage();
+
+      // Load other settings
       const data = await chrome.storage.sync.get([
         "jobScrapingEnabled",
-        "webhookEnabled",
         "notificationsEnabled",
         "checkFrequency",
         "lastViewedTimestamp",
         "schedule",
       ]);
 
       jobScrapingEnabled = data.jobScrapingEnabled !== false;
-      webhookEnabled = data.webhookEnabled !== false;
       notificationsEnabled = data.notificationsEnabled !== false;
       checkFrequency = data.checkFrequency || 5;
       lastViewedTimestamp = data.lastViewedTimestamp || Date.now();
@@ -217,7 +219,6 @@ try {
         updateAlarm();
       }
 
-      await loadFeedSourceSettings();
       initializeLastViewedTimestamp();
 
       if (typeof updateNotificationsEnabled === "function") {
@@ -454,26 +455,27 @@ try {
   }
 
   // Import the functions from the new files
-  importScripts("errorHandling.js");
-  importScripts("jobScraping.js");
-  importScripts("activityLog.js");
-  importScripts("webhook.js");
-  importScripts("notifications.js");
-  importScripts("utils.js");
+  importScripts(
+    "errorHandling.js",
+    "jobScraping.js",
+    "activityLog.js",
+    "webhook.js",
+    "notifications.js",
+    "utils.js",
+    "storage.js"
+  );
 
   // Import the webhook functions at the top of background.js
   importScripts("webhook.js", "activityLog.js");
 
   async function processJobs(newJobs) {
     try {
       console.log("Starting processJobs with", newJobs.length, "new jobs");
+      addToActivityLog(`Processing ${newJobs.length} new jobs`);
 
-      // Get webhook settings
-      const webhookSettings = await chrome.storage.sync.get([
-        "webhookUrl",
-        "webhookEnabled",
-      ]);
-      console.log("Current webhook settings:", webhookSettings);
+      // Get enabled pairs
+      const enabledPairs = await getEnabledPairs();
+      console.log("Active search-webhook pairs:", enabledPairs.length);
 
       // Get existing jobs
       const data = await chrome.storage.local.get(["scrapedJobs"]);
@@ -486,56 +488,88 @@ try {
 
       // Process each new job
       for (const newJob of newJobs) {
-        if (!existingJobs.some((job) => job.url === newJob.url)) {
-          updatedJobs.push(newJob);
-          addedJobsCount++;
+        // Skip if job already exists
+        if (existingJobs.some((job) => job.url === newJob.url)) {
+          continue;
+        }
+
+        // Add source information to the job if not already present
+        if (!newJob.source) {
+          // Find the exact matching pair by search URL
+          const sourcePair = enabledPairs.find(
+            (pair) => pair.searchUrl === newJob.sourceUrl && pair.webhookUrl
+          );
+          if (sourcePair) {
+            newJob.source = {
+              name: sourcePair.name,
+              searchUrl: sourcePair.searchUrl,
+              webhookUrl: sourcePair.webhookUrl,
+            };
+          }
+        }
+
+        updatedJobs.push(newJob);
+        addedJobsCount++;
 
-          // Send to webhook if enabled and URL is set
-          if (webhookSettings.webhookEnabled && webhookSettings.webhookUrl) {
-            console.log("Sending job to webhook:", {
+        // Strict webhook pairing check
+        if (newJob.source?.searchUrl && newJob.source?.webhookUrl) {
+          try {
+            await sendToWebhook(newJob.source.webhookUrl, [newJob]);
+            addToActivityLog(
+              `Successfully sent job to webhook: ${newJob.title} (${newJob.source.name})`
+            );
+          } catch (error) {
+            console.error("Failed to send job to webhook:", error);
+            addToActivityLog(
+              `Failed to send job to webhook for ${newJob.source.name}: ${error.message}`
+            );
+            logAndReportError("Webhook send error", error, {
+              pairName: newJob.source.name,
               jobTitle: newJob.title,
-              webhookUrl: webhookSettings.webhookUrl,
             });
-
-            try {
-              await sendToWebhook(webhookSettings.webhookUrl, [newJob]);
-              addToActivityLog(
-                `Successfully sent job to webhook: ${newJob.title}`
-              );
-            } catch (error) {
-              console.error("Failed to send job to webhook:", error);
-              addToActivityLog(
-                `Failed to send job to webhook: ${error.message}`
-              );
-            }
-          } else {
-            console.log("Webhook not enabled or URL not set:", webhookSettings);
           }
+        } else {
+          console.log(
+            `Skipping webhook for job: ${newJob.title} - No webhook URL configured`
+          );
+          addToActivityLog(
+            `Skipped webhook for job from ${
+              newJob.source?.name || "unknown source"
+            } (no webhook URL configured)`
+          );
         }
       }
 
       // Combine and store jobs
       const allJobs = [...updatedJobs, ...existingJobs].slice(0, 100);
-
       await chrome.storage.local.set({ scrapedJobs: allJobs });
       addToActivityLog(
         `Added ${addedJobsCount} new jobs. Total jobs: ${allJobs.length}`
       );
 
       // Update badge and notify
+      newJobsCount += addedJobsCount;
       updateBadge();
 
       if (addedJobsCount > 0) {
-        chrome.runtime.sendMessage({ type: "jobsUpdate", jobs: allJobs });
+        try {
+          chrome.runtime.sendMessage({ type: "jobsUpdate", jobs: allJobs });
+        } catch (error) {
+          console.error("Failed to update settings page:", error);
+        }
 
-        // Use the sendNotification from notifications.js
-        sendNotification(
-          `Found ${addedJobsCount} new job${addedJobsCount > 1 ? "s" : ""}!`
-        );
+        if (notificationsEnabled) {
+          sendNotification(
+            `Found ${addedJobsCount} new job${addedJobsCount > 1 ? "s" : ""}!`
+          );
+        }
       }
     } catch (error) {
       console.error("Error in processJobs:", error);
-      logAndReportError("Error in processJobs", error);
+      logAndReportError("Error in processJobs", error, {
+        jobCount: newJobs?.length,
+        addedJobsCount,
+      });
     }
   }