fix: follow up issue #13 data ingestion and parsing

.gitignore

@@ -12,6 +12,7 @@ export_test/
 fit/
 test_everything.py
 tests/
+debug/
 dist/
 *.egg-info/
 *.db-shm

README.md

@@ -72,20 +72,25 @@ Add the MCP server to Claude Code, Claude Desktop, or any MCP client. The config
 }
 ```
 
-**Git clone** — use absolute paths to the venv:
+**Git clone** — use absolute paths to the venv and set `GARMIN_DATA_DIR` so the MCP server finds your database:
 
 ```json
 {
   "mcpServers": {
     "garmin": {
       "command": "/absolute/path/to/garmin-givemydata/venv/bin/python",
       "args": ["/absolute/path/to/garmin-givemydata/run_mcp.py"],
-      "cwd": "/absolute/path/to/garmin-givemydata"
+      "cwd": "/absolute/path/to/garmin-givemydata",
+      "env": {
+        "GARMIN_DATA_DIR": "/absolute/path/to/garmin-givemydata"
+      }
     }
   }
 }
 ```
 
+> **Note:** `GARMIN_DATA_DIR` tells the MCP server where `garmin.db` lives. Without it, the server falls back to `~/.garmin-givemydata/` which may not be where your data is if you cloned to a custom location.
+
 Save this as:
 - **Claude Code:** `.mcp.json` in your project root, or `~/.claude/settings.json` under `mcpServers` for global access
 - **Claude Desktop:** `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows)
@@ -110,7 +115,10 @@ Restart your client and run `/mcp` to approve the server. Then ask:
     "garmin": {
       "command": "C:\\Users\\jane\\code\\garmin-givemydata\\venv\\Scripts\\python.exe",
       "args": ["C:\\Users\\jane\\code\\garmin-givemydata\\run_mcp.py"],
-      "cwd": "C:\\Users\\jane\\code\\garmin-givemydata"
+      "cwd": "C:\\Users\\jane\\code\\garmin-givemydata",
+      "env": {
+        "GARMIN_DATA_DIR": "C:\\Users\\jane\\code\\garmin-givemydata"
+      }
     }
   }
 }

garmin_client/client.py

@@ -43,6 +43,7 @@
 
 DEFAULT_PROFILE_DIR = Path.home() / ".garmin-client" / "browser_profile"
 
+CONNECT_URL = "https://connect.garmin.com/app/"
 SSO_LOGIN_URL = (
     "https://sso.garmin.com/portal/sso/en-US/sign-in"
     "?clientId=GarminConnect"
@@ -117,6 +118,7 @@ def __init__(
         self._xvfb_proc = None
         self._xvfb_display = None
         self._xvfb_prev_display = None
+        self._save_raw_enabled = False
 
     # ── Display management ───────────────────────────────────────
 
@@ -353,9 +355,9 @@ def _type_slowly(self, text: str, delay_s: float = 0.03) -> None:
     def login(self, timeout_ms: int = 600000) -> bool:
         self._launch_browser()
 
-        log.debug("Navigating to connect.garmin.com/modern/")
+        log.debug("Navigating to %s", CONNECT_URL)
         try:
-            self._uc_navigate("https://connect.garmin.com/modern/", 12)
+            self._uc_navigate(CONNECT_URL, 12)
         except Exception as e:
             log.debug("Initial navigation error (expected for fresh profile): %s", e)
         time.sleep(3)
@@ -368,20 +370,28 @@ def login(self, timeout_ms: int = 600000) -> bool:
                 print("Already logged in (session restored)")
                 self._save_session()
                 return True
-            log.debug("On app page but no CSRF — session invalid, proceeding with login")
+            # If we are on a connect page but CSRF failed, don't clear cookies yet.
+            # Just try one navigation to /modern/ to see if it wakes up.
+            log.debug("On app page but no CSRF — attempting to refresh context")
             try:
-                self._uc_navigate(SSO_LOGIN_URL, 12)
+                self._uc_navigate(CONNECT_URL, 12)
                 time.sleep(3)
+                if self._post_login_setup():
+                    print("Already logged in (session restored after refresh)")
+                    self._save_session()
+                    return True
             except Exception:
                 pass
 
         print("Logging in...")
 
-        try:
-            self._driver.delete_all_cookies()
-            log.debug("Cleared stale cookies")
-        except Exception as e:
-            log.debug("Cookie clear error: %s", e)
+        # Only clear cookies if we are on the SSO login page (fresh login needed)
+        if self._is_on_login_page():
+            try:
+                self._driver.delete_all_cookies()
+                log.debug("Cleared stale cookies for fresh login")
+            except Exception as e:
+                log.debug("Cookie clear error: %s", e)
 
         for attempt in range(3):
             try:
@@ -534,7 +544,7 @@ def _read_mfa():
             if mfa_prompted and poll > 0 and poll % 30 == 0:
                 log.debug("Stuck on SSO after MFA — trying to navigate to app...")
                 try:
-                    self._driver.get("https://connect.garmin.com/modern/")
+                    self._driver.get(CONNECT_URL)
                     time.sleep(2)
                     url = self._driver.current_url
                     if "connect.garmin.com" in url and "sso.garmin.com" not in url:
@@ -616,7 +626,7 @@ def _post_login_setup(self) -> bool:
         if "/modern/" not in current:
             log.debug("Navigating to /modern/ for CSRF (was on %s)", current)
             try:
-                self._driver.get("https://connect.garmin.com/modern/")
+                self._driver.get(CONNECT_URL)
             except Exception:
                 pass
             time.sleep(3)
@@ -670,7 +680,7 @@ def _ensure_on_garmin(self) -> None:
         except Exception:
             return
         if "connect.garmin.com" not in current or "sso.garmin.com" in current:
-            self._driver.get("https://connect.garmin.com/modern/")
+            self._driver.get(CONNECT_URL)
             time.sleep(2)
 
     # ── Public API ───────────────────────────────────────────────
@@ -735,6 +745,43 @@ def download_file(self, api_path: str) -> Optional[bytes]:
             return bytes(result["data"])
         return None
 
+    # ── Save raw debug data ─────────────────────────────────────
+
+    def _save_raw(self, name: str, data):
+        """Save raw JSON response under the ``debug/raw`` directory (next to browser_profile)."""
+        if not self.profile_dir:
+            return
+        raw_dir = self.profile_dir.parent / "debug" / "raw"
+        raw_dir.mkdir(parents=True, exist_ok=True)
+        safe_name = name.replace("/", "_").replace("?", "_").replace("=", "_").replace(":", "_")
+        try:
+            payload = json.dumps(data, indent=2, sort_keys=True)
+            file_path = raw_dir / f"{safe_name}.json"
+
+            if file_path.exists():
+                try:
+                    if file_path.read_text() == payload:
+                        return
+                except Exception:
+                    pass
+
+                suffix = 2
+                while True:
+                    candidate = raw_dir / f"{safe_name}__{suffix}.json"
+                    if not candidate.exists():
+                        file_path = candidate
+                        break
+                    try:
+                        if candidate.read_text() == payload:
+                            return
+                    except Exception:
+                        pass
+                    suffix += 1
+
+            file_path.write_text(payload)
+        except Exception as e:
+            log.debug("Could not save raw data: %s", e)
+
     # ── Batch fetching ───────────────────────────────────────────
 
     def _fetch_batch(self, rest: dict, gql: dict) -> dict:
@@ -802,6 +849,15 @@ def _fetch_batch(self, rest: dict, gql: dict) -> dict:
         if result and "error" in result:
             log.warning("_fetch_batch JS error: %s", result["error"])
             return {}
+
+        # Save raw payloads and failures for later replay/debugging.
+        if self._save_raw_enabled and result:
+            for name, res in result.items():
+                if res.get("status") == 200 and res.get("data") is not None:
+                    self._save_raw(name, res["data"])
+                else:
+                    self._save_raw(name, res)
+
         return result or {}
 
     def _date_chunks(self, start: str, end: str, max_days: int = 28) -> list:
@@ -822,6 +878,7 @@ def fetch_all(
         end_date: Optional[str] = None,
         on_batch=None,
         known_activity_ids: Optional[set] = None,
+        save_raw: bool = False,
     ) -> dict:
         """Fetch all data from Garmin Connect.
 
@@ -831,8 +888,13 @@ def fetch_all(
             ``on_batch(endpoint_name, data, cal_date=None)`` called after each
             successful fetch.
         known_activity_ids : set, optional
-            Activity IDs that already have detail data — these will be skipped.
+            Activity IDs that already have detail data (splits, HR zones, weather).
+            These will be skipped during per-activity detail fetching.
+        save_raw : bool, default False
+            Whether to save raw JSON responses under the ``debug/raw`` directory
+            (next to ``browser_profile``).
         """
+        self._save_raw_enabled = save_raw
         today = target_date or date.today().isoformat()
         e_date = end_date or today
         s_date = start_date or (date.fromisoformat(today) - timedelta(days=30)).isoformat()

garmin_client/endpoints.py

@@ -52,6 +52,7 @@ def full_range_rest(display_name: str, start_date: str, end_date: str) -> dict:
         "vo2max_trend": f"/gc-api/metrics-service/metrics/maxmet/daily/{start_date}/{end_date}",
         "personal_records": f"/gc-api/personalrecord-service/personalrecord/prs/{dn}",
         "personal_record_types": f"/gc-api/personalrecord-service/personalrecordtype/prtypes/{dn}",
+        "lactate_threshold": "/gc-api/biometric-service/biometric/latestLactateThreshold",
     }
 
 
@@ -68,7 +69,6 @@ def full_range_graphql(display_name: str, start_date: str, end_date: str) -> dic
         "vo2max_cycling": f'query{{vo2MaxScalar(startDate:"{start_date}", endDate:"{end_date}", sport:"CYCLING")}}',
         "weight": f'query{{weightScalar(startDate:"{start_date}", endDate:"{end_date}")}}',
         "blood_pressure": f'query{{bloodPressureScalar(startDate:"{start_date}", endDate:"{end_date}")}}',
-        "activities_range": f'query{{activitiesScalar(displayName:"{dn}", startTimestampLocal:"{start_date}T00:00:00.00", endTimestampLocal:"{end_date}T23:59:59.99")}}',
         "activity_stats_all": f'query{{activityStatsScalar(aggregation:"daily", startDate:"{start_date}", endDate:"{end_date}", activityType:"all")}}',
         "activity_stats_running": f'query{{activityStatsScalar(aggregation:"daily", startDate:"{start_date}", endDate:"{end_date}", activityType:"running")}}',
         "activity_stats_cycling": f'query{{activityStatsScalar(aggregation:"daily", startDate:"{start_date}", endDate:"{end_date}", activityType:"cycling")}}',
@@ -119,12 +119,14 @@ def daily_rest(display_name: str, date: str) -> dict:
         "intensity_minutes": f"/gc-api/wellness-service/wellness/daily/im/{date}",
         "hydration": f"/gc-api/usersummary-service/usersummary/hydration/allData/{date}",
         "body_battery_events": f"/gc-api/wellness-service/wellness/bodyBattery/events/{date}",
+        "nutrition_meals": f"/gc-api/nutrition-service/meals/{date}",
         "fitness_age": f"/gc-api/fitnessage-service/fitnessage/{date}",
         "wellness_activity": f"/gc-api/wellnessactivity-service/activity/summary/{date}",
         "daily_movement": f"/gc-api/wellness-service/wellness/dailyMovement?calendarDate={date}",
         "endurance_score": f"/gc-api/metrics-service/metrics/endurancescore?calendarDate={date}",
         "hill_score": f"/gc-api/metrics-service/metrics/hillscore?calendarDate={date}",
         "race_predictions": f"/gc-api/metrics-service/metrics/racepredictions/daily/{dn}?fromCalendarDate={date}&toCalendarDate={date}",
+        "hrv_timeline": f"/gc-api/hrv-service/hrv/{date}",
     }
 
 

garmin_givemydata.py

@@ -121,6 +121,7 @@ def fetch_direct_to_db(
     conn,
     start_date: str,
     end_date: str,
+    save_raw: bool = False,
 ) -> None:
     """Fetch data and save each batch directly to SQLite."""
     counts = {}
@@ -157,6 +158,7 @@ def on_batch(endpoint_name, data, cal_date=None):
             end_date=end_date,
             on_batch=on_batch,
             known_activity_ids=known_activity_ids,
+            save_raw=save_raw,
         )
     else:
         # Calculate total chunks for progress reporting
@@ -184,6 +186,7 @@ def on_batch(endpoint_name, data, cal_date=None):
                 end_date=chunk_end.isoformat(),
                 on_batch=on_batch,
                 known_activity_ids=known_activity_ids,
+                save_raw=save_raw,
             )
 
             new_total = sum(counts.values())
@@ -248,6 +251,9 @@ def main():
     fetch_group.add_argument("--full", action="store_true", help="Force full historical fetch")
     fetch_group.add_argument("--days", type=int, help="Fetch last N days")
     fetch_group.add_argument("--since", type=str, help="Fetch from date (YYYY-MM-DD)")
+    fetch_group.add_argument(
+        "--save-raw", action="store_true", help="Save raw JSON responses to debug/raw for debugging"
+    )
     fetch_group.add_argument(
         "--no-files",
         action="store_true",
@@ -519,7 +525,7 @@ def main():
             print("Login failed!")
             sys.exit(1)
 
-        fetch_direct_to_db(client, conn, start, end)
+        fetch_direct_to_db(client, conn, start, end, save_raw=args.save_raw)
 
         # Report actual row counts from the database (not upsert operations)
         tables = db_query(