diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..2701db5 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,39 @@ +# šŸ› ļø Surya Tools + +Utility tools and scripts to support the Surya foundation model ecosystem. + +## Available Tools + +### šŸ“” SDO Real-Time Data Fetcher + +Lightweight tool for fetching real-time Solar Dynamics Observatory observations. + +**Location**: `tools/sdo_data_fetcher/` + +**Purpose**: +- Fetch latest SDO/AIA and HMI observations +- Download specific wavelengths for custom analysis +- Real-time solar activity monitoring +- Quick data exploration without full dataset downloads + +**Quick Start**: +```bash +cd tools/sdo_data_fetcher +pip install -r requirements.txt +python sdo_fetcher_v2.py --list +``` + +See [sdo_data_fetcher/README.md](sdo_data_fetcher/README.md) for detailed documentation. + +--- + +## Contributing New Tools + +We welcome additional tools that support the Surya ecosystem! Consider contributing: +- Data preprocessing utilities +- Visualization tools +- Custom dataset generators +- Analysis scripts +- Integration helpers + +Please follow the repository's contribution guidelines when adding new tools. diff --git a/tools/sdo_data_fetcher/README.md b/tools/sdo_data_fetcher/README.md new file mode 100644 index 0000000..26410d1 --- /dev/null +++ b/tools/sdo_data_fetcher/README.md @@ -0,0 +1,133 @@ +# šŸŒž SDO Real-Time Data Fetcher + +A lightweight Python tool for fetching live Solar Dynamics Observatory (SDO) browse imagery for Surya-related experimentation and monitoring. + +## Overview + +While Surya primarily uses preprocessed SDO datasets, this tool provides a simple way to fetch current AIA and HMI observations for: + +- real-time solar activity monitoring +- quick data exploration +- recent-event validation workflows +- prototype preprocessing and inference pipelines + +## What's New + +The downloader now supports redundant live providers so it can continue working when a single upstream host is unavailable. + +### Provider fallback chain + +By default, the fetchers try providers in this order: + +1. `lmsal` ā€” LMSAL Sun Today browse imagery +2. `jsoc` ā€” Stanford JSOC latest HMI imagery +3. `nasa` ā€” NASA SDO browse imagery +4. `helioviewer` ā€” Helioviewer rendered imagery + +This is available through both `sdo_fetcher_v2.py` and `sdo_data_fetcher.py` with `--provider auto`. + +## Quick Start + +```bash +cd tools/sdo_data_fetcher +pip install -r requirements.txt +``` + +### Basic usage + +```bash +# Latest AIA 171 image using automatic fallback +python sdo_fetcher_v2.py --source AIA_171 + +# Force a specific provider +python sdo_fetcher_v2.py --source AIA_171 --provider lmsal + +# Download latest HMI magnetogram from JSOC +python sdo_fetcher_v2.py --source HMI_Magnetogram --provider jsoc + +# Download multiple channels +python sdo_fetcher_v2.py --multiple +``` + +### Original fetcher + +```bash +python sdo_data_fetcher.py --source AIA_193 --provider auto +``` + +## Available providers + +- `auto` +- `lmsal` +- `jsoc` +- `nasa` +- `helioviewer` + +## Available data sources + +### AIA channels + +- `AIA_94` +- `AIA_131` +- `AIA_171` +- `AIA_193` +- `AIA_211` +- `AIA_304` +- `AIA_335` +- `AIA_1600` +- `AIA_1700` + +### HMI channels + +- `HMI_Continuum` +- `HMI_Magnetogram` + +## Python example + +```python +from sdo_fetcher_v2 import SDOFetcher + +fetcher = SDOFetcher(output_dir="surya_inference_data") +metadata = fetcher.get_latest_image_direct(source="AIA_171", provider="auto") + +if metadata: + print(metadata["filepath"]) + print(metadata["provider_name"]) + print(metadata.get("observation_time")) +``` + +## Advanced examples + +Run the menu-driven helper: + +```bash +python sdo_advanced_examples.py +``` + +It includes: + +- multi-wavelength comparison downloads +- active region monitoring +- prominence monitoring +- space weather quick checks +- continuous monitoring + +## Output + +Each download writes: + +- an image file (`.jpg`, `.gif`, or `.png`, depending on provider) +- a `.json` metadata file containing provider, source, URL, and timing info + +## Notes + +- LMSAL provides daily AIA and HMI browse imagery. +- JSOC support is currently most useful for HMI live products. +- Helioviewer remains as an API fallback when browse-image hosts are unavailable. + +## References + +- NASA SDO: https://sdo.gsfc.nasa.gov/ +- LMSAL Sun Today: https://suntoday.lmsal.com/suntoday/ +- JSOC latest HMI: https://jsoc1.stanford.edu/hmi_latest.html +- Helioviewer: https://helioviewer.org/ diff --git a/tools/sdo_data_fetcher/requirements.txt b/tools/sdo_data_fetcher/requirements.txt new file mode 100644 index 0000000..0eb8cae --- /dev/null +++ b/tools/sdo_data_fetcher/requirements.txt @@ -0,0 +1 @@ +requests>=2.31.0 diff --git a/tools/sdo_data_fetcher/sdo_advanced_examples.py b/tools/sdo_data_fetcher/sdo_advanced_examples.py new file mode 100644 index 0000000..f3532e6 --- /dev/null +++ b/tools/sdo_data_fetcher/sdo_advanced_examples.py @@ -0,0 +1,292 @@ +""" +Advanced SDO Examples - Building on the basic fetcher +Demonstrates monitoring, time-series, and composite image creation +""" + +import time +from datetime import datetime, timezone +from pathlib import Path +from sdo_fetcher_v2 import SDOFetcher + + +def continuous_monitor(interval_seconds=300, sources=None, provider="auto"): + """ + Continuously monitor and download SDO images at specified intervals + + Args: + interval_seconds: Time between downloads (default: 5 minutes) + sources: List of sources to monitor (default: AIA_171) + """ + if sources is None: + sources = ["AIA_171"] + + fetcher = SDOFetcher(output_dir="monitoring") + + print(f"Starting continuous monitoring...") + print(f"Sources: {', '.join(sources)}") + print(f"Interval: {interval_seconds} seconds") + print(f"Provider mode: {provider}") + print(f"Press Ctrl+C to stop\n") + + iteration = 0 + try: + while True: + iteration += 1 + timestamp = datetime.now(timezone.utc).isoformat() + + print(f"\n{'='*60}") + print(f"Iteration #{iteration} at {timestamp}") + print(f"{'='*60}") + + for source in sources: + try: + result = fetcher.get_latest_image_direct(source, provider=provider) + if result: + print(f"āœ“ {source} downloaded successfully via {result.get('provider_name', result.get('provider', 'unknown'))}") + else: + print(f"āœ— {source} failed") + except Exception as e: + print(f"āœ— Error downloading {source}: {e}") + + print(f"\nWaiting {interval_seconds} seconds until next download...") + time.sleep(interval_seconds) + + except KeyboardInterrupt: + print(f"\n\nMonitoring stopped. Downloaded {iteration} sets of images.") + print(f"Images saved in: {fetcher.output_dir}") + + +def download_comparison_set(): + """ + Download a comparison set of multiple wavelengths + Useful for multi-wavelength solar analysis + """ + print("\n" + "="*60) + print("Downloading Multi-Wavelength Comparison Set") + print("="*60 + "\n") + + # Select complementary wavelengths + sources = [ + "AIA_171", # Quiet corona + "AIA_193", # Active regions + "AIA_304", # Prominences + "AIA_211", # Active regions (hotter) + "HMI_Magnetogram", # Magnetic field + "HMI_Continuum", # Visible surface + ] + + fetcher = SDOFetcher(output_dir="comparison_set") + results = fetcher.download_multiple(sources, provider="auto") + + print("\n" + "="*60) + print("Comparison Set Complete!") + print("="*60) + print(f"Downloaded {len(results)} images") + print("\nUse these for:") + print(" - Multi-wavelength composite images") + print(" - Temperature analysis") + print(" - Active region identification") + print(" - Prominence and filament studies") + print("="*60 + "\n") + + return results + + +def download_active_region_set(): + """ + Download wavelengths optimal for observing active regions and flares + """ + print("\n" + "="*60) + print("Downloading Active Region / Flare Observation Set") + print("="*60 + "\n") + + # Wavelengths best for active regions and flares + sources = [ + "AIA_94", # Hot flare plasma + "AIA_131", # Flaring regions + "AIA_193", # Active regions + "AIA_211", # Active regions + "HMI_Magnetogram", # Magnetic field + ] + + fetcher = SDOFetcher(output_dir="active_regions") + results = fetcher.download_multiple(sources, provider="auto") + + print("\nActive region monitoring complete!") + print("Check these images for:") + print(" - Solar flares (bright spots in 94ƅ and 131ƅ)") + print(" - Active region structure (193ƅ, 211ƅ)") + print(" - Sunspot magnetic complexity (HMI Magnetogram)") + + return results + + +def quick_space_weather_check(): + """ + Quick download for space weather assessment + """ + print("\n" + "="*60) + print("SPACE WEATHER QUICK CHECK") + print("="*60 + "\n") + + fetcher = SDOFetcher(output_dir="space_weather") + + # Get the most relevant images for space weather + sources = ["AIA_193", "HMI_Magnetogram"] + + print("Downloading key space weather indicators...") + results = fetcher.download_multiple(sources, provider="auto") + + if len(results) == 2: + print("\n" + "="*60) + print("READY FOR ANALYSIS") + print("="*60) + print("\nCheck the images for:") + print(" šŸ“ø AIA 193: Active regions and coronal holes") + print(" šŸ§² HMI Magnetogram: Complex magnetic fields (flare potential)") + print("\nLook for:") + print(" āš ļø Dark regions = coronal holes ā†’ fast solar wind") + print(" āš ļø Bright active regions = potential for flares") + print(" āš ļø Complex magnetograms = higher flare risk") + print("="*60 + "\n") + + return results + + +def download_prominence_monitoring(): + """ + Download wavelengths optimal for prominence/filament observation + """ + print("\n" + "="*60) + print("Prominence/Filament Monitoring Set") + print("="*60 + "\n") + + # Best wavelengths for prominences + sources = [ + "AIA_304", # Primary prominence wavelength + "AIA_171", # Context (corona) + "HMI_Continuum", # Visible disk + ] + + fetcher = SDOFetcher(output_dir="prominences") + results = fetcher.download_multiple(sources, provider="auto") + + print("\nProminence monitoring complete!") + print("304ƅ is best for seeing prominences on the solar limb") + + return results + + +def create_monitoring_script(): + """ + Generate a standalone monitoring script + """ + script_content = '''#!/usr/bin/env python3 +""" +Automated SDO Monitoring Script +Runs continuously and downloads images every 15 minutes +""" + +import time +from datetime import datetime, timezone +from sdo_fetcher_v2 import SDOFetcher +import logging + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler('sdo_monitor.log'), + logging.StreamHandler() + ] +) + +def main(): + fetcher = SDOFetcher(output_dir="continuous_monitoring") + sources = ["AIA_171", "AIA_193", "HMI_Magnetogram"] + interval = 900 # 15 minutes + + logging.info("Starting SDO continuous monitoring") + logging.info(f"Sources: {sources}") + logging.info(f"Interval: {interval} seconds") + + iteration = 0 + while True: + try: + iteration += 1 + logging.info(f"=== Iteration {iteration} ===") + + for source in sources: + try: + result = fetcher.get_latest_image_direct(source, provider="auto") + if result: + logging.info(f"āœ“ Downloaded {source} via {result.get('provider_name', result.get('provider', 'unknown'))}") + except Exception as e: + logging.error(f"āœ— Failed to download {source}: {e}") + + logging.info(f"Waiting {interval} seconds...") + time.sleep(interval) + + except KeyboardInterrupt: + logging.info("Monitoring stopped by user") + break + except Exception as e: + logging.error(f"Unexpected error: {e}") + time.sleep(60) # Wait 1 minute before retrying + +if __name__ == "__main__": + main() +''' + + with open("monitoring_daemon.py", 'w', encoding='utf-8') as f: + f.write(script_content) + + print("\nāœ“ Created 'monitoring_daemon.py'") + print("Run it with: python monitoring_daemon.py") + print("It will continuously download SDO images every 15 minutes") + + +def main(): + """Main menu for advanced examples""" + print("\n" + "="*60) + print("SDO Advanced Examples") + print("="*60) + print("\n1. Download multi-wavelength comparison set") + print("2. Download active region/flare observation set") + print("3. Quick space weather check") + print("4. Download prominence monitoring set") + print("5. Start continuous monitoring (Ctrl+C to stop)") + print("6. Create monitoring daemon script") + print("7. Exit") + + choice = input("\nSelect option (1-7): ").strip() + + if choice == "1": + download_comparison_set() + elif choice == "2": + download_active_region_set() + elif choice == "3": + quick_space_weather_check() + elif choice == "4": + download_prominence_monitoring() + elif choice == "5": + sources = input("Enter sources (comma-separated, or press Enter for AIA_171): ").strip() + if sources: + sources = [s.strip() for s in sources.split(",")] + else: + sources = ["AIA_171"] + interval = input("Enter interval in seconds (default 300): ").strip() + interval = int(interval) if interval else 300 + provider = input("Enter provider (auto/lmsal/jsoc/nasa/helioviewer, default auto): ").strip() or "auto" + continuous_monitor(interval, sources, provider) + elif choice == "6": + create_monitoring_script() + elif choice == "7": + print("Goodbye!") + else: + print("Invalid choice") + + +if __name__ == "__main__": + main() diff --git a/tools/sdo_data_fetcher/sdo_data_fetcher.py b/tools/sdo_data_fetcher/sdo_data_fetcher.py new file mode 100644 index 0000000..af007d9 --- /dev/null +++ b/tools/sdo_data_fetcher/sdo_data_fetcher.py @@ -0,0 +1,194 @@ +""" +SDO (Solar Dynamic Observatory) Data Fetcher + +This script fetches the latest solar images from NASA's Solar Dynamic Observatory +using the Helioviewer API. It supports multiple instruments and wavelengths. +""" + +import requests +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path +from typing import Optional, Dict +import argparse +from sdo_provider import SDOProviderClient, SDO_SOURCES + + +class SDODataFetcher: + """Fetches latest SDO data from Helioviewer API""" + + BASE_URL = "https://api.helioviewer.org/v2/" + SDO_SOURCES = SDO_SOURCES + + def __init__(self, output_dir: str = "sdo_data"): + """Initialize the fetcher with an output directory""" + self.output_dir = Path(output_dir) + self.output_dir.mkdir(exist_ok=True) + self.provider_client = SDOProviderClient(output_dir=output_dir) + + def get_latest_available_date(self, source: str = "AIA_171", provider: str = "auto") -> Optional[str]: + """Query the API for the latest available SDO observation time""" + timestamp = self.provider_client.get_latest_timestamp(source=source, provider=provider) + if timestamp: + print(f"Latest SDO data available: {timestamp}") + return timestamp + + fallback = (datetime.now(timezone.utc) - timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%S.000Z") + print(f"Using fallback date: {fallback}") + return fallback + + def get_latest_image(self, source: str = "AIA_171", image_scale: float = 2.4, provider: str = "auto") -> Optional[Dict]: + """ + Fetch the latest SDO image + + Args: + source: SDO source identifier (e.g., 'AIA_171', 'HMI_Magnetogram') + image_scale: Resolution in arcseconds per pixel (lower = higher resolution) + + Returns: + Dictionary with image metadata and file path + """ + _ = image_scale + if source not in self.SDO_SOURCES: + raise ValueError(f"Invalid source. Choose from: {list(self.SDO_SOURCES.keys())}") + + print(f"Fetching latest {source} data...") + print(f"Description: {self.SDO_SOURCES[source]['description']}") + return self.provider_client.download_latest_image(source=source, provider=provider) + + def get_latest_data_timestamp(self, source: str = "AIA_171", provider: str = "auto") -> Optional[str]: + """Get the timestamp of the latest available SDO data""" + try: + return self.provider_client.get_latest_timestamp(source=source, provider=provider) + except requests.exceptions.RequestException as e: + print(f"Error getting latest timestamp: {e}") + return None + + def download_multiple_wavelengths(self, sources: list = None, provider: str = "auto"): + """ + Download images from multiple SDO sources + + Args: + sources: List of source identifiers (defaults to common wavelengths) + """ + if sources is None: + sources = ["AIA_171", "AIA_193", "AIA_304", "HMI_Magnetogram"] + + results = [] + for source in sources: + print(f"\n{'='*60}") + result = self.get_latest_image(source, provider=provider) + if result: + results.append(result) + + print(f"\n{'='*60}") + print(f"Downloaded {len(results)} images successfully!") + return results + + @staticmethod + def list_available_sources(): + """Print all available SDO sources""" + print("\nAvailable SDO Data Sources:") + print("="*60) + for key, value in SDODataFetcher.SDO_SOURCES.items(): + print(f"{key:20} - {value['description']}") + print("\nAvailable providers:") + SDOProviderClient.list_providers() + + +def main(): + """Main CLI interface""" + parser = argparse.ArgumentParser( + description="Fetch latest SDO (Solar Dynamic Observatory) data", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Download latest AIA 171 ƅ image + python sdo_data_fetcher.py + + # Download specific wavelength + python sdo_data_fetcher.py --source AIA_304 + + # Download multiple wavelengths + python sdo_data_fetcher.py --multiple + + # List all available sources + python sdo_data_fetcher.py --list + + # Download to specific directory + python sdo_data_fetcher.py --output my_sdo_images + """ + ) + + parser.add_argument( + '--source', '-s', + type=str, + default='AIA_171', + help='SDO source to fetch (default: AIA_171)' + ) + + parser.add_argument( + '--output', '-o', + type=str, + default='sdo_data', + help='Output directory for downloaded data (default: sdo_data)' + ) + + parser.add_argument( + '--scale', + type=float, + default=2.4, + help='Image scale in arcseconds per pixel (default: 2.4)' + ) + + parser.add_argument( + '--multiple', '-m', + action='store_true', + help='Download multiple common wavelengths' + ) + + parser.add_argument( + '--list', '-l', + action='store_true', + help='List all available SDO sources' + ) + + parser.add_argument( + '--timestamp', '-t', + action='store_true', + help='Get the timestamp of latest available data' + ) + + parser.add_argument( + '--provider', '-p', + type=str, + default='auto', + help='Data provider: auto, lmsal, jsoc, nasa, helioviewer' + ) + + args = parser.parse_args() + + # List available sources and exit + if args.list: + SDODataFetcher.list_available_sources() + return + + # Initialize fetcher + fetcher = SDODataFetcher(output_dir=args.output) + + # Get latest timestamp + if args.timestamp: + timestamp = fetcher.get_latest_data_timestamp(source=args.source, provider=args.provider) + if timestamp: + print(f"Latest SDO data available at: {timestamp}") + return + + # Download data + if args.multiple: + fetcher.download_multiple_wavelengths(provider=args.provider) + else: + fetcher.get_latest_image(source=args.source, image_scale=args.scale, provider=args.provider) + + +if __name__ == "__main__": + main() diff --git a/tools/sdo_data_fetcher/sdo_fetcher_v2.py b/tools/sdo_data_fetcher/sdo_fetcher_v2.py new file mode 100644 index 0000000..7e4aaa1 --- /dev/null +++ b/tools/sdo_data_fetcher/sdo_fetcher_v2.py @@ -0,0 +1,116 @@ +""" +SDO Data Fetcher v2 - Alternative Implementation +Uses NASA's Helioviewer.org latest images API +""" + +import requests +from datetime import datetime, timezone +import json +from pathlib import Path +from typing import Optional, Dict +import argparse +from sdo_provider import SDOProviderClient, SDO_SOURCES + + +class SDOFetcher: + """Simplified SDO data fetcher using Helioviewer's latest images""" + + SDO_SOURCES = SDO_SOURCES + + def __init__(self, output_dir: str = "sdo_data"): + self.output_dir = Path(output_dir) + self.output_dir.mkdir(exist_ok=True) + self.provider_client = SDOProviderClient(output_dir=output_dir) + + def get_latest_image_png(self, source: str = "AIA_171", provider: str = "auto") -> Optional[Dict]: + """ + Fetch latest SDO image as PNG using a simpler method + + Args: + source: SDO source identifier + + Returns: + Dictionary with metadata and filepath + """ + return self.provider_client.download_latest_image(source=source, provider=provider) + + def get_latest_image_direct(self, source: str = "AIA_171", provider: str = "auto") -> Optional[Dict]: + """ + Alternative method: Fetch from SDO's direct image feed + Uses helioviewer.org's pre-rendered latest images + """ + result = self.provider_client.download_latest_image(source=source, provider=provider) + if result: + print(f"\n{'='*60}") + print(f"Success! Downloaded latest SDO {source} image") + print(f"Provider: {result.get('provider_name', result.get('provider', 'unknown'))}") + if result.get("observation_time"): + print(f"Observation time: {result['observation_time']}") + print(f"{'='*60}\n") + return result + + def download_multiple(self, sources: list = None, provider: str = "auto"): + """Download multiple wavelengths""" + if sources is None: + sources = ["AIA_171", "AIA_193", "AIA_304", "HMI_Magnetogram"] + + print(f"\nDownloading {len(sources)} different SDO images...") + print("="*60) + + results = [] + for source in sources: + result = self.get_latest_image_direct(source, provider=provider) + if result: + results.append(result) + + print(f"\n{'='*60}") + print(f"Successfully downloaded {len(results)}/{len(sources)} images") + print(f"{'='*60}\n") + + return results + + @staticmethod + def list_sources(): + """List all available sources""" + print("\n" + "="*60) + print("Available SDO Data Sources") + print("="*60) + for key, info in SDOFetcher.SDO_SOURCES.items(): + print(f" {key:20} - {info['name']} ({info['wavelength']})") + print("="*60 + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="SDO Data Fetcher v2 - Fetch latest solar images from NASA's SDO", + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument('--source', '-s', default='AIA_171', + help='SDO source (default: AIA_171)') + parser.add_argument('--output', '-o', default='sdo_data', + help='Output directory (default: sdo_data)') + parser.add_argument('--multiple', '-m', action='store_true', + help='Download multiple wavelengths') + parser.add_argument('--list', '-l', action='store_true', + help='List available sources') + parser.add_argument('--provider', '-p', default='auto', + help='Data provider: auto, lmsal, jsoc, nasa, helioviewer') + + args = parser.parse_args() + + if args.list: + SDOFetcher.list_sources() + SDOProviderClient.list_providers() + return + + fetcher = SDOFetcher(output_dir=args.output) + + if args.multiple: + fetcher.download_multiple(provider=args.provider) + else: + fetcher.get_latest_image_direct(source=args.source, provider=args.provider) + + +if __name__ == "__main__": + main() diff --git a/tools/sdo_data_fetcher/sdo_provider.py b/tools/sdo_data_fetcher/sdo_provider.py new file mode 100644 index 0000000..df7bdc4 --- /dev/null +++ b/tools/sdo_data_fetcher/sdo_provider.py @@ -0,0 +1,424 @@ +""" +Shared SDO data provider logic with automatic fallback. +""" + +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path +from typing import Dict, Iterable, Optional + +import requests + + +SDO_SOURCES = { + "AIA_94": { + "sourceId": 13, + "name": "AIA 94", + "wavelength": "94ƅ", + "description": "AIA 94 ƅ - Hot flare plasma", + "nasa_code": "0094", + "lmsal_code": "0094", + }, + "AIA_131": { + "sourceId": 14, + "name": "AIA 131", + "wavelength": "131ƅ", + "description": "AIA 131 ƅ - Flaring regions", + "nasa_code": "0131", + "lmsal_code": "0131", + }, + "AIA_171": { + "sourceId": 15, + "name": "AIA 171", + "wavelength": "171ƅ", + "description": "AIA 171 ƅ - Quiet corona and coronal loops", + "nasa_code": "0171", + "lmsal_code": "0171", + }, + "AIA_193": { + "sourceId": 16, + "name": "AIA 193", + "wavelength": "193ƅ", + "description": "AIA 193 ƅ - Hot plasma in active regions", + "nasa_code": "0193", + "lmsal_code": "0193", + }, + "AIA_211": { + "sourceId": 17, + "name": "AIA 211", + "wavelength": "211ƅ", + "description": "AIA 211 ƅ - Active regions", + "nasa_code": "0211", + "lmsal_code": "0211", + }, + "AIA_304": { + "sourceId": 18, + "name": "AIA 304", + "wavelength": "304ƅ", + "description": "AIA 304 ƅ - Chromosphere and prominence", + "nasa_code": "0304", + "lmsal_code": "0304", + }, + "AIA_335": { + "sourceId": 19, + "name": "AIA 335", + "wavelength": "335ƅ", + "description": "AIA 335 ƅ - Active regions", + "nasa_code": "0335", + "lmsal_code": "0335", + }, + "AIA_1600": { + "sourceId": 20, + "name": "AIA 1600", + "wavelength": "1600ƅ", + "description": "AIA 1600 ƅ - Upper photosphere", + "nasa_code": "1600", + "lmsal_code": "1600", + }, + "AIA_1700": { + "sourceId": 21, + "name": "AIA 1700", + "wavelength": "1700ƅ", + "description": "AIA 1700 ƅ - Temperature minimum", + "nasa_code": "1700", + "lmsal_code": "1700", + }, + "HMI_Continuum": { + "sourceId": 22, + "name": "HMI Continuum", + "wavelength": "Continuum", + "description": "HMI Continuum - Solar surface", + "nasa_code": "HMIIC", + "lmsal_code": "_HMI_cont_aiascale", + "jsoc_path": "/data/hmi/images/latest/HMI_latest_Int_1024x1024.gif", + "jsoc_timestamp_key": "continuum", + }, + "HMI_Magnetogram": { + "sourceId": 23, + "name": "HMI Magnetogram", + "wavelength": "Magnetogram", + "description": "HMI Magnetogram - Magnetic field", + "nasa_code": "HMII", + "lmsal_code": "_HMImag", + "jsoc_path": "/data/hmi/images/latest/HMI_latest_Mag_1024x1024.gif", + "jsoc_timestamp_key": "magnetogram", + }, +} + + +PROVIDER_LABELS = { + "lmsal": "LMSAL Sun Today", + "jsoc": "Stanford JSOC", + "nasa": "NASA SDO", + "helioviewer": "Helioviewer API", +} + + +AUTO_PROVIDER_ORDER = ("lmsal", "jsoc", "nasa", "helioviewer") + + +class SDOProviderClient: + """Download latest SDO imagery from multiple redundant providers.""" + + def __init__(self, output_dir: str = "sdo_data"): + self.output_dir = Path(output_dir) + self.output_dir.mkdir(exist_ok=True) + self.session = requests.Session() + + def download_latest_image(self, source: str = "AIA_171", provider: str = "auto") -> Optional[Dict]: + """Download the latest image using the requested provider or fallback chain.""" + if source not in SDO_SOURCES: + raise ValueError(f"Invalid source. Choose from: {list(SDO_SOURCES.keys())}") + + provider_order = self._resolve_provider_order(provider) + + print(f"\nFetching latest {source} image...") + print(f"Wavelength: {SDO_SOURCES[source]['wavelength']}") + print(f"Provider order: {', '.join(provider_order)}") + + last_error = None + + for provider_name in provider_order: + try: + print(f"Trying provider: {PROVIDER_LABELS[provider_name]}") + result = getattr(self, f"_download_from_{provider_name}")(source) + if result: + return result + except requests.exceptions.RequestException as e: + last_error = e + print(f"Provider {provider_name} failed: {e}") + except Exception as e: + last_error = e + print(f"Provider {provider_name} failed unexpectedly: {e}") + + if last_error: + print(f"All providers failed. Last error: {last_error}") + else: + print("All providers failed.") + return None + + def get_latest_timestamp(self, source: str = "AIA_171", provider: str = "auto") -> Optional[str]: + """Best-effort timestamp lookup using the same provider order.""" + if source not in SDO_SOURCES: + raise ValueError(f"Invalid source. Choose from: {list(SDO_SOURCES.keys())}") + + for provider_name in self._resolve_provider_order(provider): + try: + timestamp = getattr(self, f"_timestamp_from_{provider_name}")(source) + if timestamp: + return timestamp + except Exception: + continue + return None + + @staticmethod + def list_providers(): + """Print available provider names.""" + print("\nAvailable data providers:") + print("=" * 60) + print("auto - Automatic fallback chain") + for key, label in PROVIDER_LABELS.items(): + print(f"{key:12} - {label}") + + def _resolve_provider_order(self, provider: str) -> Iterable[str]: + provider = provider.lower() + if provider == "auto": + return AUTO_PROVIDER_ORDER + if provider not in PROVIDER_LABELS: + raise ValueError(f"Invalid provider. Choose from: auto, {', '.join(PROVIDER_LABELS.keys())}") + return (provider,) + + def _download_from_lmsal(self, source: str) -> Optional[Dict]: + code = SDO_SOURCES[source].get("lmsal_code") + if not code: + return None + + for day_offset in range(0, 4): + candidate_date = datetime.now(timezone.utc) - timedelta(days=day_offset) + date_path = candidate_date.strftime("%Y/%m/%d") + urls = [ + f"http://suntoday.lmsal.com/sdomedia/SunInTime/{date_path}/t{code}.jpg", + f"https://suntoday.lmsal.com/sdomedia/SunInTime/{date_path}/t{code}.jpg", + ] + + for url in urls: + try: + response = self.session.get(url, timeout=30, stream=True) + if response.status_code == 404: + response.close() + continue + + response.raise_for_status() + return self._save_response( + response=response, + source=source, + provider="lmsal", + image_url=url, + extension=".jpg", + observation_time=response.headers.get("Last-Modified") or candidate_date.strftime("%Y-%m-%d"), + extra_metadata={"date_path": date_path}, + ) + except requests.exceptions.RequestException: + continue + + return None + + def _download_from_jsoc(self, source: str) -> Optional[Dict]: + jsoc_path = SDO_SOURCES[source].get("jsoc_path") + if not jsoc_path: + return None + + url = f"https://jsoc1.stanford.edu{jsoc_path}" + response = self.session.get(url, timeout=30, stream=True) + response.raise_for_status() + + return self._save_response( + response=response, + source=source, + provider="jsoc", + image_url=url, + extension=Path(jsoc_path).suffix or ".img", + observation_time=self._timestamp_from_jsoc(source), + ) + + def _download_from_nasa(self, source: str) -> Optional[Dict]: + nasa_code = SDO_SOURCES[source].get("nasa_code") + if not nasa_code: + return None + + urls = [ + f"http://sdo.gsfc.nasa.gov/assets/img/latest/latest_1024_{nasa_code}.jpg", + f"https://sdo.gsfc.nasa.gov/assets/img/latest/latest_1024_{nasa_code}.jpg", + ] + + for url in urls: + try: + response = self.session.get(url, timeout=30, stream=True) + response.raise_for_status() + + return self._save_response( + response=response, + source=source, + provider="nasa", + image_url=url, + extension=".jpg", + observation_time=response.headers.get("Last-Modified"), + ) + except requests.exceptions.RequestException: + continue + + return None + + def _download_from_helioviewer(self, source: str) -> Optional[Dict]: + source_id = SDO_SOURCES[source]["sourceId"] + info_url = "https://api.helioviewer.org/v2/getClosestImage/" + info_params = { + "date": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "sourceId": source_id, + } + + info_response = self.session.get(info_url, params=info_params, timeout=30) + info_response.raise_for_status() + image_info = info_response.json() + image_id = image_info.get("id") + + if not image_id: + return None + + tile_url = "https://api.helioviewer.org/v2/getTile/" + tile_params = { + "id": image_id, + "x": 0, + "y": 0, + "imageScale": 2.4, + "display": "true", + } + response = self.session.get(tile_url, params=tile_params, timeout=30, stream=True) + response.raise_for_status() + + return self._save_response( + response=response, + source=source, + provider="helioviewer", + image_url=response.url, + extension=".png", + observation_time=image_info.get("date"), + extra_metadata={"image_id": image_id}, + ) + + def _timestamp_from_lmsal(self, source: str) -> Optional[str]: + code = SDO_SOURCES[source].get("lmsal_code") + if not code: + return None + + for day_offset in range(0, 4): + candidate_date = datetime.now(timezone.utc) - timedelta(days=day_offset) + date_path = candidate_date.strftime("%Y/%m/%d") + urls = [ + f"http://suntoday.lmsal.com/sdomedia/SunInTime/{date_path}/t{code}.jpg", + f"https://suntoday.lmsal.com/sdomedia/SunInTime/{date_path}/t{code}.jpg", + ] + for url in urls: + try: + response = self.session.head(url, timeout=15) + if response.status_code == 200: + return response.headers.get("Last-Modified") or candidate_date.strftime("%Y-%m-%d") + except requests.exceptions.RequestException: + continue + return None + + def _timestamp_from_jsoc(self, source: str) -> Optional[str]: + timestamp_key = SDO_SOURCES[source].get("jsoc_timestamp_key") + if not timestamp_key: + return None + + url = "https://jsoc1.stanford.edu/data/hmi/images/latest/image_times_UTC" + response = self.session.get(url, timeout=15) + response.raise_for_status() + + for line in response.text.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + if key.strip().lower() == timestamp_key: + return value.strip() + return None + + def _timestamp_from_nasa(self, source: str) -> Optional[str]: + nasa_code = SDO_SOURCES[source].get("nasa_code") + if not nasa_code: + return None + + urls = [ + f"http://sdo.gsfc.nasa.gov/assets/img/latest/latest_1024_{nasa_code}.jpg", + f"https://sdo.gsfc.nasa.gov/assets/img/latest/latest_1024_{nasa_code}.jpg", + ] + + for url in urls: + try: + response = self.session.head(url, timeout=15) + response.raise_for_status() + return response.headers.get("Last-Modified") + except requests.exceptions.RequestException: + continue + + return None + + def _timestamp_from_helioviewer(self, source: str) -> Optional[str]: + source_id = SDO_SOURCES[source]["sourceId"] + response = self.session.get( + "https://api.helioviewer.org/v2/getClosestImage/", + params={ + "date": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "sourceId": source_id, + }, + timeout=15, + ) + response.raise_for_status() + return response.json().get("date") + + def _save_response( + self, + response: requests.Response, + source: str, + provider: str, + image_url: str, + extension: str, + observation_time: Optional[str] = None, + extra_metadata: Optional[Dict] = None, + ) -> Dict: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + filepath = self.output_dir / f"SDO_{source}_{timestamp}{extension}" + + with open(filepath, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + metadata = { + "source": source, + "name": SDO_SOURCES[source]["name"], + "wavelength": SDO_SOURCES[source]["wavelength"], + "description": SDO_SOURCES[source]["description"], + "provider": provider, + "provider_name": PROVIDER_LABELS[provider], + "filepath": str(filepath), + "download_time": datetime.now(timezone.utc).isoformat(), + "image_url": image_url, + "observation_time": observation_time, + "content_type": response.headers.get("Content-Type"), + "last_modified": response.headers.get("Last-Modified"), + } + + if extra_metadata: + metadata.update(extra_metadata) + + metadata_file = filepath.with_suffix(".json") + with open(metadata_file, "w", encoding="utf-8") as f: + json.dump(metadata, f, indent=2) + + print(f"āœ“ Image saved: {filepath}") + print(f"āœ“ Metadata saved: {metadata_file}") + print(f"āœ“ Provider used: {PROVIDER_LABELS[provider]}") + + return metadata \ No newline at end of file