remove upload for cache, show url

Author: marcorudolphflex

tests/test_web/test_simulation_cache.py

@@ -218,9 +218,7 @@ def _test_job_run_cache(monkeypatch, basic_simulation):
     assert counters["download"] == 0
 
 
-@pytest.mark.parametrize("structure_key", ["polyslab"])
-@pytest.mark.parametrize("monitor_key", ["mode"])
-def _test_autograd_cache(monkeypatch, structure_key, monitor_key):
+def _test_autograd_cache(monkeypatch):
     counters = _patch_run_pipeline(monkeypatch)
     cache = resolve_simulation_cache(use_cache=True)
     cache.clear()
@@ -349,4 +347,4 @@ def test_cache_end_to_end(monkeypatch, tmp_path, tmp_path_factory, basic_simulat
     _test_cache_eviction_by_size(monkeypatch, tmp_path_factory, basic_simulation)
     _test_run_cache_hit_async(monkeypatch, basic_simulation, tmp_path)
     _test_job_run_cache(monkeypatch, basic_simulation)
-    _test_autograd_cache(monkeypatch, basic_simulation)
+    _test_autograd_cache(monkeypatch)

tidy3d/web/api/container.py

@@ -566,6 +566,7 @@ class BatchData(Tidy3dBaseModel, Mapping):
     verbose: bool = pd.Field(
         True, title="Verbose", description="Whether to print info messages and progressbars."
     )
+
     cached_tasks: Optional[dict[TaskName, bool]] = pd.Field(
         None,
         title="Cached Tasks",
@@ -578,6 +579,13 @@ class BatchData(Tidy3dBaseModel, Mapping):
         description="Whether to use local cache for retrieving Simulation results.",
     )
 
+    is_downloaded: Optional[bool] = pd.Field(
+        False,
+        title="Is Downloaded",
+        description="Whether the simulation data was downloaded before.",
+    )
+
+
     def load_sim_data(self, task_name: str) -> WorkflowDataType:
         """Load a simulation data object from file by task name."""
         task_data_path = self.task_paths[task_name]
@@ -592,7 +600,7 @@ def load_sim_data(self, task_name: str) -> WorkflowDataType:
             verbose=False,
             from_cache=from_cache,
             use_cache=self.use_cache,
-            replace_existing=False,
+            replace_existing=not (from_cache or self.is_downloaded),
         )
 
     def __getitem__(self, task_name: TaskName) -> WorkflowDataType:
@@ -1207,21 +1215,23 @@ def load(self, path_dir: str = DEFAULT_DATA_DIR, replace_existing: bool = False)
             task_ids[task_name] = self.jobs[task_name].task_id
 
         loaded = {task_name: job.load_if_cached for task_name, job in self.jobs.items()}
+
+        self.download(path_dir=path_dir, replace_existing=replace_existing)
+
         data = BatchData(
             task_paths=task_paths,
             task_ids=task_ids,
             verbose=self.verbose,
             cached_tasks=loaded,
             use_cache=self.use_cache,
+            is_downloaded=True,
         )
 
         for task_name, job in self.jobs.items():
             if isinstance(job.simulation, ModeSolver):
                 job_data = data[task_name]
                 job.simulation._patch_data(data=job_data)
 
-        self.download(path_dir=path_dir, replace_existing=replace_existing)
-
         return data
 
     def delete(self) -> None:

tidy3d/web/api/webapi.py

@@ -139,16 +139,26 @@ def restore_simulation_if_cached(
     path: str,
     use_cache: Optional[bool] = None,
     reduce_simulation: Literal["auto", True, False] = "auto",
+    verbose: bool = True,
 ) -> bool:
     simulation_cache = resolve_simulation_cache(use_cache)
     copied_from_cache = False
     if simulation_cache is not None:
         sim_for_cache = simulation
         if isinstance(simulation, (ModeSolver, ModeSimulation)):
             sim_for_cache = get_reduced_simulation(simulation, reduce_simulation)
-        entry = simulation_cache.try_fetch(simulation=sim_for_cache)
+        entry = simulation_cache.try_fetch(simulation=sim_for_cache, verbose=verbose)
         if entry is not None:
             copied_from_cache = _copy_simulation_data_from_cache_entry(entry, path)
+            cached_task_id = entry.metadata.get("task_id")
+            cached_workflow_type = entry.metadata.get("workflow_type")
+            if cached_task_id is not None and cached_workflow_type is not None and verbose:
+                console = get_logging_console() if verbose else None
+                url, _ = _get_task_urls(
+                        cached_workflow_type,
+                        simulation,
+                        cached_task_id)
+                console.log(f"Loaded simulation from local cache.\nView cached task using web UI at [link={url}]'{url}'[/link].")
     return copied_from_cache
 
 
@@ -278,7 +288,7 @@ def run(
         Monitor progress of each of the running tasks.
     """
     copied_from_cache = restore_simulation_if_cached(
-        simulation=simulation, path=path, use_cache=use_cache, reduce_simulation=reduce_simulation
+        simulation=simulation, path=path, use_cache=use_cache, reduce_simulation=reduce_simulation, verbose=verbose
     )
 
     if not copied_from_cache:
@@ -320,6 +330,26 @@ def run(
         simulation._patch_data(data=data)
     return data
 
+def _get_task_urls(
+    task_type: str,
+    simulation: WorkflowType,
+    resource_id: str,
+    folder_id: Optional[str] = None,
+    group_id: Optional[str] = None,
+) -> tuple[str, Optional[str]]:
+    """Log task and folder links to the web UI."""
+    print("task_type:", task_type)
+    if (task_type in ["RF", "COMPONENT_MODELER", "TERMINAL_COMPONENT_MODELER"]) and isinstance(simulation, TerminalComponentModeler):
+        url = _get_url_rf(group_id or resource_id)
+    else:
+        url = _get_url(resource_id)
+
+    if folder_id is not None:
+        folder_url = _get_folder_url(folder_id)
+    else:
+        folder_url = None
+    return url, folder_url
+
 
 @wait_for_connection
 def upload(
@@ -441,16 +471,9 @@ def upload(
                 f"Cost of {solver_name} simulations is subject to change in the future."
             )
         if task_type in GUI_SUPPORTED_TASK_TYPES:
-            if (task_type == "RF") and (isinstance(simulation, TerminalComponentModeler)):
-                url = _get_url_rf(group_id or resource_id)
-                folder_url = _get_folder_url(task.folder_id)
-                console.log(f"View task using web UI at [link={url}]'{url}'[/link].")
-                console.log(f"Task folder: [link={folder_url}]'{task.folder_name}'[/link].")
-            else:
-                url = _get_url(resource_id)
-                folder_url = _get_folder_url(task.folder_id)
-                console.log(f"View task using web UI at [link={url}]'{url}'[/link].")
-                console.log(f"Task folder: [link={folder_url}]'{task.folder_name}'[/link].")
+            url, folder_url = _get_task_urls(task_type, simulation, resource_id, task.folder_id, group_id)
+            console.log(f"View task using web UI at [link={url}]'{url}'[/link].")
+            console.log(f"Task folder: [link={folder_url}]'{task.folder_name}'[/link].")
 
     remote_sim_file = SIM_FILE_HDF5_GZ
     if task_type == "MODE_SOLVER":

tidy3d/web/cache.py

@@ -301,36 +301,20 @@ def _fetch(self, key: str) -> Optional[CacheEntry]:
             self._touch(entry)
             return entry
 
-    def fetch_by_task(self, task_id: str) -> Optional[CacheEntry]:
-        """Retrieve an entry by task id."""
-        with self._lock:
-            for entry in self._iter_entries():
-                metadata = entry.metadata
-                task_ids = metadata.get("task_ids", [])
-                if task_id in task_ids and entry.exists():
-                    if not entry.verify():
-                        self._remove_entry(entry)
-                        return None
-                    self._touch(entry)
-                    return entry
-        return None
-
     def __len__(self) -> int:
         """Return number of valid cache entries."""
         with self._lock:
             return sum(1 for _ in self._iter_entries())
 
     def _store(
-        self, key: str, task_id: Optional[str], source_path: Path, metadata: dict[str, Any]
+        self, key: str, source_path: Path, metadata: dict[str, Any]
     ) -> Optional[CacheEntry]:
         """Store a new cache entry from ``source_path``.
 
         Parameters
         ----------
         key : str
             Cache key computed from simulation hash and runtime context.
-        task_id : str, optional
-            Server task id associated with this artifact.
         source_path : Path
             Location of the artifact to cache.
         metadata : dict[str, Any]
@@ -358,11 +342,6 @@ def _store(
         metadata["last_used"] = now_iso
         metadata["checksum"] = checksum
         metadata["file_size"] = file_size
-        if task_id:
-            task_ids = list(metadata.get("task_ids", []))
-            if task_id not in task_ids:
-                task_ids.append(task_id)
-            metadata["task_ids"] = task_ids
 
         _write_metadata(tmp_meta, metadata)
         try:
@@ -508,7 +487,7 @@ def try_fetch(
                 return None
             if verbose:
                 log.info(
-                    "Simulation cache hit for workflow '%s'; using local results.", workflow_type
+                    f"Simulation cache hit for workflow '{workflow_type}'; using local results."
                 )
 
             return entry
@@ -543,16 +522,13 @@ def store_result(
             metadata = build_entry_metadata(
                 simulation_hash=simulation_hash,
                 workflow_type=workflow_type,
-                runtime_context={
-                    "task_id": task_id,
-                },
+                task_id=task_id,
                 version=version,
-                extras={"path": str(Path(path))},
+                path=Path(path),
             )
 
             self._store(
                 key=cache_key,
-                task_id=task_id,  # keeps a reverse link for legacy fetch_by_task
                 source_path=Path(path),
                 metadata=metadata,
             )
@@ -672,19 +648,17 @@ def build_entry_metadata(
     *,
     simulation_hash: str,
     workflow_type: str,
-    runtime_context: dict[str, Any],
+    task_id: str,
     version: str,
-    extras: Optional[dict[str, Any]] = None,
+    path: Path,
 ) -> dict[str, Any]:
     """Create metadata dictionary for a cache entry."""
 
     metadata: dict[str, Any] = {
         "simulation_hash": simulation_hash,
         "workflow_type": workflow_type,
-        "runtime_context": _canonicalize(runtime_context),
         "versions": _canonicalize(version),
-        "task_ids": [],
+        "task_id": task_id,
+        "path": str(path),
     }
-    if extras:
-        metadata.update(_canonicalize(extras))
     return metadata