feat: improve caching mechanism for download_dir on ingest (Unstructured-IO#314)

* `unstructured-ingest` now uses a default `--download_dir` of `$HOME/.cache/unstructured/ingest`
rather than a "tmp-ingest-" dir in the working directory.
* `unstructured-ingest` no longer re-downloads files when --preserve-downloads
is used without --download-dir.

Author: alvarobartt

Diff for: .gitignore

@@ -131,8 +131,7 @@ dmypy.json
 
 # ingest outputs
 /structured-output
-# ingest temporary files
-/tmp-ingest*
+
 # suggested ingest mirror directory
 /mirror
 

Diff for: CHANGELOG.md

@@ -1,11 +1,17 @@
-## 0.5.2-dev1
+## 0.5.2-dev0
 
 ### Enhancements
 
+* `unstructured-ingest` now uses a default `--download_dir` of `$HOME/.cache/unstructured/ingest`
+rather than a "tmp-ingest-" dir in the working directory.
+
 ### Features
 
 ### Fixes
 
+* `unstructured-ingest` no longer re-downloads files when --preserve-downloads
+is used without --download-dir.
+
 ## 0.5.1
 
 ### Enhancements

Diff for: unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.2-dev1"  # pragma: no cover
+__version__ = "0.5.2-dev0"  # pragma: no cover

Diff for: unstructured/ingest/connector/s3_connector.py

@@ -175,7 +175,6 @@ def initialize(self):
             raise ValueError(
                 f"No objects found in {self.config.s3_url} -- response list object is {response}",
             )
-        os.mkdir(self.config.download_dir)
 
     def _list_objects(self):
         response = self.s3_cli.list_objects_v2(**self._list_objects_kwargs)

Diff for: unstructured/ingest/main.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
+import hashlib
 import multiprocessing as mp
-import random
-import string
 import sys
+from pathlib import Path
 
 import click
 
@@ -147,17 +147,17 @@ def run(self):
 @click.option(
     "--re-download/--no-re-download",
     default=False,
-    help="Re-download files from s3 even if they are already present in --download-dir.",
+    help="Re-download files even if they are already present in --download-dir.",
 )
 @click.option(
     "--download-dir",
-    help="Where s3 files are downloaded to, defaults to tmp-ingest-<6 random chars>.",
+    help="Where files are downloaded to, defaults to `$HOME/.cache/unstructured/ingest/<SHA256>`.",
 )
 @click.option(
     "--preserve-downloads",
     is_flag=True,
     default=False,
-    help="Preserve downloaded s3 files. Otherwise each file is removed after being processed "
+    help="Preserve downloaded files. Otherwise each file is removed after being processed "
     "successfully.",
 )
 @click.option(
@@ -169,7 +169,7 @@ def run(self):
     "--reprocess",
     is_flag=True,
     default=False,
-    help="Reprocess a downloaded file from s3 even if the relevant structured output .json file "
+    help="Reprocess a downloaded file even if the relevant structured output .json file "
     "in --structured-output-dir already exists.",
 )
 @click.option(
@@ -204,9 +204,31 @@ def main(
     if not preserve_downloads and download_dir:
         print("Warning: not preserving downloaded files but --download_dir is specified")
     if not download_dir:
-        download_dir = "tmp-ingest-" + "".join(
-            random.choice(string.ascii_letters) for i in range(6)
-        )
+        cache_path = Path.home() / ".cache" / "unstructured" / "ingest"
+        if not cache_path.exists():
+            cache_path.mkdir(parents=True, exist_ok=True)
+        if s3_url:
+            hashed_dir_name = hashlib.sha256(s3_url.encode("utf-8"))
+        elif github_url:
+            hashed_dir_name = hashlib.sha256(
+                f"{github_url}_{github_branch}".encode("utf-8"),
+            )
+        elif subreddit_name:
+            hashed_dir_name = hashlib.sha256(
+                subreddit_name.encode("utf-8"),
+            )
+        elif wikipedia_page_title:
+            hashed_dir_name = hashlib.sha256(
+                wikipedia_page_title.encode("utf-8"),
+            )
+        else:
+            raise ValueError("No connector-specific option was specified!")
+        download_dir = cache_path / hashed_dir_name.hexdigest()[:10]
+        if preserve_downloads:
+            print(
+                f"Warning: preserving downloaded files but --download-dir is not specified,"
+                f" using {download_dir}",
+            )
     if s3_url:
         doc_connector = S3Connector(
             config=SimpleS3Config(