From 22953f84fa74c334cacbc6ec24c3a9eb87b4f5c8 Mon Sep 17 00:00:00 2001 From: Philip Bauer Date: Sat, 7 Oct 2023 09:22:10 +0200 Subject: [PATCH 1/3] Start allowing export & import to have one json-file per content item --- src/collective/exportimport/export_content.py | 47 ++++++++++++++++- src/collective/exportimport/import_content.py | 50 ++++++++++++++++++- .../exportimport/templates/export_content.pt | 6 +++ .../exportimport/templates/import_content.pt | 28 ++++++++--- 4 files changed, 123 insertions(+), 8 deletions(-) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index 0c9135b3..089e0ebe 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -181,7 +181,50 @@ def __call__( content_generator = self.export_content() number = 0 - if download_to_server: + + # Export each item to a separate json-file + if download_to_server == 2: + directory = config.CENTRAL_DIRECTORY + if directory: + if not os.path.exists(directory): + os.makedirs(directory) + logger.info("Created central export/import directory %s", directory) + else: + cfg = getConfiguration() + directory = cfg.clienthome + + # Use the filename (Plone.json) as target for files (Plone/1.json) + directory = os.path.join(directory, filename[:-5]) + if not os.path.exists(directory): + os.makedirs(directory) + logger.info("Created directory to hold content: %s", directory) + + self.start() + for number, datum in enumerate(content_generator, start=1): + filename = "{}.json".format(number) + filepath = os.path.join(directory, filename) + with open(filepath, "w") as f: + json.dump(datum, f, sort_keys=True, indent=4) + if number: + if self.errors and self.write_errors: + errors = {"unexported_paths": self.errors} + json.dump(errors, f, indent=4) + msg = _(u"Exported {} items ({}) to {} with {} errors").format( + number, ", ".join(self.portal_type), directory, len(self.errors) + ) + logger.info(msg) + api.portal.show_message(msg, self.request) + + if self.include_blobs == 1: + # remove marker interface + noLongerProvides(self.request, IBase64BlobsMarker) + elif self.include_blobs == 2: + noLongerProvides(self.request, IPathBlobsMarker) + self.finish() + self.request.response.redirect(self.request["ACTUAL_URL"]) + + # Export all items into one json-file in the filesystem + elif download_to_server: directory = config.CENTRAL_DIRECTORY if directory: if not os.path.exists(directory): @@ -218,6 +261,8 @@ def __call__( noLongerProvides(self.request, IPathBlobsMarker) self.finish() self.request.response.redirect(self.request["ACTUAL_URL"]) + + # Export as one json-file through the browser else: with tempfile.TemporaryFile(mode="w+") as f: self.start() diff --git a/src/collective/exportimport/import_content.py b/src/collective/exportimport/import_content.py index d2ce0fc4..75d77870 100644 --- a/src/collective/exportimport/import_content.py +++ b/src/collective/exportimport/import_content.py @@ -7,6 +7,7 @@ from DateTime import DateTime from datetime import timedelta from Persistence import PersistentMapping +from pathlib import Path from plone import api from plone.api.exc import InvalidParameterError from plone.dexterity.interfaces import IDexterityFTI @@ -73,6 +74,31 @@ def get_absolute_blob_path(obj, blob_path): return abs_path +def filesystem_walker(path=None): + root = Path(path) + assert(root.is_dir()) + + # first import json-files directly in the path + json_files = [i for i in root.glob("*.json") if i.stem.isdecimal()] + for json_file in sorted(json_files, key=lambda i: int(i.stem)): + logger.debug("Importing %s", json_file) + item = json.loads(json_file.read_text()) + item["json_file"] = str(json_file) + if item: + yield item + + # then import json-files of any containing folders + folders = [i for i in root.iterdir() if i.is_dir() and i.name.isdecimal()] + for folder in sorted(folders, key=lambda i: int(i.name)): + json_files = [i for i in folder.glob("*.json") if i.stem.isdecimal()] + for json_file in sorted(json_files, key=lambda i: int(i.stem)): + logger.debug("Importing %s", json_file) + item = json.loads(json_file.read_text()) + item["json_file"] = str(json_file) + if item: + yield item + + class ImportContent(BrowserView): template = ViewPageTemplateFile("templates/import_content.pt") @@ -112,7 +138,8 @@ def __call__( return_json=False, limit=None, server_file=None, - iterator=None + iterator=None, + server_directory=False, ): request = self.request self.limit = limit @@ -192,6 +219,11 @@ def __call__( msg = self.do_import(iterator) api.portal.show_message(msg, self.request) + if server_directory: + self.start() + msg = self.do_import(filesystem_walker(server_directory)) + api.portal.show_message(msg, self.request) + self.finish() if return_json: @@ -243,6 +275,22 @@ def server_files(self): listing.sort() return listing + @property + def server_directories(self): + # Adapted from ObjectManager.list_imports, which lists zexps. + listing = [] + for directory in self.import_paths: + if not os.path.isdir(directory): + continue + # import pdb; pdb.set_trace() + listing += [ + os.path.join(directory, f) + for f in os.listdir(directory) + if os.path.isdir(os.path.join(directory, f)) and f not in listing + ] + listing.sort() + return listing + def do_import(self, data): start = datetime.now() alsoProvides(self.request, IMigrationMarker) diff --git a/src/collective/exportimport/templates/export_content.pt b/src/collective/exportimport/templates/export_content.pt index a7672239..2b61501e 100644 --- a/src/collective/exportimport/templates/export_content.pt +++ b/src/collective/exportimport/templates/export_content.pt @@ -162,6 +162,12 @@ Save to file on server +
+ + +
diff --git a/src/collective/exportimport/templates/import_content.pt b/src/collective/exportimport/templates/import_content.pt index dab24813..63c5f423 100644 --- a/src/collective/exportimport/templates/import_content.pt +++ b/src/collective/exportimport/templates/import_content.pt @@ -17,14 +17,16 @@
+

You can also select a json-file or a directory holding json-files on the server in the following locations:

+ + -

Or you can choose a file that is already uploaded on the server in one of these paths:

-
    -
    • -
-

No files found.

+

Import from a json-file

+

No json-files found.

- +
+ + + +
+ +
From 9ee5b89cf913cf56575b80d1a646e9031427d960 Mon Sep 17 00:00:00 2001 From: Philip Bauer Date: Sat, 7 Oct 2023 09:45:23 +0200 Subject: [PATCH 2/3] try pathlib2 in py2 --- base.cfg | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/base.cfg b/base.cfg index dcc4d471..db378a0d 100644 --- a/base.cfg +++ b/base.cfg @@ -53,3 +53,4 @@ pip = [versions:python27] # Last pyrsistent version that is python 2 compatible: pyrsistent = 0.15.7 +pathlib2 = 2.3.6 diff --git a/setup.py b/setup.py index 6980e329..a1181807 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ # which has a Py3-only release since September 2021. install_requires.append("jsonschema < 4") install_requires.append("pyrsistent < 0.16.0") + install_requires.append("pathlib2") else: install_requires.append("plone.restapi") install_requires.append("beautifulsoup4") From 59d90e2f6ba7f78bdafd7a26bc08f4f6ae5deb6a Mon Sep 17 00:00:00 2001 From: Philip Bauer Date: Sun, 8 Oct 2023 10:32:31 +0200 Subject: [PATCH 3/3] fix pathlib2 import --- src/collective/exportimport/import_content.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/collective/exportimport/import_content.py b/src/collective/exportimport/import_content.py index 75d77870..87c88445 100644 --- a/src/collective/exportimport/import_content.py +++ b/src/collective/exportimport/import_content.py @@ -7,7 +7,6 @@ from DateTime import DateTime from datetime import timedelta from Persistence import PersistentMapping -from pathlib import Path from plone import api from plone.api.exc import InvalidParameterError from plone.dexterity.interfaces import IDexterityFTI @@ -44,6 +43,11 @@ except ImportError: HAS_COLLECTION_FIX = False +if six.PY2: + from pathlib2 import Path +else: + from pathlib import Path + logger = logging.getLogger(__name__) BLOB_HOME = os.getenv("COLLECTIVE_EXPORTIMPORT_BLOB_HOME", "")