Merge pull request #223 from collective/separate_files

base.cfg

@@ -53,3 +53,4 @@ pip =
 [versions:python27]
 # Last pyrsistent version that is python 2 compatible:
 pyrsistent = 0.15.7
+pathlib2 = 2.3.6

setup.py

@@ -32,6 +32,7 @@
     # which has a Py3-only release since September 2021.
     install_requires.append("jsonschema < 4")
     install_requires.append("pyrsistent < 0.16.0")
+    install_requires.append("pathlib2")
 else:
     install_requires.append("plone.restapi")
     install_requires.append("beautifulsoup4")

src/collective/exportimport/export_content.py

@@ -181,7 +181,50 @@ def __call__(
         content_generator = self.export_content()
 
         number = 0
-        if download_to_server:
+
+        # Export each item to a separate json-file
+        if download_to_server == 2:
+            directory = config.CENTRAL_DIRECTORY
+            if directory:
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+                    logger.info("Created central export/import directory %s", directory)
+            else:
+                cfg = getConfiguration()
+                directory = cfg.clienthome
+
+            # Use the filename (Plone.json) as target for files (Plone/1.json)
+            directory = os.path.join(directory, filename[:-5])
+            if not os.path.exists(directory):
+                os.makedirs(directory)
+                logger.info("Created directory to hold content: %s", directory)
+
+            self.start()
+            for number, datum in enumerate(content_generator, start=1):
+                filename = "{}.json".format(number)
+                filepath = os.path.join(directory, filename)
+                with open(filepath, "w") as f:
+                    json.dump(datum, f, sort_keys=True, indent=4)
+            if number:
+                if self.errors and self.write_errors:
+                    errors = {"unexported_paths": self.errors}
+                    json.dump(errors, f, indent=4)
+            msg = _(u"Exported {} items ({}) to {} with {} errors").format(
+                number, ", ".join(self.portal_type), directory, len(self.errors)
+            )
+            logger.info(msg)
+            api.portal.show_message(msg, self.request)
+
+            if self.include_blobs == 1:
+                # remove marker interface
+                noLongerProvides(self.request, IBase64BlobsMarker)
+            elif self.include_blobs == 2:
+                noLongerProvides(self.request, IPathBlobsMarker)
+            self.finish()
+            self.request.response.redirect(self.request["ACTUAL_URL"])
+
+        # Export all items into one json-file in the filesystem
+        elif download_to_server:
             directory = config.CENTRAL_DIRECTORY
             if directory:
                 if not os.path.exists(directory):
@@ -218,6 +261,8 @@ def __call__(
                 noLongerProvides(self.request, IPathBlobsMarker)
             self.finish()
             self.request.response.redirect(self.request["ACTUAL_URL"])
+
+        # Export as one json-file through the browser
         else:
             with tempfile.TemporaryFile(mode="w+") as f:
                 self.start()

src/collective/exportimport/import_content.py

@@ -43,6 +43,11 @@
 except ImportError:
     HAS_COLLECTION_FIX = False
 
+if six.PY2:
+    from pathlib2 import Path
+else:
+    from pathlib import Path
+
 logger = logging.getLogger(__name__)
 BLOB_HOME = os.getenv("COLLECTIVE_EXPORTIMPORT_BLOB_HOME", "")
 
@@ -73,6 +78,31 @@ def get_absolute_blob_path(obj, blob_path):
         return abs_path
 
 
+def filesystem_walker(path=None):
+    root = Path(path)
+    assert(root.is_dir())
+
+    # first import json-files directly in the path
+    json_files = [i for i in root.glob("*.json") if i.stem.isdecimal()]
+    for json_file in sorted(json_files, key=lambda i: int(i.stem)):
+        logger.debug("Importing %s", json_file)
+        item = json.loads(json_file.read_text())
+        item["json_file"] = str(json_file)
+        if item:
+            yield item
+
+    # then import json-files of any containing folders
+    folders = [i for i in root.iterdir() if i.is_dir() and i.name.isdecimal()]
+    for folder in sorted(folders, key=lambda i: int(i.name)):
+        json_files = [i for i in folder.glob("*.json") if i.stem.isdecimal()]
+        for json_file in sorted(json_files, key=lambda i: int(i.stem)):
+            logger.debug("Importing %s", json_file)
+            item = json.loads(json_file.read_text())
+            item["json_file"] = str(json_file)
+            if item:
+                yield item
+
+
 class ImportContent(BrowserView):
 
     template = ViewPageTemplateFile("templates/import_content.pt")
@@ -112,7 +142,8 @@ def __call__(
         return_json=False,
         limit=None,
         server_file=None,
-        iterator=None
+        iterator=None,
+        server_directory=False,
     ):
         request = self.request
         self.limit = limit
@@ -192,6 +223,11 @@ def __call__(
             msg = self.do_import(iterator)
             api.portal.show_message(msg, self.request)
 
+        if server_directory:
+            self.start()
+            msg = self.do_import(filesystem_walker(server_directory))
+            api.portal.show_message(msg, self.request)
+
         self.finish()
 
         if return_json:
@@ -243,6 +279,22 @@ def server_files(self):
         listing.sort()
         return listing
 
+    @property
+    def server_directories(self):
+        # Adapted from ObjectManager.list_imports, which lists zexps.
+        listing = []
+        for directory in self.import_paths:
+            if not os.path.isdir(directory):
+                continue
+            # import pdb; pdb.set_trace()
+            listing += [
+                os.path.join(directory, f)
+                for f in os.listdir(directory)
+                if os.path.isdir(os.path.join(directory, f)) and f not in listing
+            ]
+        listing.sort()
+        return listing
+
     def do_import(self, data):
         start = datetime.now()
         alsoProvides(self.request, IMigrationMarker)

src/collective/exportimport/templates/export_content.pt

@@ -162,6 +162,12 @@
                   Save to file on server
                 </label>
               </div>
+              <div class="form-check">
+                <input class="form-check-input" type="radio" name="download_to_server:int" value="2" id="separate_files">
+                <label for="separate_files" class="form-check-label" i18n:translate="">
+                  Save each item as a separate file on the server
+                </label>
+              </div>
             </div>
 
 

src/collective/exportimport/templates/import_content.pt

@@ -17,14 +17,16 @@
                 <input type="file" name="jsonfile"/><br/>
             </div>
 
+            <p>You can also select a json-file or a directory holding json-files on the server in the following locations:</p>
+            <ul>
+              <li tal:repeat="path view/import_paths"><code tal:content="path" /></li>
+            </ul>
+
             <tal:block define="server_files view/server_files">
-              <p i18n:translate="server_paths_list">Or you can choose a file that is already uploaded on the server in one of these paths:</p>
-              <ul>
-                <li tal:repeat="path view/import_paths"><code tal:content="path" /></li>
-              </ul>
-              <p tal:condition="not:server_files" i18n:translate="">No files found.</p>
+              <p i18n:translate="server_paths_list">Import from a json-file</p>
+              <p tal:condition="not:server_files" i18n:translate="">No json-files found.</p>
               <div class="field mb-3" tal:condition="server_files">
-                <label for="server_file" i18n:translate="">File on server to import:</label>
+                <label for="server_file" i18n:translate="">json-file on server to import:</label>
                 <br />
                 <select id="server_file" name="server_file">
                   <option selected="" value="" title="" i18n:translate="">Choose one</option>
@@ -34,6 +36,20 @@
               </div>
             </tal:block>
 
+            <tal:block define="server_directories view/server_directories">
+              <p i18n:translate="server_paths_list">Import from a directory that holds individual json-files per item:</p>
+              <p tal:condition="not:server_directories" i18n:translate="">No directories to import from found.</p>
+              <div class="field mb-3" tal:condition="server_directories">
+                <label for="server_directory" i18n:translate="">Directory on server to import:</label>
+                <br />
+                <select id="server_directory" name="server_directory">
+                  <option selected="" value="" title="" i18n:translate="">Choose one</option>
+                  <option tal:repeat="directoryname server_directories" tal:content="directoryname" tal:attributes="value directoryname">
+                  </option>
+                </select>
+              </div>
+            </tal:block>
+
             <div class="field mb-3">
               <label for="include_blobs" i18n:translate="">Handle existing content</label>
               <span class="formHelp" i18n:translate="">