update code-example doing the full migration to volto

README.rst

@@ -1725,6 +1725,8 @@ You need to have the Blocks Conversion Tool (https://github.com/plone/blocks-con
 
 See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrate-to-volto.html for more details on the changes the migration to Volto does.
 
+This code was used in real projects multiple times and is proven to work.
+After the migration you need to restart the instance to make all changes work.
 
 .. code-block:: python
 
@@ -1741,22 +1743,30 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
     from plone.volto.setuphandlers import remove_behavior
     from Products.CMFPlone.utils import get_installer
     from Products.Five import BrowserView
+    from Products.ZCatalog.ProgressHandler import ZLogHandler
     from zope.interface import alsoProvides
 
     import requests
     import transaction
 
     logger = getLogger(__name__)
 
+    # Add you own project-specific add-ons here
     DEFAULT_ADDONS = []
 
+    VERSIONED_TYPES = [
+        "Document",
+        "News Item",
+        "Event",
+        "Link",
+    ]
+
 
     class ImportAll(BrowserView):
 
         def __call__(self):
 
             request = self.request
-
             # Check if Blocks-conversion-tool is running
             headers = {
                 "Accept": "application/json",
@@ -1783,14 +1793,19 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
                 if not installer.is_product_installed(addon):
                     installer.install_product(addon)
 
+            # Disable versioning before import
+            for portal_type in VERSIONED_TYPES:
+                remove_behavior(portal_type, "plone.versioning")
+                remove_behavior(portal_type, "plone.locking")
+
             # Fake the target being a classic site even though plone.volto is installed...
             # 1. Allow Folders and Collections (they are disabled in Volto by default)
             portal_types = api.portal.get_tool("portal_types")
             portal_types["Collection"].global_allow = True
             portal_types["Folder"].global_allow = True
             # 2. Enable richtext behavior (otherwise no text will be imported)
-            for type_ in ["Document", "News Item", "Event"]:
-                add_behavior(type_, "plone.richtext")
+            for portal_type in ["Document", "News Item", "Event"]:
+                add_behavior(portal_type, "plone.richtext")
 
             transaction.commit()
             cfg = getConfiguration()
@@ -1800,6 +1815,7 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
             view = api.content.get_view("import_content", portal, request)
             request.form["form.submitted"] = True
             request.form["commit"] = 500
+            # Change "Plone.json" to the name of your export file
             view(server_file="Plone.json", return_json=True)
             transaction.commit()
 
@@ -1827,7 +1843,12 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
                     logger.info(f"Missing file: {path}")
 
             # Optional: Run html-fixers on richtext
-            fixers = [anchor_fixer]
+            fixers = [
+                table_class_fixer,
+                img_variant_fixer,
+                scale_unscaled_images,
+                fix_image_align,
+            ]
             results = fix_html_in_content_fields(fixers=fixers)
             msg = "Fixed html for {} content items".format(results)
             logger.info(msg)
@@ -1840,6 +1861,11 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
             transaction.get().note(msg)
             transaction.commit()
 
+            # Add blocks behavior to collections to convert richtext to blocks
+            for portal_type in ["Collection"]:
+                add_behavior(portal_type, "volto.blocks")
+
+            # Update linksintegrity
             view = api.content.get_view("updateLinkIntegrityInformation", portal, request)
             results = view.update()
             msg = f"Updated linkintegrity for {results} items"
@@ -1858,16 +1884,18 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
 
             # This uses the blocks-conversion-tool to migrate to blocks
             logger.info("Start migrating richtext to blocks...")
-            migrate_richtext_to_blocks()
+            migrate_richtext_to_blocks(purge_richtext=True)
             msg = "Finished migrating richtext to blocks"
             transaction.get().note(msg)
             transaction.commit()
 
-            # Reuse the migration-form from plon.volto to do some more tasks
+            # Reuse the migration-form from plone.volto to do some more tasks
             view = api.content.get_view("migrate_to_volto", portal, request)
-            # Yes, wen want to migrate default pages
+            # Yes, we want to migrate default pages
             view.migrate_default_pages = True
             view.slate = True
+            view.purge_richtext = True
+            view.service_url = "http://localhost:5000/html"
             logger.info("Start migrating Folders to Documents...")
             view.do_migrate_folders()
             msg = "Finished migrating Folders to Documents!"
@@ -1884,6 +1912,34 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
             reset_dates()
             transaction.commit()
 
+            # Reindex created and modified
+            catalog = api.portal.get_tool("portal_catalog")
+            pghandler = ZLogHandler(5000)
+            catalog.reindexIndex(["created", "modified"], None, pghandler=pghandler)
+
+            # re-enable versioning and add initial versions
+            for portal_type in VERSIONED_TYPES:
+                add_behavior(portal_type, "plone.versioning")
+                add_behavior(portal_type, "plone.locking")
+            logger.info("Creating initial versions")
+            portal_repository = api.portal.get_tool("portal_repository")
+            brains = api.content.find(portal_type=VERSIONED_TYPES, sort_on="path")
+            total = len(brains)
+            for index, brain in enumerate(brains):
+                obj = brain.getObject()
+                try:
+                    portal_repository.save(obj=obj, comment="Imported Version")
+                except FileTooLargeToVersionError:
+                    pass
+                if not index % 1000:
+                    msg = f"Created versions for {index} of {total} items."
+                    logger.info(msg)
+                    transaction.get().note(msg)
+                    transaction.commit()
+            msg = "Created initial versions"
+            transaction.get().note(msg)
+            transaction.commit()
+
             # Disallow folders and collections again
             portal_types["Collection"].global_allow = False
             portal_types["Folder"].global_allow = False
@@ -1892,22 +1948,117 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
             for type_ in ["Document", "News Item", "Event"]:
                 remove_behavior(type_, "plone.richtext")
 
+            # Remove contentimport to also drop the BrowserLayer
+            if installer.is_product_installed("contentimport"):
+                installer.uninstall_product("contentimport")
+
+                logger.info("Finished import_all")
             return request.response.redirect(portal.absolute_url())
 
 
-    def anchor_fixer(text, obj=None):
-        """Remove anchors since they are not supported by Volto yet"""
+    def table_class_fixer(text, obj=None):
+        if "table" not in text:
+            return text
+        dropped_classes = [
+            "MsoNormalTable",
+            "MsoTableGrid",
+        ]
+        replaced_classes = {
+            "invisible": "invisible-grid",
+        }
+        soup = BeautifulSoup(text, "html.parser")
+        for table in soup.find_all("table"):
+            table_classes = table.get("class", [])
+            for dropped in dropped_classes:
+                if dropped in table_classes:
+                    table_classes.remove(dropped)
+            for old, new in replaced_classes.items():
+                if old in table_classes:
+                    table_classes.remove(old)
+                    table_classes.append(new)
+            # all tables get the default bootstrap table class
+            if "table" not in table_classes:
+                table_classes.insert(0, "table")
+
+        return soup.decode()
+
+
+    def img_variant_fixer(text, obj=None):
+        """Set image-variants"""
+        if not text:
+            return text
+
+        picture_variants = api.portal.get_registry_record("plone.picture_variants")
+        scale_variant_mapping = {
+            k: v["sourceset"][0]["scale"] for k, v in picture_variants.items()
+        }
+        scale_variant_mapping["thumb"] = "mini"
+        fallback_variant = "preview"
+
+        soup = BeautifulSoup(text, "html.parser")
+        for tag in soup.find_all("img"):
+            if "data-val" not in tag.attrs:
+                # maybe external image
+                continue
+            scale = tag["data-scale"]
+            variant = scale_variant_mapping.get(scale, fallback_variant)
+            tag["data-picturevariant"] = variant
+
+            classes = tag["class"]
+            new_class = f"picture-variant-{variant}"
+            if new_class not in classes:
+                classes.append(new_class)
+                tag["class"] = classes
+
+        return soup.decode()
+
+
+    def scale_unscaled_images(text, obj=None):
+        """Scale unscaled image"""
+        if not text:
+            return text
+        fallback_scale = "huge"
+
         soup = BeautifulSoup(text, "html.parser")
-        for link in soup.find_all("a"):
-            if not link.get("href") and not link.text:
-                # drop empty links (e.g. anchors)
-                link.decompose()
-            elif not link.get("href") and link.text:
-                # drop links without a href but keep the text
-                link.unwrap()
+        for tag in soup.find_all("img"):
+            if "data-val" not in tag.attrs:
+                # maybe external image
+                continue
+
+            scale = tag["data-scale"]
+            # Prevent unscaled images!
+            if not scale:
+                scale = fallback_scale
+                tag["data-scale"] = fallback_scale
+            if not tag["src"].endswith(scale):
+                tag["src"] = tag["src"] + "/" + scale
+
         return soup.decode()
 
 
+    def fix_image_align(text, obj=None):
+        """Replace align='xx' with css-classes"""
+        if not text:
+            return text
+
+        soup = BeautifulSoup(text, "html.parser")
+        for tag in soup.find_all("img"):
+            if "align" not in tag.attrs:
+                continue
+
+            classes = tag.get("class", [])
+            direction = tag["align"]
+            if direction == "left":
+                classes.append("image-left")
+            elif direction == "right":
+                classes.append("image-right")
+            if "image-inline" in classes:
+                classes.remove("image-inline")
+            del tag["align"]
+        return soup.decode()
+
+
+
 Migrate very old Plone Versions with data created by collective.jsonify
 -----------------------------------------------------------------------