diff --git a/README.rst b/README.rst index 27a4018f..bf243d19 100644 --- a/README.rst +++ b/README.rst @@ -1725,6 +1725,8 @@ You need to have the Blocks Conversion Tool (https://github.com/plone/blocks-con See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrate-to-volto.html for more details on the changes the migration to Volto does. +This code was used in real projects multiple times and is proven to work. +After the migration you need to restart the instance to make all changes work. .. code-block:: python @@ -1741,6 +1743,7 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat from plone.volto.setuphandlers import remove_behavior from Products.CMFPlone.utils import get_installer from Products.Five import BrowserView + from Products.ZCatalog.ProgressHandler import ZLogHandler from zope.interface import alsoProvides import requests @@ -1748,15 +1751,22 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat logger = getLogger(__name__) + # Add you own project-specific add-ons here DEFAULT_ADDONS = [] + VERSIONED_TYPES = [ + "Document", + "News Item", + "Event", + "Link", + ] + class ImportAll(BrowserView): def __call__(self): request = self.request - # Check if Blocks-conversion-tool is running headers = { "Accept": "application/json", @@ -1783,14 +1793,19 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat if not installer.is_product_installed(addon): installer.install_product(addon) + # Disable versioning before import + for portal_type in VERSIONED_TYPES: + remove_behavior(portal_type, "plone.versioning") + remove_behavior(portal_type, "plone.locking") + # Fake the target being a classic site even though plone.volto is installed... # 1. Allow Folders and Collections (they are disabled in Volto by default) portal_types = api.portal.get_tool("portal_types") portal_types["Collection"].global_allow = True portal_types["Folder"].global_allow = True # 2. Enable richtext behavior (otherwise no text will be imported) - for type_ in ["Document", "News Item", "Event"]: - add_behavior(type_, "plone.richtext") + for portal_type in ["Document", "News Item", "Event"]: + add_behavior(portal_type, "plone.richtext") transaction.commit() cfg = getConfiguration() @@ -1800,6 +1815,7 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat view = api.content.get_view("import_content", portal, request) request.form["form.submitted"] = True request.form["commit"] = 500 + # Change "Plone.json" to the name of your export file view(server_file="Plone.json", return_json=True) transaction.commit() @@ -1827,7 +1843,12 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat logger.info(f"Missing file: {path}") # Optional: Run html-fixers on richtext - fixers = [anchor_fixer] + fixers = [ + table_class_fixer, + img_variant_fixer, + scale_unscaled_images, + fix_image_align, + ] results = fix_html_in_content_fields(fixers=fixers) msg = "Fixed html for {} content items".format(results) logger.info(msg) @@ -1840,6 +1861,11 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat transaction.get().note(msg) transaction.commit() + # Add blocks behavior to collections to convert richtext to blocks + for portal_type in ["Collection"]: + add_behavior(portal_type, "volto.blocks") + + # Update linksintegrity view = api.content.get_view("updateLinkIntegrityInformation", portal, request) results = view.update() msg = f"Updated linkintegrity for {results} items" @@ -1858,16 +1884,18 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat # This uses the blocks-conversion-tool to migrate to blocks logger.info("Start migrating richtext to blocks...") - migrate_richtext_to_blocks() + migrate_richtext_to_blocks(purge_richtext=True) msg = "Finished migrating richtext to blocks" transaction.get().note(msg) transaction.commit() - # Reuse the migration-form from plon.volto to do some more tasks + # Reuse the migration-form from plone.volto to do some more tasks view = api.content.get_view("migrate_to_volto", portal, request) - # Yes, wen want to migrate default pages + # Yes, we want to migrate default pages view.migrate_default_pages = True view.slate = True + view.purge_richtext = True + view.service_url = "http://localhost:5000/html" logger.info("Start migrating Folders to Documents...") view.do_migrate_folders() msg = "Finished migrating Folders to Documents!" @@ -1884,6 +1912,34 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat reset_dates() transaction.commit() + # Reindex created and modified + catalog = api.portal.get_tool("portal_catalog") + pghandler = ZLogHandler(5000) + catalog.reindexIndex(["created", "modified"], None, pghandler=pghandler) + + # re-enable versioning and add initial versions + for portal_type in VERSIONED_TYPES: + add_behavior(portal_type, "plone.versioning") + add_behavior(portal_type, "plone.locking") + logger.info("Creating initial versions") + portal_repository = api.portal.get_tool("portal_repository") + brains = api.content.find(portal_type=VERSIONED_TYPES, sort_on="path") + total = len(brains) + for index, brain in enumerate(brains): + obj = brain.getObject() + try: + portal_repository.save(obj=obj, comment="Imported Version") + except FileTooLargeToVersionError: + pass + if not index % 1000: + msg = f"Created versions for {index} of {total} items." + logger.info(msg) + transaction.get().note(msg) + transaction.commit() + msg = "Created initial versions" + transaction.get().note(msg) + transaction.commit() + # Disallow folders and collections again portal_types["Collection"].global_allow = False portal_types["Folder"].global_allow = False @@ -1892,22 +1948,117 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat for type_ in ["Document", "News Item", "Event"]: remove_behavior(type_, "plone.richtext") + # Remove contentimport to also drop the BrowserLayer + if installer.is_product_installed("contentimport"): + installer.uninstall_product("contentimport") + + logger.info("Finished import_all") return request.response.redirect(portal.absolute_url()) - def anchor_fixer(text, obj=None): - """Remove anchors since they are not supported by Volto yet""" + def table_class_fixer(text, obj=None): + if "table" not in text: + return text + dropped_classes = [ + "MsoNormalTable", + "MsoTableGrid", + ] + replaced_classes = { + "invisible": "invisible-grid", + } + soup = BeautifulSoup(text, "html.parser") + for table in soup.find_all("table"): + table_classes = table.get("class", []) + for dropped in dropped_classes: + if dropped in table_classes: + table_classes.remove(dropped) + for old, new in replaced_classes.items(): + if old in table_classes: + table_classes.remove(old) + table_classes.append(new) + # all tables get the default bootstrap table class + if "table" not in table_classes: + table_classes.insert(0, "table") + + return soup.decode() + + + def img_variant_fixer(text, obj=None): + """Set image-variants""" + if not text: + return text + + picture_variants = api.portal.get_registry_record("plone.picture_variants") + scale_variant_mapping = { + k: v["sourceset"][0]["scale"] for k, v in picture_variants.items() + } + scale_variant_mapping["thumb"] = "mini" + fallback_variant = "preview" + + soup = BeautifulSoup(text, "html.parser") + for tag in soup.find_all("img"): + if "data-val" not in tag.attrs: + # maybe external image + continue + scale = tag["data-scale"] + variant = scale_variant_mapping.get(scale, fallback_variant) + tag["data-picturevariant"] = variant + + classes = tag["class"] + new_class = f"picture-variant-{variant}" + if new_class not in classes: + classes.append(new_class) + tag["class"] = classes + + return soup.decode() + + + def scale_unscaled_images(text, obj=None): + """Scale unscaled image""" + if not text: + return text + fallback_scale = "huge" + soup = BeautifulSoup(text, "html.parser") - for link in soup.find_all("a"): - if not link.get("href") and not link.text: - # drop empty links (e.g. anchors) - link.decompose() - elif not link.get("href") and link.text: - # drop links without a href but keep the text - link.unwrap() + for tag in soup.find_all("img"): + if "data-val" not in tag.attrs: + # maybe external image + continue + + scale = tag["data-scale"] + # Prevent unscaled images! + if not scale: + scale = fallback_scale + tag["data-scale"] = fallback_scale + if not tag["src"].endswith(scale): + tag["src"] = tag["src"] + "/" + scale + return soup.decode() + def fix_image_align(text, obj=None): + """Replace align='xx' with css-classes""" + if not text: + return text + + soup = BeautifulSoup(text, "html.parser") + for tag in soup.find_all("img"): + if "align" not in tag.attrs: + continue + + classes = tag.get("class", []) + direction = tag["align"] + if direction == "left": + classes.append("image-left") + elif direction == "right": + classes.append("image-right") + if "image-inline" in classes: + classes.remove("image-inline") + del tag["align"] + return soup.decode() + + + Migrate very old Plone Versions with data created by collective.jsonify -----------------------------------------------------------------------