Skip to content

Commit

Permalink
Merge branch 'main' into fix-unboundlocalerror
Browse files Browse the repository at this point in the history
  • Loading branch information
pbauer authored Nov 12, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents d9df27b + e1fada2 commit 1f61856
Showing 6 changed files with 210 additions and 29 deletions.
8 changes: 8 additions & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
@@ -7,11 +7,19 @@ Changelog

- Add and run a black version, that is compatible with Python 2.
[pgrunewald]

- Fix ``AtributeError: 'NamedImage' object has no attribute '_blob'`` similar to #236.
[petschki]

- Make zcml conditions more specific so that migration aliases don't trigger them
[reinhardt]

- Fix several ``UnboundLocalError`` exceptions when doing automated import
with ``return_json=True``
[petschki]

- Suppress events when importing ordering to not trigger ContainerModifiedEvent.
[pbauer]

1.12 (2024-03-08)
-----------------
187 changes: 170 additions & 17 deletions README.rst
Original file line number Diff line number Diff line change
@@ -854,7 +854,8 @@ Re-enable versioning and create initial versions after all imports and fixes are
for index, brain in enumerate(brains):
obj = brain.getObject()
try:
portal_repository.save(obj=obj, comment="Imported Version")
if not portal_repository.getHistoryMetadata(obj=obj):
portal_repository.save(obj=obj, comment="Imported Version")
except FileTooLargeToVersionError:
pass
if not index % 1000:
@@ -1725,6 +1726,8 @@ You need to have the Blocks Conversion Tool (https://github.com/plone/blocks-con

See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrate-to-volto.html for more details on the changes the migration to Volto does.

This code was used in real projects multiple times and is proven to work.
After the migration you need to restart the instance to make all changes work.

.. code-block:: python
@@ -1741,22 +1744,30 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
from plone.volto.setuphandlers import remove_behavior
from Products.CMFPlone.utils import get_installer
from Products.Five import BrowserView
from Products.ZCatalog.ProgressHandler import ZLogHandler
from zope.interface import alsoProvides
import requests
import transaction
logger = getLogger(__name__)
# Add you own project-specific add-ons here
DEFAULT_ADDONS = []
VERSIONED_TYPES = [
"Document",
"News Item",
"Event",
"Link",
]
class ImportAll(BrowserView):
def __call__(self):
request = self.request
# Check if Blocks-conversion-tool is running
headers = {
"Accept": "application/json",
@@ -1783,14 +1794,19 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
if not installer.is_product_installed(addon):
installer.install_product(addon)
# Disable versioning before import
for portal_type in VERSIONED_TYPES:
remove_behavior(portal_type, "plone.versioning")
remove_behavior(portal_type, "plone.locking")
# Fake the target being a classic site even though plone.volto is installed...
# 1. Allow Folders and Collections (they are disabled in Volto by default)
portal_types = api.portal.get_tool("portal_types")
portal_types["Collection"].global_allow = True
portal_types["Folder"].global_allow = True
# 2. Enable richtext behavior (otherwise no text will be imported)
for type_ in ["Document", "News Item", "Event"]:
add_behavior(type_, "plone.richtext")
for portal_type in ["Document", "News Item", "Event"]:
add_behavior(portal_type, "plone.richtext")
transaction.commit()
cfg = getConfiguration()
@@ -1800,6 +1816,7 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
view = api.content.get_view("import_content", portal, request)
request.form["form.submitted"] = True
request.form["commit"] = 500
# Change "Plone.json" to the name of your export file
view(server_file="Plone.json", return_json=True)
transaction.commit()
@@ -1827,7 +1844,12 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
logger.info(f"Missing file: {path}")
# Optional: Run html-fixers on richtext
fixers = [anchor_fixer]
fixers = [
table_class_fixer,
img_variant_fixer,
scale_unscaled_images,
fix_image_align,
]
results = fix_html_in_content_fields(fixers=fixers)
msg = "Fixed html for {} content items".format(results)
logger.info(msg)
@@ -1840,6 +1862,11 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
transaction.get().note(msg)
transaction.commit()
# Add blocks behavior to collections to convert richtext to blocks
for portal_type in ["Collection"]:
add_behavior(portal_type, "volto.blocks")
# Update linksintegrity
view = api.content.get_view("updateLinkIntegrityInformation", portal, request)
results = view.update()
msg = f"Updated linkintegrity for {results} items"
@@ -1858,16 +1885,18 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
# This uses the blocks-conversion-tool to migrate to blocks
logger.info("Start migrating richtext to blocks...")
migrate_richtext_to_blocks()
migrate_richtext_to_blocks(purge_richtext=True)
msg = "Finished migrating richtext to blocks"
transaction.get().note(msg)
transaction.commit()
# Reuse the migration-form from plon.volto to do some more tasks
# Reuse the migration-form from plone.volto to do some more tasks
view = api.content.get_view("migrate_to_volto", portal, request)
# Yes, wen want to migrate default pages
# Yes, we want to migrate default pages
view.migrate_default_pages = True
view.slate = True
view.purge_richtext = True
view.service_url = "http://localhost:5000/html"
logger.info("Start migrating Folders to Documents...")
view.do_migrate_folders()
msg = "Finished migrating Folders to Documents!"
@@ -1884,6 +1913,35 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
reset_dates()
transaction.commit()
# Reindex created and modified
catalog = api.portal.get_tool("portal_catalog")
pghandler = ZLogHandler(5000)
catalog.reindexIndex(["created", "modified"], None, pghandler=pghandler)
# re-enable versioning and add initial versions
for portal_type in VERSIONED_TYPES:
add_behavior(portal_type, "plone.versioning")
add_behavior(portal_type, "plone.locking")
logger.info("Creating initial versions")
portal_repository = api.portal.get_tool("portal_repository")
brains = api.content.find(portal_type=VERSIONED_TYPES, sort_on="path")
total = len(brains)
for index, brain in enumerate(brains):
obj = brain.getObject()
try:
if not portal_repository.getHistoryMetadata(obj=obj):
portal_repository.save(obj=obj, comment="Imported Version")
except FileTooLargeToVersionError:
pass
if not index % 1000:
msg = f"Created versions for {index} of {total} items."
logger.info(msg)
transaction.get().note(msg)
transaction.commit()
msg = "Created initial versions"
transaction.get().note(msg)
transaction.commit()
# Disallow folders and collections again
portal_types["Collection"].global_allow = False
portal_types["Folder"].global_allow = False
@@ -1892,22 +1950,117 @@ See https://6.docs.plone.org/backend/upgrading/version-specific-migration/migrat
for type_ in ["Document", "News Item", "Event"]:
remove_behavior(type_, "plone.richtext")
# Remove contentimport to also drop the BrowserLayer
if installer.is_product_installed("contentimport"):
installer.uninstall_product("contentimport")
logger.info("Finished import_all")
return request.response.redirect(portal.absolute_url())
def anchor_fixer(text, obj=None):
"""Remove anchors since they are not supported by Volto yet"""
def table_class_fixer(text, obj=None):
if "table" not in text:
return text
dropped_classes = [
"MsoNormalTable",
"MsoTableGrid",
]
replaced_classes = {
"invisible": "invisible-grid",
}
soup = BeautifulSoup(text, "html.parser")
for table in soup.find_all("table"):
table_classes = table.get("class", [])
for dropped in dropped_classes:
if dropped in table_classes:
table_classes.remove(dropped)
for old, new in replaced_classes.items():
if old in table_classes:
table_classes.remove(old)
table_classes.append(new)
# all tables get the default bootstrap table class
if "table" not in table_classes:
table_classes.insert(0, "table")
return soup.decode()
def img_variant_fixer(text, obj=None):
"""Set image-variants"""
if not text:
return text
picture_variants = api.portal.get_registry_record("plone.picture_variants")
scale_variant_mapping = {
k: v["sourceset"][0]["scale"] for k, v in picture_variants.items()
}
scale_variant_mapping["thumb"] = "mini"
fallback_variant = "preview"
soup = BeautifulSoup(text, "html.parser")
for tag in soup.find_all("img"):
if "data-val" not in tag.attrs:
# maybe external image
continue
scale = tag["data-scale"]
variant = scale_variant_mapping.get(scale, fallback_variant)
tag["data-picturevariant"] = variant
classes = tag["class"]
new_class = f"picture-variant-{variant}"
if new_class not in classes:
classes.append(new_class)
tag["class"] = classes
return soup.decode()
def scale_unscaled_images(text, obj=None):
"""Scale unscaled image"""
if not text:
return text
fallback_scale = "huge"
soup = BeautifulSoup(text, "html.parser")
for link in soup.find_all("a"):
if not link.get("href") and not link.text:
# drop empty links (e.g. anchors)
link.decompose()
elif not link.get("href") and link.text:
# drop links without a href but keep the text
link.unwrap()
for tag in soup.find_all("img"):
if "data-val" not in tag.attrs:
# maybe external image
continue
scale = tag["data-scale"]
# Prevent unscaled images!
if not scale:
scale = fallback_scale
tag["data-scale"] = fallback_scale
if not tag["src"].endswith(scale):
tag["src"] = tag["src"] + "/" + scale
return soup.decode()
def fix_image_align(text, obj=None):
"""Replace align='xx' with css-classes"""
if not text:
return text
soup = BeautifulSoup(text, "html.parser")
for tag in soup.find_all("img"):
if "align" not in tag.attrs:
continue
classes = tag.get("class", [])
direction = tag["align"]
if direction == "left":
classes.append("image-left")
elif direction == "right":
classes.append("image-right")
if "image-inline" in classes:
classes.remove("image-inline")
del tag["align"]
return soup.decode()
Migrate very old Plone Versions with data created by collective.jsonify
-----------------------------------------------------------------------

1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@
install_requires.append("attrs < 22")
install_requires.append("backports.functools-lru-cache < 2")
install_requires.append("beautifulsoup4 < 4.10")
install_requires.append("ijson < 3.3.0")
# plone.restapi depends on plone.schema, which depends on jsonschema,
# which has a Py3-only release since September 2021.
install_requires.append("jsonschema < 4")
26 changes: 17 additions & 9 deletions src/collective/exportimport/configure.zcml
Original file line number Diff line number Diff line change
@@ -101,30 +101,38 @@
/>

<!-- Serializers -->
<adapter zcml:condition="installed Products.Archetypes"
<!--
We check for a subpackage of Products.Archetypes to avoid false positives
due to code aliases in migration code
-->
<adapter zcml:condition="installed Products.Archetypes.atapi"
factory=".serializer.ATFileFieldSerializer" />
<adapter zcml:condition="installed Products.Archetypes"
<adapter zcml:condition="installed Products.Archetypes.atapi"
factory=".serializer.ATImageFieldSerializer" />
<adapter zcml:condition="installed Products.Archetypes"
<adapter zcml:condition="installed Products.Archetypes.atapi"
factory=".serializer.ATImageFieldSerializerForBlobPaths" />

<adapter zcml:condition="installed Products.TALESField"
factory=".serializer.ATTalesFieldSerializer" />

<adapter zcml:condition="installed Products.Archetypes"
<adapter zcml:condition="installed Products.Archetypes.atapi"
factory=".serializer.ATTextFieldSerializer" />
<configure zcml:condition="installed Products.Archetypes">
<configure zcml:condition="installed Products.Archetypes.atapi">
<adapter zcml:condition="installed plone.app.contenttypes"
factory=".serializer.SerializeTopicToJson" />
</configure>

<adapter zcml:condition="installed plone.app.blob"
<!--
We check for a subpackage of plone.app.blob to avoid false positives
due to code aliases in migration code
-->
<adapter zcml:condition="installed plone.app.blob.config"
factory=".serializer.ATFileFieldSerializerWithBlobs" />
<adapter zcml:condition="installed plone.app.blob"
<adapter zcml:condition="installed plone.app.blob.config"
factory=".serializer.ATImageFieldSerializerWithBlobs" />
<adapter zcml:condition="installed plone.app.blob"
<adapter zcml:condition="installed plone.app.blob.config"
factory=".serializer.ATFileFieldSerializerWithBlobPaths" />
<adapter zcml:condition="installed plone.app.blob"
<adapter zcml:condition="installed plone.app.blob.config"
factory=".serializer.ATImageFieldSerializerWithBlobPaths" />

<!-- Serializers -->
2 changes: 1 addition & 1 deletion src/collective/exportimport/import_other.py
Original file line number Diff line number Diff line change
@@ -482,7 +482,7 @@ def import_ordering(self, data):
ordered = IOrderedContainer(obj.__parent__, None)
if not ordered:
continue
ordered.moveObjectToPosition(obj.getId(), item["order"])
ordered.moveObjectToPosition(obj.getId(), item["order"], suppress_events=True)
if not index % 1000:
logger.info(
u"Ordered {} ({}%) of {} items".format(
Loading

0 comments on commit 1f61856

Please sign in to comment.