diff --git a/CHANGES/5324.feature b/CHANGES/5324.feature new file mode 100644 index 0000000000..1f08a1cb0b --- /dev/null +++ b/CHANGES/5324.feature @@ -0,0 +1 @@ +Adapted PulpImport/Export to allow update django-import-export==4.x. diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py index e448dc158a..7d1e798aa6 100644 --- a/pulpcore/app/importexport.py +++ b/pulpcore/app/importexport.py @@ -48,6 +48,10 @@ def _write_export(the_tarfile, resource, dest_dir=None): # the data in batches to memory and concatenate the json lists via string manipulation. with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file: if isinstance(resource.queryset, QuerySet): + # If we don't have any of "these" - skip writing + if resource.queryset.count() == 0: + return + temp_file.write("[") def process_batch(batch): diff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py index a8dd73eb7f..e7fd0e8a17 100644 --- a/pulpcore/app/modelresource.py +++ b/pulpcore/app/modelresource.py @@ -62,8 +62,8 @@ def before_import_row(self, row, **kwargs): # the export converts None to blank strings but sha384 and sha512 have unique constraints # that get triggered if they are blank. convert checksums back into None if they are blank. for checksum in ALL_KNOWN_CONTENT_CHECKSUMS: - if row[checksum] == "": - row[checksum] = None + if row[checksum] == "" or checksum not in settings.ALLOWED_CONTENT_CHECKSUMS: + del row[checksum] def set_up_queryset(self): """ @@ -109,7 +109,7 @@ def get_queryset(self, value, row, *args, **kwargs): return qs def render(self, value, obj=None, **kwargs): - return value.sha256 + return value.sha256 if value else "" class ContentArtifactResource(QueryModelResource): diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py index 5c9641545c..3480348150 100644 --- a/pulpcore/app/tasks/importer.py +++ b/pulpcore/app/tasks/importer.py @@ -233,6 +233,9 @@ def _import_file(fpath, resource_class, retry=False): """ try: log.info(f"Importing file {fpath}.") + if not os.path.isfile(fpath): + log.info("...empty - skipping.") + return [] with open(fpath, "r") as json_file: resource = resource_class() log.info(f"...Importing resource {resource.__class__.__name__}.") @@ -241,6 +244,8 @@ def _import_file(fpath, resource_class, retry=False): # overlapping content. for batch_str in _impfile_iterator(json_file): data = Dataset().load(StringIO(batch_str)) + if not data: + return [] if retry: curr_attempt = 1 @@ -272,6 +277,7 @@ def _import_file(fpath, resource_class, retry=False): try: a_result = resource.import_data(data, raise_errors=True) except Exception as e: # noqa log on ANY exception and then re-raise + log.error(e) log.error(f"FATAL import-failure importing {fpath}") raise else: diff --git a/pulpcore/plugin/importexport.py b/pulpcore/plugin/importexport.py index 2ceb25182a..af2cbfb0f3 100644 --- a/pulpcore/plugin/importexport.py +++ b/pulpcore/plugin/importexport.py @@ -41,6 +41,9 @@ def set_up_queryset(self): def dehydrate_pulp_domain(self, content): return str(content.pulp_domain_id) + def render(self, value, obj=None, **kwargs): + return super().render(value, obj, coerce_to_string=False, **kwargs) + def __init__(self, repo_version=None): self.repo_version = repo_version if repo_version: diff --git a/pyproject.toml b/pyproject.toml index dcfdcf96e2..80d6107e18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "Django~=4.2.0", # LTS version, switch only if we have a compelling reason to". "django-filter>=23.1,<=25.1", # Uses CalVer, not released often https://github.com/carltongibson/django-filter "django-guid>=3.3.0,<3.6", # Looks like only bugfixes in z-Stream. - "django-import-export>=2.9,<3.4.0", + "django-import-export>=4.3,<5.0", "django-lifecycle>=1.0,<=1.2.4", "djangorestframework>=3.14.0,<=3.16.1", "djangorestframework-queryfields>=1.0,<=1.1.0", @@ -59,7 +59,7 @@ dependencies = [ "python-gnupg>=0.5.0,<0.6", # Looks like only bugfixes in z-Stream [changelog only in git] "PyYAML>=5.1.1,<6.1", # Looks like only bugfixes in z-Stream. "redis>=4.3.0,<6.5", # Looks like only bugfixes in z-Stream. - "tablib>=3.5.0,<3.6", # 3.6.0 breaks with import export. Not sure about semver. + "tablib>=3.7.0,<4.0", # 3.6.0 breaks with import export. Not sure about semver. "url-normalize>=1.4.3,<2.3", # SemVer. https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md#changelog "uuid6>=2023.5.2,<=2025.0.1", "whitenoise>=5.0,<6.12.0",