Skip to content

Commit

Permalink
Fix modular - non-modular comparisons in content audit [RHELDST-13955] (
Browse files Browse the repository at this point in the history 
#169)

* content-audit: Skip modular RPM comparisons [RHELDST-13955]

Previously, modular and non-modular RPMs were wrongly being compared.
Let's skip modular RPMs entirely for now, leaving modular RPM support
for a separate effort.

* [pre-commit.ci] auto fixes from black and isort code formatting

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
@negillett @pre-commit-ci
negillett and pre-commit-ci[bot] committed Jul 2, 2024
1 parent ee10a95 commit 1b936dd
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 87 deletions.
109 changes: 49 additions & 60 deletions tests/test_content_audit_task.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def _setup_population_sources(pulp):
distributors=[distributor_rhel_2],
)

unit_1 = RpmUnit(
rpm_1 = RpmUnit(
name="gcc",
version="9.0.1",
release="200",
Expand All @@ -59,7 +59,7 @@ def _setup_population_sources(pulp):
requires=[],
provides=[],
)
unit_2 = RpmUnit(
rpm_2 = RpmUnit(
name="bind",
version="10",
release="200",
Expand All @@ -70,7 +70,19 @@ def _setup_population_sources(pulp):
requires=[],
provides=[],
)
unit_3 = ModulemdUnit(
rpm_3 = RpmUnit( # modular, should be skipped
name="bind",
version="12",
release="2.module+el8+2248+23d5e2f2",
epoch="0",
arch="x86_64",
sourcerpm="bind-12-2.module+el8+2248+23d5e2f2.src.rpm",
filename="bind-12-2.module+el8+2248+23d5e2f2.noarch.rpm",
)
rpm_4 = RpmUnit(name="httpd.src", version="1", release="2", arch="x86_64")
rpm_5 = RpmUnit(name="pkg-debuginfo.foo", version="1", release="2", arch="x86_64")
rpm_6 = RpmUnit(name="package-name-abc", version="1", release="2", arch="x86_64")
module_1 = ModulemdUnit(
name="fake_name",
stream="fake_stream",
version=10,
Expand All @@ -79,9 +91,11 @@ def _setup_population_sources(pulp):
artifacts=[
"test-0:1.24-3.module+el8.1.0+2934+dec45db7.noarch",
"test-0:1.24-3.module+el8.1.0+2934+dec45db7.src",
"bind-0:12-2.module+el8+2248+23d5e2f2.noarch",
"bind-0:12-2.module+el8+2248+23d5e2f2.src",
],
)
unit_4 = ModulemdUnit(
module_2 = ModulemdUnit(
name="some_module1",
stream="fake_stream",
version=10,
Expand All @@ -92,48 +106,34 @@ def _setup_population_sources(pulp):
"test-1:1.24-3.module+el8.1.0+2934+dec45db7.src",
],
)
unit_5 = ModulemdUnit(
name="some_module2",
stream="fake_stream",
version=10,
context="b7fad3bf",
arch="x86_64",
artifacts=[
"test-2:1.24-3.module+el8.1.0+2934+dec45db7.noarch",
"test-2:1.24-3.module+el8.1.0+2934+dec45db7.src",
],
)
unit_6 = ModulemdDefaultsUnit(
default_1 = ModulemdDefaultsUnit(
name="some_module_defaults1",
stream="fake_stream",
repo_id="ubi_repo",
profiles={"1.1": ["default"], "1.0": []},
)
unit_7 = ModulemdDefaultsUnit(
default_2 = ModulemdDefaultsUnit(
name="some_module_defaults2",
stream="fake_stream",
repo_id="ubi_repo",
profiles={"1.0": ["default"]},
)
unit_8 = RpmUnit(name="httpd.src", version="1", release="2", arch="x86_64")
unit_9 = RpmUnit(name="pkg-debuginfo.foo", version="1", release="2", arch="x86_64")
unit_10 = RpmUnit(name="package-name-abc", version="1", release="2", arch="x86_64")

pulp.insert_units(rhel_repo_1, [unit_1, unit_3, unit_5, unit_7, unit_9])
pulp.insert_units(rhel_repo_2, [unit_2, unit_4, unit_6, unit_8, unit_10])
pulp.insert_units(rhel_repo_1, [rpm_1, rpm_3, rpm_5, module_1, default_1])
pulp.insert_units(rhel_repo_2, [rpm_2, rpm_4, rpm_6, module_2, default_2])
pulp.insert_units(
ubi_repo,
[
unit_1,
unit_2,
unit_3,
unit_4,
unit_5,
unit_6,
unit_7,
unit_8,
unit_9,
unit_10,
rpm_1,
rpm_2,
rpm_3,
rpm_4,
rpm_5,
rpm_6,
module_1,
module_2,
default_1,
default_2,
],
)

Expand All @@ -160,75 +160,63 @@ def test_content_audit_outdated(pulp, caplog):
ubi_config_version="8",
content_set="cs_rpm_out",
)
unit_1 = RpmUnit(
rpm_1 = RpmUnit(
name="gcc",
version="8.2.1", # outdated
release="200",
epoch="1",
arch="x86_64",
sourcerpm="gcc_src-1-0.src.rpm",
filename="gcc-10.200.x86_64.rpm",
requires=[],
provides=[],
)
unit_2 = RpmUnit(
rpm_2 = RpmUnit(
name="bind",
version="10",
release="200",
epoch="1",
arch="x86_64",
sourcerpm="bind_src-1-0.src.rpm",
filename="bind-10.200.x86_64.rpm",
requires=[],
provides=[],
)
unit_3 = ModulemdUnit(
name="some_module1",
module_1 = ModulemdUnit(
name="fake_name",
stream="fake_stream",
version=7, # outdated
version=10,
context="b7fad3bf",
arch="x86_64",
artifacts=[
"test-0:1.24-3.module+el8.1.0+2934+dec45db7.noarch",
"test-0:1.24-3.module+el8.1.0+2934+dec45db7.src",
"bind-0:12-2.module+el8+2248+23d5e2f2.noarch",
"bind-0:12-2.module+el8+2248+23d5e2f2.src",
],
)
unit_4 = ModulemdUnit(
name="some_module2",
module_2 = ModulemdUnit(
name="some_module1",
stream="fake_stream",
version=10,
version=7, # outdated
context="b7fad3bf",
arch="x86_64",
artifacts=[
"test-1:1.24-3.module+el8.1.0+2934+dec45db7.noarch",
"test-1:1.24-3.module+el8.1.0+2934+dec45db7.src",
"test-0:5.module+el8.1.0+2934+dec45db7.noarch",
"test-0:5.module+el8.1.0+2934+dec45db7.src",
],
)
unit_5 = ModulemdDefaultsUnit(
default_1 = ModulemdDefaultsUnit(
name="some_module_defaults1",
stream="fake_stream",
repo_id="outdated_ubi_repo",
profiles={"1.0": ["default"]}, # outdated
)
unit_6 = ModulemdDefaultsUnit(
default_2 = ModulemdDefaultsUnit(
name="some_module_defaults2",
stream="fake_stream",
repo_id="outdated_ubi_repo",
profiles={"1.0": ["default"]},
)
unit_7 = ModulemdUnit(
name="fake_name",
stream="fake_stream",
version=10,
context="b7fad3bf",
arch="x86_64",
artifacts=[
"test-0:1.24-3.module+el8.1.0+2934+dec45db7.noarch",
"test-0:1.24-3.module+el8.1.0+2934+dec45db7.src",
],
)
pulp.insert_units(
ubi_repo, [unit_1, unit_2, unit_3, unit_4, unit_5, unit_6, unit_7]
ubi_repo,
[rpm_1, rpm_2, module_1, module_2, default_1, default_2],
)

with mock.patch("ubi_manifest.worker.tasks.depsolver.utils.Client") as client:
Expand All @@ -240,6 +228,7 @@ def test_content_audit_outdated(pulp, caplog):

# should have logged warnings
expected_logs = [
"[outdated_ubi_repo] Skipping modular RPM bind-12-2.module+el8+2248+23d5e2f2.noarch.rpm",
"[outdated_ubi_repo] UBI modulemd 'some_module1:fake_stream' version is outdated (current: 7, latest: 10)",
"[outdated_ubi_repo] UBI modulemd_defaults 'some_module_defaults1:fake_stream' version is outdated",
"[outdated_ubi_repo] UBI rpm 'gcc' version is outdated (current: ('0', '8.2.1', '200'), latest: ('0', '9.0.1', '200'))",
Expand Down
13 changes: 7 additions & 6 deletions tests/test_depsolver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
PackageToExclude,
UbiUnit,
)
from ubi_manifest.worker.tasks.depsolver.rpm_depsolver import Depsolver
from ubi_manifest.worker.tasks.depsolver.rpm_depsolver import (
Depsolver,
get_pkgs_from_all_modules,
)

from .utils import create_and_insert_repo, rpmdeps_from_names

Expand Down Expand Up @@ -193,9 +196,7 @@ def test_get_pkgs_from_all_modules(pulp):

pulp.insert_units(repo, [unit_1, unit_2])

ft = depsolver._get_pkgs_from_all_modules([repo])

result = ft.result()
filenames = get_pkgs_from_all_modules([repo])

# there are 4 filenames according from 2 modulemd units
expected_filenames = set(
Expand All @@ -207,8 +208,8 @@ def test_get_pkgs_from_all_modules(pulp):
]
)

assert len(result) == 4
assert result == expected_filenames
assert len(filenames) == 4
assert filenames == expected_filenames


def test_get_source_pkgs(pulp):
Expand Down
15 changes: 12 additions & 3 deletions ubi_manifest/worker/tasks/content_audit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from ubi_manifest.worker.tasks.depsolve import filter_whitelist, get_content_config
from ubi_manifest.worker.tasks.depsolver.models import PackageToExclude, UbiUnit
from ubi_manifest.worker.tasks.depsolver.pulp_queries import search_units
from ubi_manifest.worker.tasks.depsolver.rpm_depsolver import get_pkgs_from_all_modules
from ubi_manifest.worker.tasks.depsolver.ubi_config import UbiConfigLoader
from ubi_manifest.worker.tasks.depsolver.utils import (
RELATION_CMP_MAP,
Expand All @@ -25,7 +26,7 @@

_LOG = logging.getLogger(__name__)

RPM_FIELDS = ["name", "version", "release", "arch"]
RPM_FIELDS = ["name", "version", "release", "arch", "filename"]
MD_FIELDS = ["name", "stream", "version", "context", "arch"]


Expand Down Expand Up @@ -66,9 +67,12 @@ def content_audit_task() -> None:
in_mds_fts: list[Future[set[UbiUnit]]] = []
in_mdds_fts: list[Future[set[UbiUnit]]] = []

for in_repo in client.search_repository(
in_repos = client.search_repository(
Criteria.with_id(out_repo.population_sources)
):
)
in_mod_rpm_filenames = get_pkgs_from_all_modules(in_repos)

for in_repo in in_repos:
# get all corresponding units currently on input repo
in_rpms_fts.append(
search_units(
Expand Down Expand Up @@ -116,6 +120,11 @@ def content_audit_task() -> None:
# check that all content is up-to-date
out_rpms_result = out_rpms.result()
for in_rpm in _latest_input_rpms(in_rpms_fts):
if in_rpm.filename in in_mod_rpm_filenames:
_LOG.debug(
"[%s] Skipping modular RPM %s", out_repo.id, in_rpm.filename
)
continue
for out_rpm in out_rpms_result.copy():
if (out_rpm.name, out_rpm.arch) == (in_rpm.name, in_rpm.arch):
_compare_versions(out_repo.id, out_rpm, in_rpm)
Expand Down
35 changes: 17 additions & 18 deletions ubi_manifest/worker/tasks/depsolver/rpm_depsolver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,22 @@
MAX_WORKERS = int(os.getenv("UBI_MANIFEST_DEPSOLVER_WORKERS", "8"))


def get_pkgs_from_all_modules(repos: list[YumRepository]) -> set[str]:
"""
Search for modulemds in all input repos and extract rpm filenames.
"""

def extract_modular_filenames() -> set[str]:
filenames = set()
for module in modules: # type: ignore [attr-defined]
filenames |= set(module.artifacts_filenames)

return filenames

modules = search_modulemds([Criteria.true()], repos)
return extract_modular_filenames()


class Depsolver:
def __init__(
self,
Expand Down Expand Up @@ -81,23 +97,6 @@ def __enter__(self) -> Depsolver:
def __exit__(self, *args: Any, **kwargs: Any) -> None:
self._executor.__exit__(*args, **kwargs)

def _get_pkgs_from_all_modules(
self, repos: list[YumRepository]
) -> Future[set[str]]:
"""
Search for modulemds in all input repos and extract rpm filenames.
"""

def extract_modular_filenames() -> set[str]:
filenames = set()
for module in modules: # type: ignore [attr-defined]
filenames |= set(module.artifacts_filenames)

return filenames

modules = search_modulemds([Criteria.true()], repos)
return f_proxy(self._executor.submit(extract_modular_filenames))

def get_base_packages(
self,
repos: list[YumRepository],
Expand Down Expand Up @@ -269,7 +268,7 @@ def run(self) -> None:
# Get modular rpms if they are not already populated from the previous run of the depsolver
if not self._modular_rpm_filenames:
self._modular_rpm_filenames.update(
self._get_pkgs_from_all_modules(pulp_repos) # type: ignore [arg-type]
f_proxy(self._executor.submit(get_pkgs_from_all_modules, pulp_repos)) # type: ignore [arg-type]
)

merged_blacklist = list(
Expand Down

0 comments on commit 1b936dd

Please sign in to comment.