Merge pull request #548 from inknos/testing-podman-pnext

Improve testing against distro and podman-next

Author: openshift-merge-bot[bot]

.cirrus.yml

@@ -102,12 +102,13 @@ jobs:
     dist_git_branches:
       - fedora-branched # rawhide updates are created automatically
 
+
   # Test linting on the codebase
   # This test might break based on the OS and lint used, so we follow fedora-latest as a reference
   - job: tests
     trigger: pull_request
-    identifier: upstream-sanity
-    tmt_plan: /upstream/sanity
+    identifier: distro-sanity
+    tmt_plan: /distro/sanity
     packages: [python-podman-fedora]
     targets:
       - fedora-latest-stable
@@ -118,7 +119,7 @@ jobs:
   - job: tests
     trigger: pull_request
     identifier: unittest-coverage
-    tmt_plan: /upstream/unittest_coverage
+    tmt_plan: /distro/unittest_coverage
     packages: [python-podman-fedora]
     targets:
       - fedora-latest-stable
@@ -130,30 +131,43 @@ jobs:
   # run all tests for all python versions on all fedoras
   - job: tests
     trigger: pull_request
-    identifier: upstream-all-fedora
-    tmt_plan: /upstream/all
+    identifier: distro-fedora-all
+    tmt_plan: /distro/all_python
     packages: [python-podman-fedora]
     targets:
       - fedora-all
+
+  # run tests for the rawhide python version using podman-next packages
+  - job: tests
+    trigger: pull_request
+    identifier: podman-next-fedora-base
+    tmt_plan: /pnext/base_python
+    packages: [python-podman-fedora]
+    targets:
+      - fedora-rawhide
     tf_extra_params:
       environments:
         - artifacts:
           - type: repository-file
             id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
+    manual_trigger: true
+    labels:
+      - pnext
+      - podman-next
 
   - job: tests
     trigger: pull_request
-    identifier: upstream-base-centos
-    tmt_plan: /upstream/base
+    identifier: distro-centos-base
+    tmt_plan: /distro/base_python
     packages: [python-podman-centos]
     targets:
       - centos-stream-9
       - centos-stream-10
 
   - job: tests
     trigger: pull_request
-    identifier: upstream-base-rhel
-    tmt_plan: /upstream/base
+    identifier: distro-rhel-base
+    tmt_plan: /distro/base_python
     packages: [python-podman-rhel]
     targets:
       - epel-9

.packit.yaml

@@ -45,6 +45,45 @@ pip install tox
 tox -e coverage
 ```
 
+#### Advanced testing
+
+Always prefer to run `tox` directly, even when you want to run a specific test or scenario.
+Instead of running `pytest` directly, you should run:
+
+```
+tox -e py -- podman/tests/integration/test_container_create.py -k test_container_directory_volume_mount
+```
+
+If you'd like to test against a specific `tox` environment you can do:
+
+```
+tox -e py12 -- podman/tests/integration/test_container_create.py -k test_container_directory_volume_mount
+```
+
+Pass pytest options after `--`.
+
+#### Testing future features
+
+Since `podman-py` follows stable releases of `podman`, tests are thought to be run against
+libpod's versions that are commonly installed in the distributions. Tests can be versioned,
+but preferably they should not. Occasionally, upstream can diverge and have features that
+are not included in a specific version of libpod, or that will be included eventually.
+To run a test against such changes, you need to have
+[podman-next](https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next) installed.
+Then, you need to mark the test as `@pytest.mark.pnext`. Marked tests willbe excluded from the
+runs, unless you pass `--pnext` as a cli option.
+Preferably, this should be a rare case and it's better to use this marker as a temporary solution,
+with the goal of removing the marker within few PRs.
+
+To run these tests use:
+
+```
+tox -e py -- --pnext -m pnext podman/tests/integration/test_container_create.py -k test_container_mounts_without_rw_as_default
+```
+
+The option `--pnext` **enables** the tests with the `pnext` pytest marker, and `-m pnext` will run
+the marked tests **only**.
+
 ## Submitting changes
 
 - Create a github pull request (PR)

CONTRIBUTING.md

@@ -26,10 +26,16 @@ tests: tox
 	# see tox.ini for environment variable settings
 	$(PYTHON) -m tox -e coverage,py39,py310,py311,py312,py313
 
+.PHONY: tests-ci-base-python-podman-next
+tests-ci-base-python-podman-next:
+	$(PYTHON) -m tox -e py -- --pnext -m pnext
+
 .PHONY: tests-ci-base-python
 tests-ci-base-python:
 	$(PYTHON) -m tox -e coverage,py
 
+# TODO: coverage is probably not necessary here and in tests-ci-base-python
+# 		but for now it's ok to leave it here so it's run
 .PHONY: tests-ci-all-python
 tests-ci-all-python:
 	$(PYTHON) -m tox -e coverage,py39,py310,py311,py312,py313

Makefile

@@ -1,5 +1,5 @@
 # podman-py
-[![Build Status](https://api.cirrus-ci.com/github/containers/podman-py.svg)](https://cirrus-ci.com/github/containers/podman-py/main)
+[![PyPI Latest Version](https://img.shields.io/pypi/v/podman)](https://pypi.org/project/podman/)
 
 This python package is a library of bindings to use the RESTful API of [Podman](https://github.com/containers/podman).
 It is currently under development and contributors are welcome!

README.md

@@ -12,16 +12,6 @@ prepare:
       - python3-pip
       - podman
 
-  - name: enable rhcontainerbot/podman-next update podman
-    when: initiator == packit
-    how: shell
-    script: |
-      COPR_REPO_FILE="/etc/yum.repos.d/*podman-next*.repo"
-      if compgen -G $COPR_REPO_FILE > /dev/null; then
-          sed -i -n '/^priority=/!p;$apriority=1' $COPR_REPO_FILE
-      fi
-      dnf -y upgrade --allowerasing
-
   - name: pip dependencies
     how: shell
     script:
@@ -34,20 +24,46 @@ prepare:
         - cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys%
         - cat /root/.ssh/id_ecdsa.pub >>/root/.ssh/authorized_keys
 
-/upstream:
+# Run tests agains Podman Next builds.
+# These tests should NOT overlap with the ones who run in the distro plan and should only include
+# tests against upcoming features or upstream tests that we need to run for reasons.
+/pnext:
+    prepare+:
+      - name: enable rhcontainerbot/podman-next update podman
+        when: initiator == packit
+        how: shell
+        script: |
+          COPR_REPO_FILE="/etc/yum.repos.d/*podman-next*.repo"
+          if compgen -G $COPR_REPO_FILE > /dev/null; then
+              sed -i -n '/^priority=/!p;$apriority=1' $COPR_REPO_FILE
+          fi
+          dnf -y upgrade --allowerasing
+
+    /base_python:
+        summary: Run Tests Upstream PRs for base Python
+        discover+:
+            filter: tag:pnext
+
+    adjust+:
+        enabled: false
+        when: initiator is not defined or initiator != packit
+
+
+# Run tests against Podman buids installed from the distribution.
+/distro:
     /sanity:
         summary: Run Sanity and Coverage checks on Python Podman
         discover+:
             # we want to change this to tag:stable once all the coverage tests are fixed
             filter: tag:lint
 
-    /base:
-        summary: Run Python Podman Tests on Upstream PRs for base Python
+    /base_python:
+        summary: Run Tests Upstream for base Python
         discover+:
             filter: tag:base
 
-    /all:
-        summary: Run Python Podman Tests on Upstream PRs for all Python versions
+    /all_python:
+        summary: Run Tests Upstream PRs for all Python versions
         prepare+:
           - name: install all python versions
             how: install
@@ -60,6 +76,7 @@ prepare:
         discover+:
             filter: tag:matrix
 
+    # TODO: replace with /coverage and include integration tests coverage
     /unittest_coverage:
         summary: Run Unit test coverage
         discover+:
@@ -69,9 +86,11 @@ prepare:
         enabled: false
         when: initiator is not defined or initiator != packit
 
+# Run tests against downstream Podman. These tests should be the all_python only since the sanity
+# of code is tested in the distro environment
 /downstream:
     /all:
-      summary: Run Python Podman Tests on bodhi / errata and dist-git PRs
+      summary: Run Tests on bodhi / errata and dist-git PRs
       prepare+:
         - name: install all python versions
           how: install

plans/main.fmf

@@ -0,0 +1,21 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--pnext", action="store_true", default=False, help="run tests against podman_next copr"
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "pnext: mark test as run against podman_next")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--pnext"):
+        # --pnext given in cli: run tests marked as pnext
+        return
+    podman_next = pytest.mark.skip(reason="need --pnext option to run")
+    for item in items:
+        if "pnext" in item.keywords:
+            item.add_marker(podman_next)