Investigate why no data on rgw metrics of some deployments test_monitoring_negative.py and test_monitoring_defaults.py

[DanielOsypenko]

https://url.corp.redhat.com/fac5a9c - Proxy
https://url.corp.redhat.com/b5f64a3 - Compact mode

output similar to:

measure_stop_ceph_osd = {'first_run': False, 'metadata': None, 'prometheus_alerts': [{'activeAt': '2024-09-04T05:57:39.826325362Z', 'annotatio...mespace': 'openshift-monitoring', 'severity': 'none'}, 'state': 'firing', ...}, ...], 'result': 'rook-ceph-osd-2', ...}
threading_lock = <unlocked _thread.RLock object owner=0 count=0 at 0x7efbee6693c0>


@blue_squad
@tier3
@bugzilla("2203795")
@pytest.mark.polarion_id("OCS-2734")
@skipif_managed_service
def test_ceph_metrics_presence_when_osd_down(measure_stop_ceph_osd, threading_lock):
    """
    Since ODF 4.9 ceph metrics covering disruptions will be available only
    when there are some disruptions to report, as noted in BZ 2028649.

    This test case covers this behaviour for one stopped/disabled OSD.
    """
    prometheus = PrometheusAPI(threading_lock=threading_lock)
    metrics_expected = list(metrics.ceph_metrics_healthy)
    # metrics which should be present with one OSD down
    for mtr in ("ceph_pg_degraded", "ceph_pg_undersized"):
        assert mtr in metrics.ceph_metrics, "test code needs to be updated"
        # make sure the test code is consistent with metrics module
        metrics_expected.append(mtr)
    # metrics which should not be present with one OSD down
    for mtr in ["ceph_pg_clean"]:
        assert mtr in metrics.ceph_metrics, "test code needs to be updated"
        metrics_expected.remove(mtr)
    metrics_without_results = metrics.get_missing_metrics(
        prometheus,
        metrics_expected,
        current_platform=config.ENV_DATA["platform"].lower(),
        start=measure_stop_ceph_osd["start"],
        stop=measure_stop_ceph_osd["stop"],
    )
    msg = (
        "Prometheus should provide some value(s) for all tested metrics, "
        "so that the list of metrics without results is empty."
    )

  assert metrics_without_results == [], msg
E       AssertionError: Prometheus should provide some value(s) for all tested metrics, so that the list of metrics without results is empty.
E       assert ['ceph_rgw_pu...w_get_b', ...] == []
E         Left contains 8 more items, first extra item: 'ceph_rgw_put'
E         Full diff:
E           [
E         -  ,
E         +  'ceph_rgw_put',
E         +  'ceph_rgw_put_initial_lat_sum',
E         +  'ceph_rgw_put_initial_lat_count',
E         +  'ceph_rgw_get_initial_lat_sum',
E         +  'ceph_rgw_get_initial_lat_count',
E         +  'ceph_rgw_get_b',
E         +  'ceph_rgw_get',
E         +  'ceph_rgw_put_b',
E           ]