From f62dd40705ac17930892e836fed05028ea97c8e7 Mon Sep 17 00:00:00 2001 From: Mohammed Junaid Date: Thu, 12 Dec 2024 02:02:07 -0600 Subject: [PATCH] 1. Support GPU indexes in -i option 2. Support device_index as conf. parameter --- babel.so/src/action.cpp | 3 +- docs/cli.md | 4 +- docs/ug1main.md | 4 +- gpup.so/src/action.cpp | 259 ++++++++++++++++++++---------------- gst.so/src/action.cpp | 4 +- iet.so/src/action.cpp | 5 +- include/gpu_util.h | 7 +- include/rvs_util.h | 5 +- mem.so/src/action.cpp | 3 +- pbqt.so/src/action.cpp | 16 ++- pebb.so/src/action.cpp | 15 ++- peqt.so/src/action.cpp | 25 +++- perf.so/src/action.cpp | 5 +- pesm.so/include/worker.h | 8 +- pesm.so/src/action.cpp | 1 + pesm.so/src/worker.cpp | 23 ++++ rvs/src/rvsexec.cpp | 6 +- rvs/src/rvsexec_do_yaml.cpp | 29 +++- src/gpu_util.cpp | 123 ++++++++++++++++- src/rvs_util.cpp | 182 ++++++++++++++----------- src/rvsactionbase.cpp | 117 ++++++++-------- tst.so/src/action.cpp | 5 +- 22 files changed, 570 insertions(+), 279 deletions(-) diff --git a/babel.so/src/action.cpp b/babel.so/src/action.cpp index b2781668..c9958b50 100644 --- a/babel.so/src/action.cpp +++ b/babel.so/src/action.cpp @@ -278,7 +278,8 @@ int mem_action::get_all_selected_gpus(void) { // iterate over all available & compatible AMD GPUs amd_gpus_found = fetch_gpu_list(hip_num_gpu_devices, mem_gpus_device_index, - property_device, property_device_id, property_device_all); + property_device, property_device_id, property_device_all, + property_device_index, property_device_index_all); if (amd_gpus_found) { if (do_mem_stress_test(mem_gpus_device_index)) return 0; diff --git a/docs/cli.md b/docs/cli.md index 1d410167..2afbc699 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -22,8 +22,8 @@ -g --listGpus List all the GPUs available in the machine, that RVS supports and has visibility. --i --indexes Comma separated list of GPU ids to run test on. This overrides - the device values specified for every actions in the +-i --indexes Comma separated list of GPU ids/indexes to run test on. This overrides + the device/device_index values specified for every actions in the configuration file, including the ‘all’ value. -j --json Generate output file in JSON format. diff --git a/docs/ug1main.md b/docs/ug1main.md index e0aa4e00..4a559afd 100644 --- a/docs/ug1main.md +++ b/docs/ug1main.md @@ -255,8 +255,8 @@ The range is 0-5 with 5 being the highest verbose level. that RVS supports and has visibility. --i--indexesComma separated list of GPU ids to run test on. -This overrides the device values specified for every actions in the +-i--indexesComma separated list of GPU ids/indexes to run test on. +This overrides the device/device_index values specified for every actions in the configuration file, including the ‘all’ value. diff --git a/gpup.so/src/action.cpp b/gpup.so/src/action.cpp index 521cb37f..015c11a8 100644 --- a/gpup.so/src/action.cpp +++ b/gpup.so/src/action.cpp @@ -342,159 +342,194 @@ int gpup_action::property_io_links_get_value(uint16_t gpu_id) { * @return run result */ int gpup_action::run(void) { - std::string msg; - int sts = 0; - rvs::action_result_t action_result; - // get the action name - if (property_get(RVS_CONF_NAME_KEY, &action_name)) { - msg = "Action name missing"; - rvs::lp::Err(msg, MODULE_NAME_CAPS); + std::string msg; + int sts = 0; + rvs::action_result_t action_result; - // Action callback - action_result.state = rvs::actionstate::ACTION_COMPLETED; - action_result.status = rvs::actionstatus::ACTION_FAILED; - action_result.output = msg; - action_callback(&action_result); + // get the action name + if (property_get(RVS_CONF_NAME_KEY, &action_name)) { + msg = "Action name missing"; + rvs::lp::Err(msg, MODULE_NAME_CAPS); - return -1; - } + // Action callback + action_result.state = rvs::actionstate::ACTION_COMPLETED; + action_result.status = rvs::actionstatus::ACTION_FAILED; + action_result.output = msg; + action_callback(&action_result); + + return -1; + } - // get property value (a list of gpu id) - if (int sts = property_get_device()) { - switch (sts) { + // get property value (a list of gpu id) + if (int sts = property_get_device()) { + switch (sts) { case 1: msg = "Invalid 'device' key value."; break; case 2: msg = "Missing 'device' key."; break; - } - rvs::lp::Err(msg, MODULE_NAME_CAPS, action_name); + } + rvs::lp::Err(msg, MODULE_NAME_CAPS, action_name); - // Action callback - action_result.state = rvs::actionstate::ACTION_COMPLETED; - action_result.status = rvs::actionstatus::ACTION_FAILED; - action_result.output = msg; - action_callback(&action_result); + // Action callback + action_result.state = rvs::actionstate::ACTION_COMPLETED; + action_result.status = rvs::actionstatus::ACTION_FAILED; + action_result.output = msg; + action_callback(&action_result); - return -1; + return -1; + } + + if (int sts = property_get_device_index()) { + switch (sts) { + case 1: + msg = "Invalid 'device_index' key value."; + break; + case 2: + msg = "Missing 'device_index' key."; + break; } - // get the property value if provided - if (property_get_int(RVS_CONF_DEVICEID_KEY, - &property_device_id, 0u)) { - msg = "Invalid 'deviceid' key value."; - rvs::lp::Err(msg, MODULE_NAME_CAPS, action_name); + property_device_index_all = true; + rvs::lp::Log(msg, rvs::loginfo); + } - // Action callback - action_result.state = rvs::actionstate::ACTION_COMPLETED; - action_result.status = rvs::actionstatus::ACTION_FAILED; - action_result.output = msg; - action_callback(&action_result); + if (property_device_index.size() || property_device.size()) { + property_device_all = false; + property_device_index_all = false; + } - return -1; - } + // get the property value if provided + if (property_get_int(RVS_CONF_DEVICEID_KEY, + &property_device_id, 0u)) { + msg = "Invalid 'deviceid' key value."; + rvs::lp::Err(msg, MODULE_NAME_CAPS, action_name); - // extract properties and io_links properties names - property_split(JSON_PROP_NODE_NAME); - property_split(JSON_IO_LINK_PROP_NODE_NAME); + // Action callback + action_result.state = rvs::actionstate::ACTION_COMPLETED; + action_result.status = rvs::actionstatus::ACTION_FAILED; + action_result.output = msg; + action_callback(&action_result); - bjson = false; // already initialized in the default constructor + return -1; + } - // check for -j flag (json logging) - if (has_property("cli.-j")) { - bjson = true; - } + // extract properties and io_links properties names + property_split(JSON_PROP_NODE_NAME); + property_split(JSON_IO_LINK_PROP_NODE_NAME); - // get all AMD GPUs - vector gpu; - gpu_get_all_gpu_id(&gpu); - bool b_gpu_found = false; - if (bjson){ - if (rvs::lp::JsonActionStartNodeCreate(MODULE_NAME, action_name.c_str())){ - rvs::lp::Err("json start create failed", MODULE_NAME_CAPS, action_name); - return 1; - } + bjson = false; // already initialized in the default constructor - } + // check for -j flag (json logging) + if (has_property("cli.-j")) { + bjson = true; + } - // iterate over AMD GPUs - for (auto it = gpu.begin(); it !=gpu.end(); ++it) { - // filter by gpu_id if needed - if (property_device_id > 0) { - uint16_t dev_id; - if (!rvs::gpulist::gpu2device(*it, &dev_id)) { - if (dev_id != property_device_id) { - continue; - } - } else { - msg = "Device ID not found for GPU " + std::to_string(*it); - rvs::lp::Err(msg, MODULE_NAME, action_name); + // get all AMD GPUs + vector gpu_id; + vector gpu_idx; + + gpu_get_all_gpu_id(&gpu_id); + gpu_get_all_gpu_idx(&gpu_idx); - // Action callback - action_result.state = rvs::actionstate::ACTION_COMPLETED; - action_result.status = rvs::actionstatus::ACTION_FAILED; - action_result.output = msg; - action_callback(&action_result); + bool b_gpu_found = false; - return -1; + if (bjson){ + if (rvs::lp::JsonActionStartNodeCreate(MODULE_NAME, action_name.c_str())){ + rvs::lp::Err("json start create failed", MODULE_NAME_CAPS, action_name); + return 1; + } + } + + // iterate over AMD GPUs + for (size_t i = 0; i < gpu_id.size(); i++) { + + // filter by gpu_id if needed + if (property_device_id > 0) { + uint16_t dev_id; + if (!rvs::gpulist::gpu2device(gpu_id[i], &dev_id)) { + if (dev_id != property_device_id) { + continue; } + } else { + msg = "Device ID not found for GPU " + std::to_string(gpu_id[i]); + rvs::lp::Err(msg, MODULE_NAME, action_name); + + // Action callback + action_result.state = rvs::actionstate::ACTION_COMPLETED; + action_result.status = rvs::actionstatus::ACTION_FAILED; + action_result.output = msg; + action_callback(&action_result); + + return -1; } + } - // filter by device if needed - if (!property_device_all) { - if (std::find(property_device.begin(), property_device.end(), *it) == + // filter by device if needed + if (!property_device_all && property_device.size()) { + if (std::find(property_device.begin(), property_device.end(), gpu_id[i]) == property_device.end()) { - continue; - } + continue; } - b_gpu_found = true; + } - if (bjson){ - json_root_node = json_node_create(std::string(module_name), - action_name.c_str(), rvs::logresults); - if(json_root_node) - rvs::lp::AddString(json_root_node, RVS_JSON_LOG_GPU_ID_KEY, std::to_string(*it)); - } - // properties values - sts = property_get_value(*it); - sts = property_io_links_get_value(*it); - - if (bjson) { // json logging stuff - RVSTRACE_ - rvs::lp::AddString(json_root_node,"pass" , (sts == 0) ? "true" : "false"); - rvs::lp::LogRecordFlush(json_root_node); - json_root_node = nullptr; + // filter by device if needed + if (!property_device_index_all && property_device_index.size()) { + if (std::find(property_device_index.cbegin(), property_device_index.cend(), gpu_idx[i]) == + property_device_index.cend()) { + continue; } + } - if (sts) { - RVSTRACE_ - } - } // for all gpu_id - if(bjson){ - rvs::lp::JsonActionEndNodeCreate(); + b_gpu_found = true; + + if (bjson){ + json_root_node = json_node_create(std::string(module_name), + action_name.c_str(), rvs::logresults); + if(json_root_node) + rvs::lp::AddString(json_root_node, RVS_JSON_LOG_GPU_ID_KEY, std::to_string(gpu_id[i])); } - if (!b_gpu_found) { - msg = "No device matches criteria from configuration. "; - rvs::lp::Err(msg, MODULE_NAME, action_name); + // properties values + sts = property_get_value(gpu_id[i]); + sts = property_io_links_get_value(gpu_id[i]); - // Action callback - action_result.state = rvs::actionstate::ACTION_COMPLETED; - action_result.status = rvs::actionstatus::ACTION_FAILED; - action_result.output = msg; - action_callback(&action_result); + if (bjson) { // json logging stuff + RVSTRACE_ + rvs::lp::AddString(json_root_node,"pass" , (sts == 0) ? "true" : "false"); + rvs::lp::LogRecordFlush(json_root_node); + json_root_node = nullptr; + } - return -1; + if (sts) { + RVSTRACE_ } + } // for all gpu_id + + if(bjson){ + rvs::lp::JsonActionEndNodeCreate(); + } + if (!b_gpu_found) { + msg = "No device matches criteria from configuration. "; + rvs::lp::Err(msg, MODULE_NAME, action_name); // Action callback action_result.state = rvs::actionstate::ACTION_COMPLETED; - action_result.status = (!sts) ? rvs::actionstatus::ACTION_SUCCESS : rvs::actionstatus::ACTION_FAILED; - action_result.output = "GPUP Module action " + action_name + " completed"; + action_result.status = rvs::actionstatus::ACTION_FAILED; + action_result.output = msg; action_callback(&action_result); - return sts; + return -1; + } + + // Action callback + action_result.state = rvs::actionstate::ACTION_COMPLETED; + action_result.status = (!sts) ? rvs::actionstatus::ACTION_SUCCESS : rvs::actionstatus::ACTION_FAILED; + action_result.output = "GPUP Module action " + action_name + " completed"; + action_callback(&action_result); + + return sts; } /* diff --git a/gst.so/src/action.cpp b/gst.so/src/action.cpp index 51bdf805..6742bf42 100644 --- a/gst.so/src/action.cpp +++ b/gst.so/src/action.cpp @@ -587,8 +587,10 @@ int gst_action::get_all_selected_gpus(void) { hip_num_gpu_devices = get_num_amd_gpu_devices(); if (hip_num_gpu_devices < 1) return hip_num_gpu_devices; + amd_gpus_found = fetch_gpu_list(hip_num_gpu_devices, gst_gpus_device_index, - property_device, property_device_id, property_device_all); + property_device, property_device_id, property_device_all, + property_device_index, property_device_index_all); // iterate over all available & compatible AMD GPUs if (amd_gpus_found) { diff --git a/iet.so/src/action.cpp b/iet.so/src/action.cpp index 21ba55dd..f071a68b 100644 --- a/iet.so/src/action.cpp +++ b/iet.so/src/action.cpp @@ -659,12 +659,13 @@ int iet_action::get_all_selected_gpus(void) { hipGetDeviceCount(&hip_num_gpu_devices); if (hip_num_gpu_devices < 1) return hip_num_gpu_devices; + rsmi_init(0); // find compatible GPUs to run edp tests amd_gpus_found = fetch_gpu_list(hip_num_gpu_devices, iet_gpus_device_index, - property_device, property_device_id, property_device_all, true); // MCM checks + property_device, property_device_id, property_device_all, + property_device_index, property_device_index_all, true); // MCM checks if(!amd_gpus_found){ - msg = "No devices match criteria from the test configuation."; rvs::lp::Err(msg, MODULE_NAME_CAPS, action_name); rsmi_shut_down(); diff --git a/include/gpu_util.h b/include/gpu_util.h index 631de15f..1d52cf6f 100644 --- a/include/gpu_util.h +++ b/include/gpu_util.h @@ -1,6 +1,6 @@ /******************************************************************************** * - * Copyright (c) 2018-2022 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. * * MIT LICENSE: * Permission is hereby granted, free of charge, to any person obtaining a copy of @@ -36,6 +36,7 @@ extern int gpu_num_subdirs(const char* dirpath, const char* prefix); extern void gpu_get_all_location_id(std::vector* pgpus_location_id); extern void gpu_get_all_gpu_id(std::vector* pgpus_id); +extern void gpu_get_all_gpu_idx(std::vector* pgpus_idx); extern void gpu_get_all_device_id(std::vector* pgpus_device_id); extern void gpu_get_all_node_id(std::vector* pgpus_node_id); extern void gpu_get_all_domain_id(std::vector* pgpus_domain_id, @@ -43,6 +44,7 @@ extern void gpu_get_all_domain_id(std::vector* pgpus_domain_id, extern bool gpu_check_if_mcm_die (int idx); extern int gpu_hip_to_smi_index(int hip_index, uint32_t* smi_index); extern void gpu_get_all_pci_bdf(std::vector& ppci_bdf); +extern bool gpu_check_if_gpu_indexes (const std::vector &idx); namespace rvs { @@ -63,6 +65,7 @@ class gpulist { static int node2gpu(const uint16_t NodeID, uint16_t* pGpuID); static int location2device(const uint16_t LocationID, uint16_t* pDeviceID); static int gpu2device(const uint16_t GpuID, uint16_t* pDeviceID); + static int gpu2gpuindex(const uint16_t GpuID, uint16_t* pGpuIdx); static int location2node(const uint16_t LocationID, uint16_t* pNodeID); static int gpu2node(const uint16_t GpuID, uint16_t* pNodeID); static int gpu2domain(const uint16_t GpuID, uint16_t* pDomain); @@ -76,6 +79,8 @@ class gpulist { static std::vector location_id; //! Array of GPU IDs static std::vector gpu_id; + //! Array of GPU Indexes + static std::vector gpu_idx; //! Array of device IDs static std::vector device_id; //! Array of node IDs diff --git a/include/rvs_util.h b/include/rvs_util.h index af37a172..05f0c41d 100644 --- a/include/rvs_util.h +++ b/include/rvs_util.h @@ -131,8 +131,9 @@ int rvs_util_parse(const std::string& buff, void *json_node_create(std::string module_name, std::string action_name, int log_level); bool fetch_gpu_list(int hip_num_gpu_devices, map& gpus_device_index, - const std::vector& property_device, - const int& property_device_id, bool property_device_all, bool mcm_check = false); + const std::vector& property_device, const int& property_device_id, + bool property_device_all, const std::vector& property_device_index, + bool property_device_index_all, bool mcm_check = false); void getBDF(int idx ,unsigned int& domain,unsigned int& bus,unsigned int& device,unsigned int& function); int display_gpu_info(void); void *json_list_create(std::string lname, int log_level); diff --git a/mem.so/src/action.cpp b/mem.so/src/action.cpp index 59909952..a6d02d5f 100644 --- a/mem.so/src/action.cpp +++ b/mem.so/src/action.cpp @@ -290,7 +290,8 @@ int mem_action::get_all_selected_gpus(void) { // iterate over all available & compatible AMD GPUs amd_gpus_found = fetch_gpu_list(hip_num_gpu_devices, mem_gpus_device_index, - property_device, property_device_id, property_device_all); + property_device, property_device_id, property_device_all, + property_device_index, property_device_index_all); if (amd_gpus_found) { if (do_mem_stress_test(mem_gpus_device_index)) return 0; diff --git a/pbqt.so/src/action.cpp b/pbqt.so/src/action.cpp index 4bd5c37a..d2a4704e 100644 --- a/pbqt.so/src/action.cpp +++ b/pbqt.so/src/action.cpp @@ -297,6 +297,7 @@ int pbqt_action::create_threads() { std::string msg; std::vector gpu_id; + std::vector gpu_idx; std::vector gpu_device_id; uint16_t transfer_ix = 0; bool bmatch_found = false; @@ -304,9 +305,11 @@ int pbqt_action::create_threads() { char dstgpuid_buff[12]; gpu_get_all_gpu_id(&gpu_id); + gpu_get_all_gpu_idx(&gpu_idx); gpu_get_all_device_id(&gpu_device_id); for (size_t i = 0; i < gpu_id.size(); i++) { // all possible sources + // filter out by source device id if (property_device_id > 0) { if (property_device_id != gpu_device_id[i]) { @@ -315,7 +318,7 @@ int pbqt_action::create_threads() { } // filter out by listed sources - if (!property_device_all) { + if (!property_device_all && property_device.size()) { const auto it = std::find(property_device.cbegin(), property_device.cend(), gpu_id[i]); @@ -323,6 +326,17 @@ int pbqt_action::create_threads() { continue; } } + + // filter out by listed sources + if (!property_device_index_all && property_device_index.size()) { + const auto it = std::find(property_device_index.cbegin(), + property_device_index.cend(), + gpu_idx[i]); + if (it == property_device_index.cend()) { + continue; + } + } + // if property_device_all selected, iteration starts at 0 and all pairs covered. for (size_t j = 0; j < gpu_id.size(); j++) { // all possible peers RVSTRACE_ diff --git a/pebb.so/src/action.cpp b/pebb.so/src/action.cpp index 8b237e10..140311bc 100644 --- a/pebb.so/src/action.cpp +++ b/pebb.so/src/action.cpp @@ -169,12 +169,14 @@ bool pebb_action::get_all_pebb_config_keys(void) { int pebb_action::create_threads() { std::string msg; std::vector gpu_id; + std::vector gpu_idx; std::vector gpu_device_id; uint16_t transfer_ix = 0; bool bmatch_found = false; RVSTRACE_ gpu_get_all_gpu_id(&gpu_id); + gpu_get_all_gpu_idx(&gpu_idx); gpu_get_all_device_id(&gpu_device_id); RVSTRACE_ @@ -190,7 +192,7 @@ int pebb_action::create_threads() { // filter out by listed sources RVSTRACE_ - if (!property_device_all) { + if (!property_device_all && property_device.size()) { RVSTRACE_ const auto it = std::find(property_device.cbegin(), property_device.cend(), @@ -201,6 +203,17 @@ int pebb_action::create_threads() { } } + // filter out by listed sources + if (!property_device_index_all && property_device_index.size()) { + const auto it = std::find(property_device_index.cbegin(), + property_device_index.cend(), + gpu_idx[i]); + if (it == property_device_index.cend()) { + RVSTRACE_ + continue; + } + } + uint16_t dstnode; int srcnode; diff --git a/peqt.so/src/action.cpp b/peqt.so/src/action.cpp index 13c190c0..87c3da33 100644 --- a/peqt.so/src/action.cpp +++ b/peqt.so/src/action.cpp @@ -174,6 +174,11 @@ bool peqt_action::get_all_common_config_keys(void) { rvs::lp::Log(msg, rvs::loginfo); } + if (property_device_index.size() || property_device.size()) { + property_device_all = false; + property_device_index_all = false; + } + return res; } @@ -441,18 +446,34 @@ int peqt_action::run(void) { continue; } + uint16_t gpu_idx; + if (rvs::gpulist::gpu2gpuindex(gpu_id, &gpu_idx)) { + RVSTRACE_ + continue; + } + // check for deviceid filtering if (property_device_id > 0 && dev->device_id != property_device_id) { RVSTRACE_ continue; } - if (!property_device_all) { + if (!property_device_all && property_device.size()) { RVSTRACE_ if (find(property_device.begin(), property_device.end(), gpu_id) == property_device.end()) { RVSTRACE_ - continue; + continue; + } + } + RVSTRACE_ + + if (!property_device_index_all && property_device_index.size()) { + RVSTRACE_ + if (find(property_device_index.begin(), property_device_index.end(), gpu_idx) == + property_device_index.end()) { + RVSTRACE_ + continue; } } RVSTRACE_ diff --git a/perf.so/src/action.cpp b/perf.so/src/action.cpp index 9eb2d260..1f3e6890 100644 --- a/perf.so/src/action.cpp +++ b/perf.so/src/action.cpp @@ -1,6 +1,6 @@ /******************************************************************************** * - * Copyright (c) 2018-2022 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. * * MIT LICENSE: * Permission is hereby granted, free of charge, to any person obtaining a copy of @@ -422,7 +422,8 @@ int perf_action::get_all_selected_gpus(void) { // iterate over all available & compatible AMD GPUs amd_gpus_found = fetch_gpu_list(hip_num_gpu_devices, perf_gpus_device_index, - property_device, property_device_id, property_device_all); + property_device, property_device_id, property_device_all, + property_device_index, property_device_index_all); if (amd_gpus_found) { if (do_gpu_stress_test(perf_gpus_device_index)) return 0; diff --git a/pesm.so/include/worker.h b/pesm.so/include/worker.h index 1ad7ae5d..17981c46 100644 --- a/pesm.so/include/worker.h +++ b/pesm.so/include/worker.h @@ -1,6 +1,6 @@ /******************************************************************************** * - * Copyright (c) 2018-2022 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. * * MIT LICENSE: * Permission is hereby granted, free of charge, to any person obtaining a copy of @@ -61,6 +61,8 @@ class Worker : public rvs::ThreadBase { void set_deviceid(const int id) { device_id = id; } //! Sets GPU IDs for filtering void set_gpuids(const std::vector& GpuIds); + //! Sets GPU indexes for filtering + void set_gpuidx(const std::vector& GpuIdx); //! Sets GPU IDs for filtering (string used in messages) //! @param Devices List of devices to monitor void set_strgpuids(const std::string& Devices) { strgpuids = Devices; } @@ -81,8 +83,12 @@ class Worker : public rvs::ThreadBase { int device_id; //! GPU id filtering flag bool bfiltergpu; + //! GPU indexes filtering flag + bool bfiltergpuidx; //! list of GPU devices to monitor std::vector gpuids; + //! list of GPU device indexes to monitor + std::vector gpuidx; //! list of GPU devices to monitor (string used in messages) std::string strgpuids; //! Name of the action which initiated monitoring diff --git a/pesm.so/src/action.cpp b/pesm.so/src/action.cpp index 55bea05f..9f942433 100644 --- a/pesm.so/src/action.cpp +++ b/pesm.so/src/action.cpp @@ -238,6 +238,7 @@ int pesm_action::run(void) { pworker->set_action(*this); pworker->json(bjson); pworker->set_gpuids(property_device); + pworker->set_gpuidx(property_device_index); pworker->set_deviceid(property_device_id); // start worker thread diff --git a/pesm.so/src/worker.cpp b/pesm.so/src/worker.cpp index f263c149..6cafff15 100644 --- a/pesm.so/src/worker.cpp +++ b/pesm.so/src/worker.cpp @@ -53,6 +53,7 @@ using std::map; Worker::Worker() { bfiltergpu = false; + bfiltergpuidx = false; } Worker::~Worker() {} @@ -67,6 +68,17 @@ void Worker::set_gpuids(const std::vector& GpuIds) { } } +/** + * @brief Sets GPU Indexes for filtering + * @arg GpuIdx Array of GPU indexes + */ +void Worker::set_gpuidx(const std::vector& GpuIdx) { + gpuidx = GpuIdx; + if (gpuidx.size()) { + bfiltergpuidx = true; + } +} + /** * @brief Thread function * @@ -135,6 +147,10 @@ void Worker::run() { if (rvs::gpulist::location2gpu(dev_location_id, &gpu_id)) continue; + uint16_t gpu_idx; + if (rvs::gpulist::gpu2gpuindex(gpu_id, &gpu_idx)) + continue; + // device_id filtering if ( device_id != 0 && dev->device_id != device_id) continue; @@ -146,6 +162,13 @@ void Worker::run() { continue; } + // gpu index filtering + if (bfiltergpuidx) { + auto itgpuidx = find(gpuidx.begin(), gpuidx.end(), gpu_idx); + if (itgpuidx == gpuidx.end()) + continue; + } + rvs::lp::get_ticks(&sec, &usec); // get current speed for the link diff --git a/rvs/src/rvsexec.cpp b/rvs/src/rvsexec.cpp index dc1e3347..d6388d0a 100644 --- a/rvs/src/rvsexec.cpp +++ b/rvs/src/rvsexec.cpp @@ -457,8 +457,8 @@ void rvs::exec::do_help() { cout << "-g --listGpus List all the GPUs available in the machine, that RVS supports and\n"; cout << " has visibility.\n\n"; - cout << "-i --indexes Comma separated list of GPU ids to run test on. This overrides\n"; - cout << " the device values specified for every actions in the\n"; + cout << "-i --indexes Comma separated list of GPU ids/indexes to run test on. This overrides\n"; + cout << " the device/device_index values specified for every actions in the\n"; cout << " configuration file, including the ‘all’ value.\n\n"; cout << "-j --json Generate output file in JSON format.\n"; @@ -483,7 +483,7 @@ void rvs::exec::do_help() { cout << "-h --help Display usage information and exit.\n\n"; } -//! Reports list of AMD GPUs presnt in the system +//! Reports list of AMD GPUs present in the system int rvs::exec::do_gpu_list() { cout << "\nROCm Validation Suite (version " << RVS_VERSION_STRING << ")\n\n"; diff --git a/rvs/src/rvsexec_do_yaml.cpp b/rvs/src/rvsexec_do_yaml.cpp index 2b97488e..6452a05d 100644 --- a/rvs/src/rvsexec_do_yaml.cpp +++ b/rvs/src/rvsexec_do_yaml.cpp @@ -323,8 +323,8 @@ int rvs::exec::do_yaml_properties(const YAML::Node& node, const std::string& module_name, rvs::if1* pif1) { int sts = 0; - string indexes; + bool indexes_provided = false; if (rvs::options::has_option("-i", &indexes) && (!indexes.empty())) indexes_provided = true; @@ -338,16 +338,35 @@ int rvs::exec::do_yaml_properties(const YAML::Node& node, it->first.as())) { // pass properties collection to .so action object sts += do_yaml_properties_collection(it->second, - it->first.as(), - pif1); + it->first.as(), + pif1); } else { + // just set this one property if (indexes_provided && it->first.as() == "device") { + std::replace(indexes.begin(), indexes.end(), ',', ' '); - sts += pif1->property_set("device", indexes); + + std::vector idx; + + // parse key value into std::vector + auto strarray = str_split(indexes, " "); + + // convert str arary into uint16_t array + rvs_util_strarr_to_uintarr(strarray, &idx); + + // Check if indexes are gpu indexes or ids + if(gpu_check_if_gpu_indexes (idx)) { + sts += pif1->property_set("device_index", indexes); + sts += pif1->property_set(it->first.as(), + it->second.as()); + } else { + sts += pif1->property_set("device", indexes); + } + } else { sts += pif1->property_set(it->first.as(), - it->second.as()); + it->second.as()); } } } diff --git a/src/gpu_util.cpp b/src/gpu_util.cpp index 9b9ed7e3..2d8ea2de 100644 --- a/src/gpu_util.cpp +++ b/src/gpu_util.cpp @@ -40,6 +40,7 @@ std::vector rvs::gpulist::location_id; std::vector rvs::gpulist::gpu_id; +std::vector rvs::gpulist::gpu_idx; std::vector rvs::gpulist::device_id; std::vector rvs::gpulist::node_id; std::vector rvs::gpulist::domain_id; @@ -187,6 +188,66 @@ void gpu_get_all_device_id(std::vector* pgpus_device_id) { } } +/** + * gets all GPU device indexes + * @param pgpus_device_idx ptr to vector that will store all the GPU indexes + * @return + */ +void gpu_get_all_gpu_idx(std::vector* pgpus_gpu_idx) { + + std::map smi_map; + uint32_t smi_num_devices = 0; + uint64_t gpuid; + + rsmi_init(0); + + rsmi_status_t err = rsmi_num_monitor_devices(&smi_num_devices); + if(err == RSMI_STATUS_SUCCESS){ + for(auto i = 0; i < smi_num_devices; ++i){ + err = rsmi_dev_guid_get(i, &gpuid); + smi_map.insert({gpuid, i}); + } + } + else { + rsmi_shut_down(); + return; + } + rsmi_shut_down(); + + ifstream f_id, f_prop; + char path[KFD_PATH_MAX_LENGTH]; + + std::string prop_name; + int gpu_id; + uint32_t prop_val; + + // Discover the number of nodes: Inside nodes folder there are only folders + // that represent the node number + int num_nodes = gpu_num_subdirs(KFD_SYS_PATH_NODES, ""); + + // get all GPUs device id + for (int node_id = 0; node_id < num_nodes; node_id++) { + snprintf(path, KFD_PATH_MAX_LENGTH, "%s/%d/gpu_id", KFD_SYS_PATH_NODES, + node_id); + f_id.open(path); + snprintf(path, KFD_PATH_MAX_LENGTH, "%s/%d/properties", + KFD_SYS_PATH_NODES, node_id); + f_prop.open(path); + + f_id >> gpu_id; + + if (gpu_id != 0) { + + if(smi_map.find(gpu_id) != smi_map.end()) { + (*pgpus_gpu_idx).push_back(smi_map[gpu_id]); + } + } + + f_id.close(); + f_prop.close(); + } +} + /** * gets all GPUS nodes * @param pgpus_node_id ptr to vector that will store all the GPU nodes @@ -385,26 +446,30 @@ int gpu_hip_to_smi_index(int hip_index, uint32_t* smi_index) { return -1; } + rsmi_init(0); + rsmi_status_t err = rsmi_num_monitor_devices(&smi_num_devices); - if( err == RSMI_STATUS_SUCCESS){ + if(err == RSMI_STATUS_SUCCESS){ for(auto i = 0; i < smi_num_devices; ++i){ err = rsmi_dev_pci_id_get(i, &val_ui64); smi_map.insert({val_ui64, i}); } } else { + rsmi_shut_down(); return -1; } - + rsmi_shut_down(); + // compute device location_id (needed to match this device // with one of those found while querying the pci bus unsigned int pDom=0, pBus=0, pDev=0, pFun =0; char pciString[256] = {0}; auto hipret = hipDeviceGetPCIBusId(pciString, 256, hip_index); if (sscanf(pciString, "%04x:%02x:%02x.%01x", reinterpret_cast(&pDom), - reinterpret_cast(&pBus), - reinterpret_cast(&pDev), - reinterpret_cast(&pFun)) != 4) { + reinterpret_cast(&pBus), + reinterpret_cast(&pDev), + reinterpret_cast(&pFun)) != 4) { std::cout << "parsing error in BDF:" << pciString << std::endl; } @@ -425,6 +490,7 @@ int rvs::gpulist::Initialize() { gpu_get_all_location_id(&location_id); gpu_get_all_gpu_id(&gpu_id); + gpu_get_all_gpu_idx(&gpu_idx); gpu_get_all_device_id(&device_id); gpu_get_all_node_id(&node_id); gpu_get_all_domain_id(&domain_id, domain_loc_map); @@ -544,6 +610,24 @@ int rvs::gpulist::gpu2device(const uint16_t GpuID, uint16_t* pDeviceID) { return 0; } +/** + * @brief Given Gpu ID return GPU device index + * @param GpuID Gpu ID of a GPU + * @param pDeviceID device ID of the GPU + * @return 0 if found, -1 otherwise + **/ +int rvs::gpulist::gpu2gpuindex(const uint16_t GpuID, uint16_t* pGpuIdx) { + const auto it = std::find(gpu_id.cbegin(), + gpu_id.cend(), GpuID); + if (it == gpu_id.cend()) { + return -1; + } + + size_t pos = std::distance(gpu_id.cbegin(), it); + *pGpuIdx = gpu_idx[pos]; + return 0; +} + /** * @brief Given Gpu ID return GPU HSA Node ID @@ -628,9 +712,36 @@ int rvs::gpulist::gpu2domain(const uint16_t GpuID, uint16_t* pDomain) { return -1; } size_t pos = std::distance(gpu_id.cbegin(), it); - std::cout << "For GPU " << GpuID << " domain is " << domain_id[pos] << std::endl; *pDomain = domain_id[pos]; return 0; } +/** + * @brief Check if the indexes list is gpu indexes + * @param + * @return true if indexes are GPU indexes + **/ +bool gpu_check_if_gpu_indexes (const std::vector &index) { + + uint32_t smi_num_devices = 0; + + rsmi_init(0); + + rsmi_status_t err = rsmi_num_monitor_devices(&smi_num_devices); + if(err != RSMI_STATUS_SUCCESS) { + rsmi_shut_down(); + return false; + } + + for(auto i = 0; i < index.size(); i++) { + + if(index[i] >= smi_num_devices) { + rsmi_shut_down(); + return false; + } + } + + rsmi_shut_down(); + return true; +} diff --git a/src/rvs_util.cpp b/src/rvs_util.cpp index e636f9e8..f99b6474 100644 --- a/src/rvs_util.cpp +++ b/src/rvs_util.cpp @@ -112,83 +112,104 @@ void *json_list_create(std::string lname, int log_level){ * @out: map of gpu id to index in gpus_device_index and returns true if found */ bool fetch_gpu_list(int hip_num_gpu_devices, map& gpus_device_index, - const std::vector& property_device, - const int& property_device_id, bool property_device_all, bool mcm_check){ - bool amd_gpus_found = false; - bool mcm_die = false; - bool amd_mcm_gpu_found = false; - for (int i = 0; i < hip_num_gpu_devices; i++) { - // get GPU device properties - hipDeviceProp_t props; - hipGetDeviceProperties(&props, i); - - // compute device location_id (needed in order to identify this device - // in the gpus_id/gpus_device_id list - unsigned int pDom, pBus, pDev, pFun; - getBDF(i, pDom, pBus, pDev, pFun); - unsigned int dev_location_id = - ((((unsigned int) (pBus)) << 8) | (((unsigned int)(pDev)) << 3) | ((unsigned int)(pFun))); - uint16_t dev_domain = pDom; - uint16_t devId; - uint16_t gpu_id; - if (rvs::gpulist::domlocation2gpu(dev_domain, dev_location_id, &gpu_id)) { - continue; - } - if (rvs::gpulist::gpu2device(gpu_id, &devId)){ - continue; - } - // filter by device id if needed - if (property_device_id > 0 && property_device_id != devId) - continue; - - // check if this GPU is part of the select ones as per config - // (device = "all" or the gpu_id is in the device: list) - bool cur_gpu_selected = false; - - if (property_device_all) { - cur_gpu_selected = true; - } else { - // search for this gpu in the list - // provided under the property - auto it_gpu_id = find(property_device.begin(), - property_device.end(), - gpu_id); - - if (it_gpu_id != property_device.end()) - cur_gpu_selected = true; - } - // if mcm check enabled, print message if device is MCM - // MCM is only for mi200, so check only if gfx90a - if (mcm_check && (std::string{props.gcnArchName}.find("gfx90a") != std::string::npos) ){ - std::stringstream msg_stream; - mcm_die = gpu_check_if_mcm_die(i); - if (mcm_die) { - msg_stream.str(""); - msg_stream << "GPU ID : " << std::setw(5) << gpu_id << " - " << "Device : " << std::setw(5) << devId << - " - " << "GPU is a die/chiplet in Multi-Chip Module (MCM) GPU"; - rvs::lp::Log(msg_stream.str(), rvs::logresults); - - amd_mcm_gpu_found = true; - } - } - // excluding secondary die from test list, drops power reading substantially - if (cur_gpu_selected) { // need not exclude secondary die, just log it out. - gpus_device_index.insert - (std::pair(i, gpu_id)); - amd_gpus_found = true; - } - mcm_die = false; + const std::vector& property_device, const int& property_device_id, + bool property_device_all, const std::vector& property_device_index, + bool property_device_index_all, bool mcm_check) { + + bool amd_gpus_found = false; + bool mcm_die = false; + bool amd_mcm_gpu_found = false; + + for (int i = 0; i < hip_num_gpu_devices; i++) { + + // get GPU device properties + hipDeviceProp_t props; + hipGetDeviceProperties(&props, i); + + // compute device location_id (needed in order to identify this device + // in the gpus_id/gpus_device_id list + unsigned int pDom, pBus, pDev, pFun; + getBDF(i, pDom, pBus, pDev, pFun); + unsigned int dev_location_id = + ((((unsigned int) (pBus)) << 8) | (((unsigned int)(pDev)) << 3) | ((unsigned int)(pFun))); + uint16_t dev_domain = pDom; + uint16_t devId; + uint16_t gpuIdx; + uint16_t gpu_id; + + if (rvs::gpulist::domlocation2gpu(dev_domain, dev_location_id, &gpu_id)) { + continue; + } + if (rvs::gpulist::gpu2device(gpu_id, &devId)){ + continue; + } + if (rvs::gpulist::gpu2gpuindex(gpu_id, &gpuIdx)){ + continue; } - if (amd_mcm_gpu_found && mcm_check) { - std::stringstream msg_stream; + + // filter by device id if needed + if (property_device_id > 0 && property_device_id != devId) + continue; + + // check if this GPU is part of the select ones as per config + // (device = "all" or the gpu_id is in the device: list) + bool cur_gpu_selected = false; + + if (property_device_all || property_device_index_all) { + cur_gpu_selected = true; + } else { + // search for this gpu in the list + // provided under the property + auto it_gpu_id = find(property_device.begin(), + property_device.end(), + gpu_id); + + if (it_gpu_id != property_device.end()) + cur_gpu_selected = true; + + // search for this gpu in the list + // provided under the property + auto it_gpu_idx = find(property_device_index.begin(), + property_device_index.end(), + gpuIdx); + + if (it_gpu_idx != property_device_index.end()) + cur_gpu_selected = true; + } + + // if mcm check enabled, print message if device is MCM + // MCM is only for mi200, so check only if gfx90a + if (mcm_check && (std::string{props.gcnArchName}.find("gfx90a") != std::string::npos) ){ + std::stringstream msg_stream; + mcm_die = gpu_check_if_mcm_die(i); + if (mcm_die) { msg_stream.str(""); - msg_stream << "Note: The system has Multi-Chip Module (MCM) GPU/s." << "\n" - << "In MCM GPU, primary GPU die shows total socket (primary + secondary) power information." << "\n" - << "Secondary GPU die does not have any power information associated with it independently."<< "\n" - << "So, expect power reading from Secondary GPU die as 0."<< "\n"; + msg_stream << "GPU ID : " << std::setw(5) << gpu_id << " - " << "Device : " << std::setw(5) << devId << + " - " << "GPU is a die/chiplet in Multi-Chip Module (MCM) GPU"; rvs::lp::Log(msg_stream.str(), rvs::logresults); - } - return amd_gpus_found; + + amd_mcm_gpu_found = true; + } + } + // excluding secondary die from test list, drops power reading substantially + if (cur_gpu_selected) { // need not exclude secondary die, just log it out. + gpus_device_index.insert + (std::pair(i, gpu_id)); + amd_gpus_found = true; + } + mcm_die = false; + } + + if (amd_mcm_gpu_found && mcm_check) { + std::stringstream msg_stream; + msg_stream.str(""); + msg_stream << "Note: The system has Multi-Chip Module (MCM) GPU/s." << "\n" + << "In MCM GPU, primary GPU die shows total socket (primary + secondary) power information." << "\n" + << "Secondary GPU die does not have any power information associated with it independently."<< "\n" + << "So, expect power reading from Secondary GPU die as 0."<< "\n"; + rvs::lp::Log(msg_stream.str(), rvs::logresults); + } + return amd_gpus_found; } void getBDF(int idx ,unsigned int& domain,unsigned int& bus,unsigned int& device,unsigned int& function){ @@ -217,6 +238,7 @@ int display_gpu_info (void) { int32_t node_id; int32_t gpu_id; int32_t device_id; + int32_t smi_index; }; char buf[256]; @@ -251,6 +273,12 @@ int display_gpu_info (void) { if (rvs::gpulist::gpu2device(gpu_id, &dev_id)){ continue; } + + uint32_t smi_index; + if (gpu_hip_to_smi_index(i, &smi_index)){ + continue; + } + hipDeviceGetPCIBusId(buf, 256, i); device_info info; info.bus = buf; @@ -258,12 +286,14 @@ int display_gpu_info (void) { info.node_id = node_id; info.gpu_id = gpu_id; info.device_id = dev_id; - gpu_info_list.push_back(info); + info.smi_index = smi_index; + gpu_info_list.push_back(info); } + std::sort(gpu_info_list.begin(), gpu_info_list.end(), [](const struct device_info& a, const struct device_info& b) { - return a.node_id < b.node_id; }); + return a.smi_index < b.smi_index; }); if (!gpu_info_list.empty()) { std::cout << "Supported GPUs available:\n"; for (const auto& info : gpu_info_list) { diff --git a/src/rvsactionbase.cpp b/src/rvsactionbase.cpp index 678d3fec..309d75f7 100644 --- a/src/rvsactionbase.cpp +++ b/src/rvsactionbase.cpp @@ -1,6 +1,6 @@ /******************************************************************************** * - * Copyright (c) 2018-2022 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. * * MIT LICENSE: * Permission is hereby granted, free of charge, to any person obtaining a copy of @@ -54,6 +54,8 @@ rvs::actionbase::actionbase() { property_device_all = true; property_device_index_all = true; property_device_id = 0u; + property_device.clear(); + property_device_index.clear(); callback = nullptr; user_param = 0u; } @@ -134,90 +136,93 @@ void rvs::actionbase::sleep(const unsigned int ms) { * * */ bool rvs::actionbase::get_all_common_config_keys(void) { - string msg, sdevid, sdev; - int error; - bool bsts = true; - if (property_get(RVS_CONF_NAME_KEY, &action_name)) { + string msg, sdevid, sdev; + int error; + bool bsts = true; + + if (property_get(RVS_CONF_NAME_KEY, &action_name)) { rvs::lp::Err("Action name missing", module_name); return false; } - msg = "[" + action_name + "] " + module_name + " " + - " " + " Getting all common properties"; - rvs::lp::Log(msg, rvs::logtrace); - // check if -j flag is passed - if (has_property("cli.-j")) { - bjson = true; - } + msg = "[" + action_name + "] " + module_name + " " + + " " + " Getting all common properties"; + rvs::lp::Log(msg, rvs::logtrace); + // check if -j flag is passed + if (has_property("cli.-j")) { + bjson = true; + } - if (int sts = property_get_device()) { - switch (sts) { + if (int sts = property_get_device()) { + switch (sts) { case 1: msg = "Invalid 'device' key value."; break; case 2: msg = "Missing 'device' key."; break; - } - rvs::lp::Err(msg, module_name, action_name); - bsts = false; - } - - - if (property_get_int(RVS_CONF_DEVICEID_KEY, - &property_device_id, 0u)) { - msg = "Invalid 'deviceid' key value."; - rvs::lp::Err(msg, module_name, action_name); - bsts = false; } + rvs::lp::Err(msg, module_name, action_name); + bsts = false; + } + if (property_get_int(RVS_CONF_DEVICEID_KEY, + &property_device_id, 0u)) { + msg = "Invalid 'deviceid' key value."; + rvs::lp::Err(msg, module_name, action_name); + bsts = false; + } - if (int sts = property_get_device_index()) { - switch (sts) { + if (int sts = property_get_device_index()) { + switch (sts) { case 1: msg = "Invalid 'device_index' key value."; break; case 2: msg = "Missing 'device_index' key."; break; - } - - property_device_index_all = true; - rvs::lp::Log(msg, rvs::loginfo); } + property_device_index_all = true; + rvs::lp::Log(msg, rvs::loginfo); + } - if (property_get(RVS_CONF_PARALLEL_KEY, &property_parallel, false)) { - msg = "invalid '" + - std::string(RVS_CONF_PARALLEL_KEY) + "' key value"; - rvs::lp::Err(msg, module_name, action_name); - bsts = false; - } + if (property_device_index.size() || property_device.size()) { + property_device_all = false; + property_device_index_all = false; + } + + if (property_get(RVS_CONF_PARALLEL_KEY, &property_parallel, false)) { + msg = "invalid '" + + std::string(RVS_CONF_PARALLEL_KEY) + "' key value"; + rvs::lp::Err(msg, module_name, action_name); + bsts = false; + } - error = property_get_int + error = property_get_int (RVS_CONF_COUNT_KEY, &property_count, DEFAULT_COUNT); - if (error != 0) { - msg = "invalid '" + - std::string(RVS_CONF_COUNT_KEY) + "' key value"; - rvs::lp::Err(msg, module_name, action_name); - bsts = false; - } + if (error != 0) { + msg = "invalid '" + + std::string(RVS_CONF_COUNT_KEY) + "' key value"; + rvs::lp::Err(msg, module_name, action_name); + bsts = false; + } - error = property_get_int + error = property_get_int (RVS_CONF_WAIT_KEY, &property_wait, DEFAULT_WAIT); - if (error != 0) { - msg = "invalid '" + - std::string(RVS_CONF_WAIT_KEY) + "' key value"; - bsts = false; - } + if (error != 0) { + msg = "invalid '" + + std::string(RVS_CONF_WAIT_KEY) + "' key value"; + bsts = false; + } - if (duration_mods.find(module_name) == duration_mods.end()) - return bsts; + if (duration_mods.find(module_name) == duration_mods.end()) + return bsts; - if (property_get_int(RVS_CONF_DURATION_KEY, - &property_duration, DEFAULT_DURATION)) { + if (property_get_int(RVS_CONF_DURATION_KEY, + &property_duration, DEFAULT_DURATION)) { msg = "Invalid '" + std::string(RVS_CONF_DURATION_KEY) + - "' key"; + "' key"; rvs::lp::Err(msg, module_name, action_name); bsts = false; } @@ -230,7 +235,7 @@ bool rvs::actionbase::get_all_common_config_keys(void) { bsts = false; } - return bsts; + return bsts; } /** diff --git a/tst.so/src/action.cpp b/tst.so/src/action.cpp index f73565fd..5636d880 100644 --- a/tst.so/src/action.cpp +++ b/tst.so/src/action.cpp @@ -1,6 +1,6 @@ /******************************************************************************** * - * Copyright (c) 2018-2023 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. * * MIT LICENSE: * Permission is hereby granted, free of charge, to any person obtaining a copy of @@ -529,7 +529,8 @@ int tst_action::get_all_selected_gpus(void) { rsmi_init(0); // find compatible GPUs to run tst tests amd_gpus_found = fetch_gpu_list(hip_num_gpu_devices, tst_gpus_device_index, - property_device, property_device_id, property_device_all, true); // MCM checks + property_device, property_device_id, property_device_all, + property_device_index, property_device_index_all, true); // MCM checks if(!amd_gpus_found){ msg = "No devices match criteria from the test configuation.";