Skip to content

Commit 6a889ed

Browse files
alalekalalek
committed
Merge pull request opencv#21261 from alalek:dnn_onnx_test_filter_update
2 parents e7c5120 + f3ba88c commit 6a889ed

File tree

4 files changed

+2105
-116
lines changed

4 files changed

+2105
-116
lines changed

‎modules/dnn/test/test_common.impl.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ void initDNNTests()
368368
#if defined(HAVE_HALIDE)
369369
registerGlobalSkipTag(
370370
CV_TEST_TAG_DNN_SKIP_HALIDE
371-
)
371+
);
372372
#endif
373373
#if defined(INF_ENGINE_RELEASE)
374374
registerGlobalSkipTag(

‎modules/dnn/test/test_onnx_conformance.cpp‎

Lines changed: 99 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -898,24 +898,16 @@ static const TestCase testConformanceConfig[] = {
898898
};
899899

900900

901-
struct TestCaseInput
902-
{
903-
std::vector<std::string> input_paths;
904-
std::vector<std::string> output_paths;
905-
std::string model_path;
906-
std::string name;
907-
};
908-
909-
std::ostream& operator<<(std::ostream& os, const TestCaseInput& test_case)
901+
std::ostream& operator<<(std::ostream& os, const TestCase& test_case)
910902
{
911903
return os << test_case.name;
912904
}
913905

914-
typedef tuple<TestCaseInput, tuple<Backend, Target> > ONNXConfParams;
906+
typedef tuple<TestCase, tuple<Backend, Target> > ONNXConfParams;
915907

916908
std::string printOnnxConfParams(const testing::TestParamInfo<ONNXConfParams>& params)
917909
{
918-
TestCaseInput test_case = get<0>(params.param);
910+
TestCase test_case = get<0>(params.param);
919911
Backend backend = get<0>(get<1>(params.param));
920912
Target target = get<1>(get<1>(params.param));
921913

@@ -928,45 +920,11 @@ std::string printOnnxConfParams(const testing::TestParamInfo<ONNXConfParams>& pa
928920
return ss.str();
929921
}
930922

931-
template<typename TString>
932-
static std::string _tf(TString filename, bool required = true)
933-
{
934-
return findDataFile(std::string("dnn/onnx/") + filename, required);
935-
}
936-
937-
std::vector<TestCaseInput> readTestCases()
938-
{
939-
std::vector<TestCaseInput> ret;
940-
for (size_t i = 0; i < sizeof(testConformanceConfig) / sizeof(testConformanceConfig[0]); ++i)
941-
{
942-
const TestCase& test_case = testConformanceConfig[i];
943-
944-
TestCaseInput input;
945-
946-
std::string prefix = cv::format("conformance/node/%s", test_case.name);
947-
input.name = test_case.name;
948-
input.model_path = _tf(cv::format("%s/model.onnx", prefix.c_str()));
949-
950-
for (int i = 0; i < test_case.inputs; ++i)
951-
{
952-
input.input_paths.push_back(_tf(cv::format("%s/test_data_set_0/input_%d.pb", prefix.c_str(), i)));
953-
}
954-
955-
for (int i = 0; i < test_case.outputs; ++i)
956-
{
957-
input.output_paths.push_back(_tf(cv::format("%s/test_data_set_0/output_%d.pb", prefix.c_str(), i)));
958-
}
959-
960-
ret.push_back(input);
961-
}
962-
963-
return ret;
964-
}
965-
966923
class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
967924
{
968925
public:
969-
TestCaseInput test_case;
926+
927+
TestCase test_case;
970928
Backend backend;
971929
Target target;
972930

@@ -978,6 +936,9 @@ class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
978936
static std::set<std::string> opencl_fp16_deny_list;
979937
static std::set<std::string> opencl_deny_list;
980938
static std::set<std::string> cpu_deny_list;
939+
#ifdef HAVE_HALIDE
940+
static std::set<std::string> halide_deny_list;
941+
#endif
981942

982943
Test_ONNX_conformance()
983944
{
@@ -1059,102 +1020,121 @@ class Test_ONNX_conformance : public TestWithParam<ONNXConfParams>
10591020
"" // dummy element of non empty list
10601021
};
10611022
initDenyList(cpu_deny_list, cpu, sizeof(cpu)/sizeof(cpu[0]));
1062-
}
10631023

1064-
void checkFilterLists() const
1065-
{
1066-
const std::string& name = test_case.name;
1067-
if(parser_deny_list.find(name) != parser_deny_list.end())
1068-
{
1069-
applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1070-
}
1071-
1072-
if (backend == DNN_BACKEND_OPENCV)
1073-
{
1074-
if(global_deny_list.find(name) != global_deny_list.end())
1075-
{
1076-
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1077-
}
1078-
if((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end()))
1079-
{
1080-
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1081-
}
1082-
if((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end()))
1083-
{
1084-
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1085-
}
1086-
if((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end()))
1087-
{
1088-
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1089-
}
1090-
}
1091-
#if 0 //def HAVE_HALIDE
1092-
else if (backend == DNN_BACKEND_HALIDE)
1093-
{
1094-
#include "test_onnx_conformance_layer_filter__halide.inl.hpp"
1095-
}
1096-
#endif
1097-
#if 0 //def HAVE_INF_ENGINE
1098-
else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
1099-
{
1100-
#include "test_onnx_conformance_layer_filter__ngraph.inl.hpp"
1101-
}
1102-
#endif
1103-
#if 0 //def HAVE_VULKAN
1104-
else if (backend == DNN_BACKEND_VKCOM)
1105-
{
1106-
#include "test_onnx_conformance_layer_filter__vulkan.inl.hpp"
1107-
}
1108-
#endif
1109-
#if 0 //def HAVE_CUDA
1110-
else if (backend == DNN_BACKEND_CUDA)
1111-
{
1112-
#include "test_onnx_conformance_layer_filter__cuda.inl.hpp"
1113-
}
1024+
#ifdef HAVE_HALIDE
1025+
const char* const halide_deny_list_[] = {
1026+
#include "test_onnx_conformance_layer_filter__halide_denylist.inl.hpp"
1027+
"" // dummy element of non empty list
1028+
};
1029+
initDenyList(halide_deny_list, halide_deny_list_, sizeof(halide_deny_list_)/sizeof(halide_deny_list_[0]));
11141030
#endif
1115-
else
1116-
{
1117-
std::ostringstream ss;
1118-
ss << "No test filter available for backend ";
1119-
PrintTo(backend, &ss);
1120-
ss << ". Run test by default";
1121-
std::cout << ss.str() << std::endl;
1122-
}
11231031
}
1032+
11241033
};
11251034

11261035
std::set<std::string> Test_ONNX_conformance::parser_deny_list;
11271036
std::set<std::string> Test_ONNX_conformance::global_deny_list;
11281037
std::set<std::string> Test_ONNX_conformance::opencl_fp16_deny_list;
11291038
std::set<std::string> Test_ONNX_conformance::opencl_deny_list;
11301039
std::set<std::string> Test_ONNX_conformance::cpu_deny_list;
1040+
#ifdef HAVE_HALIDE
1041+
std::set<std::string> Test_ONNX_conformance::halide_deny_list;
1042+
#endif
11311043

11321044
TEST_P(Test_ONNX_conformance, Layer_Test)
11331045
{
1134-
std::string name = test_case.name;
1046+
const std::string& name = test_case.name;
11351047
ASSERT_FALSE(name.empty());
11361048

11371049
bool checkLayersFallbacks = true;
11381050
bool checkAccuracy = true;
11391051

1140-
checkFilterLists();
1052+
if (parser_deny_list.find(name) != parser_deny_list.end())
1053+
{
1054+
applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1055+
}
1056+
1057+
if (backend == DNN_BACKEND_OPENCV)
1058+
{
1059+
if (global_deny_list.find(name) != global_deny_list.end())
1060+
{
1061+
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1062+
}
1063+
if ((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end()))
1064+
{
1065+
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1066+
}
1067+
if ((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end()))
1068+
{
1069+
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1070+
}
1071+
if ((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end()))
1072+
{
1073+
applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1074+
}
1075+
}
1076+
#ifdef HAVE_HALIDE
1077+
else if (backend == DNN_BACKEND_HALIDE)
1078+
{
1079+
if (halide_deny_list.find(name) != halide_deny_list.end())
1080+
{
1081+
applyTestTag(CV_TEST_TAG_DNN_SKIP_HALIDE, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE);
1082+
}
1083+
}
1084+
#endif
1085+
#ifdef HAVE_INF_ENGINE
1086+
else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
1087+
{
1088+
#include "test_onnx_conformance_layer_filter__openvino.inl.hpp"
1089+
}
1090+
#endif
1091+
#if 0 //def HAVE_VULKAN
1092+
else if (backend == DNN_BACKEND_VKCOM)
1093+
{
1094+
#include "test_onnx_conformance_layer_filter__vulkan.inl.hpp"
1095+
}
1096+
#endif
1097+
#if 0 //def HAVE_CUDA
1098+
else if (backend == DNN_BACKEND_CUDA)
1099+
{
1100+
#include "test_onnx_conformance_layer_filter__cuda.inl.hpp"
1101+
}
1102+
#endif
1103+
else
1104+
{
1105+
std::ostringstream ss;
1106+
ss << "No test filter available for backend ";
1107+
PrintTo(backend, &ss);
1108+
ss << ". Run test by default";
1109+
std::cout << ss.str() << std::endl;
1110+
}
11411111

11421112
std::vector<Mat> inputs;
11431113
std::vector<Mat> ref_outputs;
11441114

1115+
std::string prefix = cv::format("dnn/onnx/conformance/node/%s", test_case.name);
1116+
11451117
Net net;
11461118
try
11471119
{
1120+
std::string model_path = findDataFile(prefix + "/model.onnx");
1121+
11481122
//cout << "Read ONNX inputs..." << endl;
1149-
std::transform(test_case.input_paths.begin(), test_case.input_paths.end(),
1150-
std::back_inserter(inputs), readTensorFromONNX);
1123+
for (int i = 0; i < test_case.inputs; ++i)
1124+
{
1125+
Mat input = readTensorFromONNX(findDataFile(prefix + cv::format("/test_data_set_0/input_%d.pb", i)));
1126+
inputs.push_back(input);
1127+
}
11511128

11521129
//cout << "Read ONNX reference outputs..." << endl;
1153-
std::transform(test_case.output_paths.begin(), test_case.output_paths.end(),
1154-
std::back_inserter(ref_outputs), readTensorFromONNX);
1130+
for (int i = 0; i < test_case.outputs; ++i)
1131+
{
1132+
Mat output = readTensorFromONNX(findDataFile(prefix + cv::format("/test_data_set_0/output_%d.pb", i)));
1133+
ref_outputs.push_back(output);
1134+
}
11551135

11561136
//cout << "Parse model..." << endl;
1157-
net = readNetFromONNX(test_case.model_path);
1137+
net = readNetFromONNX(model_path);
11581138
if (net.empty())
11591139
{
11601140
applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER);
@@ -1244,7 +1224,11 @@ TEST_P(Test_ONNX_conformance, Layer_Test)
12441224
}
12451225

12461226
INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_conformance,
1247-
testing::Combine(testing::ValuesIn(readTestCases()), dnnBackendsAndTargets()),
1248-
printOnnxConfParams);
1227+
testing::Combine(
1228+
testing::ValuesIn(testConformanceConfig),
1229+
dnnBackendsAndTargets(/*withInferenceEngine=*/true, /*withHalide=*/true)
1230+
),
1231+
printOnnxConfParams
1232+
);
12491233

12501234
};

‎modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
"test_add",
2+
"test_add_bcast",
3+
"test_averagepool_2d_ceil",
4+
"test_averagepool_2d_pads_count_include_pad",
5+
"test_averagepool_2d_precomputed_pads_count_include_pad",
6+
"test_averagepool_2d_precomputed_strides",
7+
"test_averagepool_2d_same_lower",
8+
"test_averagepool_2d_same_upper",
9+
"test_cast_FLOAT_to_STRING",
10+
"test_cast_STRING_to_FLOAT",
11+
"test_castlike_FLOAT_to_STRING_expanded",
12+
"test_castlike_STRING_to_FLOAT_expanded",
13+
"test_concat_1d_axis_negative_1",
14+
"test_concat_3d_axis_1",
15+
"test_div",
16+
"test_div_bcast",
17+
"test_elu",
18+
"test_elu_default",
19+
"test_exp",
20+
"test_flatten_axis0",
21+
"test_flatten_axis2",
22+
"test_flatten_axis3",
23+
"test_flatten_negative_axis1",
24+
"test_flatten_negative_axis2",
25+
"test_flatten_negative_axis4",
26+
"test_leakyrelu",
27+
"test_leakyrelu_default",
28+
"test_logsoftmax_axis_1",
29+
"test_logsoftmax_axis_1_expanded",
30+
"test_logsoftmax_default_axis",
31+
"test_logsoftmax_example_1",
32+
"test_logsoftmax_large_number",
33+
"test_matmul_2d",
34+
"test_matmul_3d",
35+
"test_matmul_4d",
36+
"test_maxpool_2d_dilations",
37+
"test_maxpool_2d_same_lower",
38+
"test_maxpool_with_argmax_2d_precomputed_pads",
39+
"test_maxpool_with_argmax_2d_precomputed_strides",
40+
"test_mul",
41+
"test_mul_bcast",
42+
"test_neg",
43+
"test_reduce_max_default_axes_keepdim_example",
44+
"test_reduce_max_default_axes_keepdims_random",
45+
"test_reduce_max_do_not_keepdims_example",
46+
"test_reduce_max_do_not_keepdims_random",
47+
"test_reduce_max_keepdims_example",
48+
"test_reduce_max_keepdims_random",
49+
"test_reduce_max_negative_axes_keepdims_example",
50+
"test_reduce_max_negative_axes_keepdims_random",
51+
"test_relu",
52+
"test_sigmoid",
53+
"test_softmax_axis_1",
54+
"test_softmax_axis_1_expanded",
55+
"test_softmax_default_axis",
56+
"test_softmax_large_number",
57+
"test_sub",
58+
"test_sub_bcast",
59+
"test_tanh",
60+
"test_upsample_nearest",

0 commit comments

Comments
 (0)