|
| 1 | +// Copyright (c) Microsoft Corporation. All rights reserved. |
| 2 | +// Licensed under the MIT License. |
| 3 | + |
| 4 | +#include <filesystem> |
| 5 | +#include <map> |
| 6 | +#include <string> |
| 7 | + |
| 8 | +#include "core/session/onnxruntime_cxx_api.h" |
| 9 | +#include "core/framework/float16.h" |
| 10 | + |
| 11 | +#include "test/util/include/test/test_environment.h" |
| 12 | +#include "test/optimizer/qdq_test_utils.h" |
| 13 | + |
| 14 | +#include "gtest/gtest.h" |
| 15 | +#include "gmock/gmock.h" |
| 16 | + |
| 17 | +using namespace ONNX_NAMESPACE; |
| 18 | +using namespace onnxruntime::logging; |
| 19 | + |
| 20 | +extern std::unique_ptr<Ort::Env> ort_env; |
| 21 | + |
| 22 | +class OVEP_BF16_Tests : public ::testing::TestWithParam<std::string> {}; |
| 23 | + |
| 24 | +namespace detail { |
| 25 | +auto ConstructModel() { |
| 26 | + using namespace onnxruntime; |
| 27 | + using namespace test; |
| 28 | + |
| 29 | + std::unordered_map<std::string, int> domain_to_version; |
| 30 | + domain_to_version[kOnnxDomain] = 19; |
| 31 | + Model model("Bfloat16Tester", true, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), |
| 32 | + domain_to_version, {}, DefaultLoggingManager().DefaultLogger()); |
| 33 | + |
| 34 | + Graph& graph = model.MainGraph(); |
| 35 | + ModelTestBuilder builder(graph); |
| 36 | + auto dim = 4; |
| 37 | + std::vector<float> input_data(dim, 1.0f); |
| 38 | + auto* input = builder.MakeInput<float>({dim}, input_data); |
| 39 | + builder.graph_.SetInputs({input}); |
| 40 | + |
| 41 | + auto* cast_to_bf16 = builder.MakeIntermediate(); |
| 42 | + Node& cast_node = builder.AddNode("Cast", {input}, {cast_to_bf16}, ""); |
| 43 | + cast_node.AddAttribute("to", static_cast<int64_t>(ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16)); |
| 44 | + |
| 45 | + std::vector<onnxruntime::BFloat16> weight_data(dim * dim); |
| 46 | + for (std::size_t i = 0; i < weight_data.size(); ++i) |
| 47 | + weight_data[i] = onnxruntime::BFloat16(static_cast<float>(i % dim) / dim); |
| 48 | + auto* weights = builder.MakeInitializer<onnxruntime::BFloat16>({dim, dim}, weight_data); |
| 49 | + |
| 50 | + auto* matmul_out = builder.MakeIntermediate(); |
| 51 | + builder.AddNode("MatMul", {cast_to_bf16, weights}, {matmul_out}); |
| 52 | + |
| 53 | + std::vector<onnxruntime::BFloat16> weight_data_2(dim * dim); |
| 54 | + for (std::size_t i = 0; i < weight_data_2.size(); ++i) |
| 55 | + weight_data_2[i] = onnxruntime::BFloat16(static_cast<float>(i % dim) / dim); |
| 56 | + auto* weights_2 = builder.MakeInitializer<onnxruntime::BFloat16>({dim, dim}, weight_data_2); |
| 57 | + |
| 58 | + auto* matmul_out_2 = builder.MakeIntermediate(); |
| 59 | + builder.AddNode("MatMul", {matmul_out, weights_2}, {matmul_out_2}); |
| 60 | + |
| 61 | + auto* output = builder.MakeOutput(); |
| 62 | + Node& cast2_node = builder.AddNode("Cast", {matmul_out_2}, {output}); |
| 63 | + cast2_node.AddAttribute("to", static_cast<int64_t>(ONNX_NAMESPACE::TensorProto_DataType_FLOAT)); |
| 64 | + |
| 65 | + builder.SetGraphOutputs(); |
| 66 | + auto st = model.MainGraph().Resolve(); |
| 67 | + if (st != Status::OK()) |
| 68 | + throw std::runtime_error(st.ErrorMessage()); |
| 69 | + return model; |
| 70 | +} |
| 71 | + |
| 72 | +auto ProbeDevice(const std::string& device) { |
| 73 | + static std::map<std::string, bool> is_present; |
| 74 | + if (is_present.find(device) == is_present.end()) { |
| 75 | + Ort::SessionOptions sessionOptions; |
| 76 | + std::unordered_map<std::string, std::string> ov_options; |
| 77 | + ov_options["device_type"] = device; |
| 78 | + try { |
| 79 | + sessionOptions.AppendExecutionProvider_OpenVINO_V2(ov_options); |
| 80 | + is_present[device] = true; |
| 81 | + } catch (...) { |
| 82 | + is_present[device] = false; |
| 83 | + } |
| 84 | + } |
| 85 | + return is_present[device]; |
| 86 | +} |
| 87 | +} // namespace detail |
| 88 | + |
| 89 | +namespace onnxruntime { |
| 90 | +namespace test { |
| 91 | + |
| 92 | +TEST_P(OVEP_BF16_Tests, TestModelConversion) { |
| 93 | + Ort::SessionOptions sessionOptions; |
| 94 | + std::unordered_map<std::string, std::string> ov_options; |
| 95 | + const auto& device = GetParam(); |
| 96 | + if (!::detail::ProbeDevice(device)) |
| 97 | + GTEST_SKIP() << device + " is not available on this machine"; |
| 98 | + |
| 99 | + ov_options["device_type"] = device; |
| 100 | + auto model = ::detail::ConstructModel(); |
| 101 | + sessionOptions.AppendExecutionProvider_OpenVINO_V2(ov_options); |
| 102 | + |
| 103 | + std::string model_data; |
| 104 | + model.ToProto().SerializeToString(&model_data); |
| 105 | + auto model_data_span = AsByteSpan(model_data.data(), model_data.size()); |
| 106 | + try { |
| 107 | + Ort::Session session(*ort_env, model_data_span.data(), model_data_span.size(), sessionOptions); |
| 108 | + } catch (...) { |
| 109 | + FAIL(); |
| 110 | + } |
| 111 | +} |
| 112 | +INSTANTIATE_TEST_SUITE_P(OVEP_Tests, |
| 113 | + OVEP_BF16_Tests, |
| 114 | + ::testing::Values("CPU", "GPU", "NPU")); |
| 115 | +} // namespace test |
| 116 | +} // namespace onnxruntime |
0 commit comments