Skip to content

Commit

Permalink
[AUTO] refine the logic of checking if target device is virtual device (
Browse files Browse the repository at this point in the history
openvinotoolkit#28307)

### Details:
 - refine the logic to check if the target device is virtual device.
- Fixed the issue that AUTO:NPU,CPU goes to CPU directly (mis-detecting
NPU.AUTO_DETECT device as AUTO device, can be reproduced on LNL
machince)

### Tickets:
 - CVS-160153

---------

Co-authored-by: Chen Peter <peter.chen@intel.com>
  • Loading branch information
yangwang201911 and peterchen-intel authored Jan 14, 2025
1 parent c8c1438 commit 308cb00
Show file tree
Hide file tree
Showing 7 changed files with 161 additions and 18 deletions.
29 changes: 15 additions & 14 deletions src/inference/src/dev/core_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,6 @@ void stripDeviceName(std::string& device, const std::string& substr) {
}
}

bool is_virtual_device(const std::string& device_name) {
return (device_name.find("AUTO") != std::string::npos || device_name.find("MULTI") != std::string::npos ||
device_name.find("HETERO") != std::string::npos || device_name.find("BATCH") != std::string::npos);
};

/**
* @brief Converts / flattens ov::device::properties from
* @code
Expand Down Expand Up @@ -123,7 +118,8 @@ ov::AnyMap flatten_sub_properties(const std::string& user_device_name, const ov:
auto subprop_device_name =
secondary_property->first.substr(subprop_device_name_pos + std::strlen(ov::device::properties.name()) + 1);
// flattening is performed only when config is applicable (see docs for ov::is_config_applicable)
if (ov::is_config_applicable(user_device_name, subprop_device_name) || is_virtual_device(user_device_name)) {
if (ov::is_config_applicable(user_device_name, subprop_device_name) ||
ov::is_virtual_device(user_device_name)) {
// 2.1. keep the secondary property for the other virtual devices, but repack them
auto device_properties = result_properties.find(ov::device::properties.name());
if (device_properties == result_properties.end()) {
Expand Down Expand Up @@ -168,7 +164,7 @@ ov::AnyMap flatten_sub_properties(const std::string& user_device_name, const ov:
// example: core.compile_model("GPU.1", ov::device::properties("GPU", ov::prop1));
update_result_properties(secondary_property->second.as<ov::AnyMap>());
secondary_property = secondary_properties.erase(secondary_property);
} else if (is_virtual_device(user_device_name)) {
} else if (ov::is_virtual_device(user_device_name)) {
// 2.2. keep the secondary property for the other virtual devices
secondary_property++;
continue;
Expand Down Expand Up @@ -200,7 +196,7 @@ struct DevicePriority {
};

DevicePriority get_device_priority_property(const std::string& device_name) {
return is_virtual_device(device_name)
return ov::is_virtual_device(device_name)
? DevicePriority{ov::device::priorities.name(), MatchType::EXACT}
:
// ov::device::properties(GPU.0) can be applied for GPU tile identified by GPU.0.0
Expand All @@ -212,7 +208,7 @@ void clean_batch_properties(const std::string& deviceName, ov::AnyMap& config, c
if (deviceName.find("BATCH") == std::string::npos) {
const auto& batch_timeout_mode = config.find(property_name);
if (batch_timeout_mode != config.end()) {
if (!is_virtual_device(deviceName))
if (!ov::is_virtual_device(deviceName))
config.erase(batch_timeout_mode);
}
}
Expand Down Expand Up @@ -257,6 +253,11 @@ bool ov::is_config_applicable(const std::string& user_device_name, const std::st
return false;
}

bool ov::is_virtual_device(const std::string& device_name) {
return (device_name.find("AUTO") == 0 || device_name.find("MULTI") == 0 || device_name.find("HETERO") == 0 ||
device_name.find("BATCH") == 0);
};

namespace {
ov::Parsed parse_device_config(const std::string& device_name,
const ov::CoreConfig& core_config,
Expand Down Expand Up @@ -345,7 +346,7 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName,
auto parsed = parse_device_config(deviceName, coreConfig, config, keep_auto_batch_property);

// remove core properties for HW devices
if (!is_virtual_device(parsed._deviceName)) {
if (!ov::is_virtual_device(parsed._deviceName)) {
// note: ov::cache_dir kept as plugin may require it
CoreConfig::remove_core_skip_cache_dir(parsed._config);
}
Expand Down Expand Up @@ -637,7 +638,7 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const {
const auto& plugin_name = plugin_impl->get_device_name();

// Check that device plugin name is the same as requested for HW plugins
if (!plugin_name.empty() && !is_virtual_device(plugin_name)) {
if (!plugin_name.empty() && !ov::is_virtual_device(plugin_name)) {
OPENVINO_ASSERT(deviceName.find(plugin_name) != std::string::npos,
ov::util::wstring_to_string(desc.libraryLocation.c_str()),
" is used for ",
Expand Down Expand Up @@ -979,7 +980,7 @@ ov::SoPtr<ov::IRemoteContext> ov::CoreImpl::create_context(const std::string& de
ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name,
const ov::AnyMap& user_properties,
const bool keep_core_property) const {
if (is_virtual_device(full_device_name)) {
if (ov::is_virtual_device(full_device_name)) {
// Considerations:
// 1. in case of virtual devices all the magic will happen on the level when
// virtual device calls ICore::get_supported_property for real HW devices
Expand Down Expand Up @@ -1087,7 +1088,7 @@ std::shared_ptr<const ov::Model> ov::CoreImpl::apply_auto_batching(const std::sh
// e.g. to deduce the #requests correctly
// proxy plugin should also keep the config
// otherwise, no need for this config key in the rest of loading
if (!is_virtual_device(deviceName) && !is_proxy_device(deviceName))
if (!ov::is_virtual_device(deviceName) && !is_proxy_device(deviceName))
config.erase(batch_mode);
if (disabled)
return model;
Expand Down Expand Up @@ -1625,7 +1626,7 @@ ov::CoreConfig::CacheConfig ov::CoreConfig::get_cache_config_for_device(const ov
// if plugin does not explicitly support cache_dir, and if plugin is not virtual, we need to remove
// it from config
if (!util::contains(plugin.get_property(ov::supported_properties), ov::cache_dir) &&
!is_virtual_device(plugin.get_name())) {
!ov::is_virtual_device(plugin.get_name())) {
parsedConfig.erase(ov::cache_dir.name());
}
return tempConfig;
Expand Down
9 changes: 9 additions & 0 deletions src/inference/src/dev/core_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,15 @@ Parsed parseDeviceNameIntoConfig(const std::string& deviceName,
*/
bool is_config_applicable(const std::string& device_name, const std::string& device_name_to_parse);

/**
* @brief Checks whether the dvice is virtual device
*
* @param device_name Target device
* @return true if target device is virtual device(e.g. AUTO, AUTO:XPU, AUTO:XPU.x, MULTI, MULTI:XPU, MULTI:XPU.x,
* HETERO:XPU, HETERO:XPU.x, BATCH:XPU, BATCH:XPU.x)
*/
bool is_virtual_device(const std::string& device_name);

class CoreImpl : public ov::ICore, public std::enable_shared_from_this<ov::ICore> {
private:
mutable std::map<std::string, ov::Plugin> plugins;
Expand Down
44 changes: 44 additions & 0 deletions src/inference/tests/unit/core.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,50 @@ TEST(CoreTests_check_device_name, is_config_applicable) {
ASSERT_EQ(ov::is_config_applicable("BATCH", "BATCH:DEVICE"), false);
}

TEST(CoreTests, check_if_virtual_device) {
ASSERT_EQ(ov::is_virtual_device("DEVICE"), false);
ASSERT_EQ(ov::is_virtual_device("DEVICE.x"), false);
ASSERT_EQ(ov::is_virtual_device("DEVICE.AUTO_DETECT"), false);
// AUTO
ASSERT_EQ(ov::is_virtual_device("AUTO"), true);
ASSERT_EQ(ov::is_virtual_device("AUTO:DEVICE"), true);
ASSERT_EQ(ov::is_virtual_device("AUTO:DEVICE,DEVICE"), true);
ASSERT_EQ(ov::is_virtual_device("AUTO:DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("AUTO:DEVICE.AUTO_DETECT"), true);
ASSERT_EQ(ov::is_virtual_device("AUTO:DEVICE.AUTO_DETECT, DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("AUTO:DEVICE.HETERO_DETECT, DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("AUTO:DEVICE.BATCH_DETECT, DEVICE.x"), true);

// MULTI
ASSERT_EQ(ov::is_virtual_device("MULTI"), true);
ASSERT_EQ(ov::is_virtual_device("MULTI:DEVICE"), true);
ASSERT_EQ(ov::is_virtual_device("MULTI:DEVICE,DEVICE"), true);
ASSERT_EQ(ov::is_virtual_device("MULTI:DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("MULTI:DEVICE.AUTO_DETECT"), true);
ASSERT_EQ(ov::is_virtual_device("MULTI:DEVICE.MULTI_DETECT, DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("MULTI:DEVICE.HETERO_DETECT, DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("MULTI:DEVICE.BATCH_DETECT, DEVICE.x"), true);

// HETERO
ASSERT_EQ(ov::is_virtual_device("HETERO"), true);
ASSERT_EQ(ov::is_virtual_device("HETERO:DEVICE"), true);
ASSERT_EQ(ov::is_virtual_device("HETERO:DEVICE,DEVICE"), true);
ASSERT_EQ(ov::is_virtual_device("HETERO:DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("HETERO:DEVICE.AUTO_DETECT"), true);
ASSERT_EQ(ov::is_virtual_device("HETERO:DEVICE.MULTI_DETECT, DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("HETERO:DEVICE.HETERO_DETECT, DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("HETERO:DEVICE.BATCH_DETECT, DEVICE.x"), true);

// BATCH
ASSERT_EQ(ov::is_virtual_device("BATCH"), true);
ASSERT_EQ(ov::is_virtual_device("BATCH:DEVICE"), true);
ASSERT_EQ(ov::is_virtual_device("BATCH:DEVICE.x"), true);
ASSERT_EQ(ov::is_virtual_device("BATCH:DEVICE.AUTO_DETECT"), true);
ASSERT_EQ(ov::is_virtual_device("BATCH:DEVICE.MULTI_DETECT"), true);
ASSERT_EQ(ov::is_virtual_device("BATCH:DEVICE.HETERO_DETECT"), true);
ASSERT_EQ(ov::is_virtual_device("BATCH:DEVICE.BATCH_DETECT"), true);
}

TEST(CoreTests_parse_device_config, get_device_config) {
auto check_parsed_config = [&](const std::string& device,
const ov::AnyMap& config,
Expand Down
17 changes: 13 additions & 4 deletions src/plugins/auto/src/plugin.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,16 @@ std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& model,
OPENVINO_NOT_IMPLEMENTED;
}

bool Plugin::is_meta_device(const std::string& priorities) const {
std::vector<std::string> candidate_devices = m_plugin_config.parse_priorities_devices(priorities);
for (const auto& device : candidate_devices) {
if (device.find("AUTO") == 0 || device.find("MULTI") == 0) {
return true;
}
}
return false;
}

std::vector<DeviceInformation> Plugin::parse_meta_devices(const std::string& priorities,
const ov::AnyMap& properties) const {
std::vector<DeviceInformation> meta_devices;
Expand Down Expand Up @@ -365,11 +375,10 @@ std::shared_ptr<ov::ICompiledModel> Plugin::compile_model_impl(const std::string
std::unordered_map<std::string, ov::Any> multi_model_config;
std::vector<DeviceInformation> meta_devices;
auto priorities = load_config.get_property(ov::device::priorities);
if (priorities.empty() && !work_mode_auto)
if (priorities.empty() && !work_mode_auto)
OPENVINO_THROW("KEY_MULTI_DEVICE_PRIORITIES key is not set for ", get_device_name());
if (priorities.find("AUTO") != std::string::npos || priorities.find("MULTI") != std::string::npos) {
OPENVINO_THROW("The device candidate list should not include the meta plugin for ", get_device_name());
}
if (is_meta_device(priorities))
OPENVINO_THROW("The meta device should not in the device candidate list: ", priorities);
// check the configure and check if need to set PerfCounters configure to device
// and set filter configure
auto auto_s_context = std::make_shared<ScheduleContext>();
Expand Down
1 change: 1 addition & 0 deletions src/plugins/auto/src/plugin.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class Plugin : public ov::IPlugin {
std::shared_ptr<ov::ICompiledModel> compile_model(const std::string& model_path,
const ov::AnyMap& properties) const override;

MOCKTESTMACRO bool is_meta_device(const std::string& priorities) const;
MOCKTESTMACRO std::vector<auto_plugin::DeviceInformation> parse_meta_devices(const std::string & devices_requests_cfg,
const ov::AnyMap& properties) const;

Expand Down
1 change: 1 addition & 0 deletions src/plugins/auto/tests/unit/include/gmock_plugin.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ namespace mock_auto_plugin {
class MockAutoPlugin : public Plugin {
public:
MOCK_METHOD((std::string), get_device_list, ((const ov::AnyMap&)), (const, override));
MOCK_METHOD((bool), is_meta_device, ((const std::string&)), (const, override));
MOCK_METHOD((std::list<DeviceInformation>),
get_valid_device,
((const std::vector<DeviceInformation>&), const std::string&),
Expand Down
78 changes: 78 additions & 0 deletions src/plugins/auto/tests/unit/meta_device_check_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright (C) 2018-2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "include/auto_unit_test.hpp"

using namespace ov::mock_auto_plugin;
using ConfigParams = std::tuple<std::string, // Priority devices
bool // if throw exception
>;
class IsMetaDeviceInCandidateListTest : public tests::AutoTest, public ::testing::TestWithParam<ConfigParams> {
public:
static std::string getTestCaseName(testing::TestParamInfo<ConfigParams> obj) {
std::string priorityDevices;
bool expectedRet;
std::tie(priorityDevices, expectedRet) = obj.param;
std::ostringstream result;
result << "priorityDevices_" << priorityDevices;
if (expectedRet) {
result << "_expection_true";
} else {
result << "_expection_false";
}
return result.str();
}

void SetUp() override {
ON_CALL(*plugin, is_meta_device).WillByDefault([this](const std::string& priorityDevices) {
return plugin->Plugin::is_meta_device(priorityDevices);
});
std::tie(priorityDevices, expectedRet) = GetParam();
}

protected:
// get Parameter
std::string priorityDevices;
bool expectedRet;
};

TEST_P(IsMetaDeviceInCandidateListTest, CheckMetaDeviceWithPriority) {
EXPECT_CALL(*plugin, is_meta_device(_)).Times(1);
ASSERT_EQ(plugin->is_meta_device(priorityDevices), expectedRet);
}

const std::vector<ConfigParams> testConfigs = {ConfigParams{"CPU", false},
ConfigParams{"GPU", false},
ConfigParams{"OTHER", false},
ConfigParams{"GPU.0", false},
ConfigParams{"GPU.1", false},
ConfigParams{"GPU.AUTO_DETECT", false},
ConfigParams{"GPU.MULTI_DETECT", false},
ConfigParams{"OTHER.AUTO_DETECT", false},
ConfigParams{"CPU,OTHER.AUTO_DETECT", false},
ConfigParams{"GPU,OTHER.AUTO_DETECT", false},
ConfigParams{"GPU,OTHER.MULTI_DETECT", false},
ConfigParams{"AUTO", true},
ConfigParams{"AUTO,CPU", true},
ConfigParams{"OTHER,AUTO", true},
ConfigParams{"AUTO:CPU", true},
ConfigParams{"AUTO:OTHER", true},
ConfigParams{"AUTO.AUTO_DETECT", true},
ConfigParams{"AUTO:AUTO.AUTO_DETECT", true},
ConfigParams{"CPU,AUTO:AUTO.AUTO_DETECT", true},
ConfigParams{"MULTI", true},
ConfigParams{"CPU,MULTI", true},
ConfigParams{"CPU,MULTI.0", true},
ConfigParams{"MULTI:CPU", true},
ConfigParams{"GPU,MULTI:CPU", true},
ConfigParams{"MULTI:OTHER", true},
ConfigParams{"MULTI.AUTO_DETECT", true},
ConfigParams{"MULTI:AUTO.AUTO_DETECT", true},
ConfigParams{"GPU,MULTI:AUTO.AUTO_DETECT", true},
ConfigParams{"GPU,MULTI:MULTI.AUTO_DETECT", true}};

INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests,
IsMetaDeviceInCandidateListTest,
::testing::ValuesIn(testConfigs),
IsMetaDeviceInCandidateListTest::getTestCaseName);

0 comments on commit 308cb00

Please sign in to comment.