Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion sdk_v2/cpp/include/foundry_local/foundry_local_c.h
Original file line number Diff line number Diff line change
Expand Up @@ -1036,7 +1036,12 @@ struct flCatalogApi {
/// Register a model in a local catalog without taking ownership of its assets.
/// `model_path` must identify a model directory containing genai_config.json.
/// `model_id` must use the canonical `<name>:<version>` format and be unique in the local catalog.
/// The metadata is copied; model identity and location are taken only from the explicit arguments.
/// `metadata.task` is required. Application-owned properties, including display name, publisher, runtime,
/// modalities, and arbitrary string or integer properties, are preserved; missing values receive authoritative
/// defaults when available.
/// Identity, alias, type, timestamps, context length, and prompt templates are derived by the SDK from the
/// arguments, registration time, and genai_config.json. Caller-supplied location and internal catalog metadata are
/// ignored.
FL_API_STATUS(RegisterModel, _In_ flCatalog* catalog, _In_ const char* model_path,
_In_ const char* model_id, _In_ const flModelInfo* metadata,
_Outptr_ flModel** out_model);
Expand Down
5 changes: 4 additions & 1 deletion sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h
Original file line number Diff line number Diff line change
Expand Up @@ -808,7 +808,10 @@ class ICatalog {
const std::string& variant_name = {},
int max_versions = 50) = 0;

/// Register existing local model assets. `model_id` must use `<name>:<version>`; metadata is copied.
/// Register existing local model assets. `model_id` must use `<name>:<version>` and `metadata.Task()` is required.
/// Application-owned properties are preserved and may receive authoritative defaults. Identity, alias, type,
/// timestamps, context length, and prompt templates are SDK-derived; caller location and internal metadata are
/// ignored.
/// The catalog does not take ownership of `model_path` and never deletes its contents.
virtual std::unique_ptr<IModel> RegisterModel(const std::string& model_path, const std::string& model_id,
const ModelInfo& metadata) = 0;
Expand Down
119 changes: 116 additions & 3 deletions sdk_v2/cpp/src/catalog/local_model_catalog.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include "catalog/local_model_catalog.h"

#include "exception.h"
#include "inferencing/execution_provider.h"
#include "inferencing/generative/genai_config.h"
#include "util/file_lock.h"
#include "util/time_utils.h"
Expand Down Expand Up @@ -50,6 +51,70 @@ struct ParsedModelId {
int version;
};

std::optional<std::string> ConfigExecutionProvider(const GenAIConfig& config) {
const auto config_provider = config.DefaultProvider();
if (config_provider.empty()) {
return std::nullopt;
}

auto provider = EPUtils::StringtoEP(config_provider);
if (provider == ExecutionProvider::kUnknown) {
return std::nullopt;
}

return std::string(EPUtils::EPtoRegistrationName(provider));
}

std::optional<std::string_view> DeviceTypeForExecutionProvider(std::string_view provider_name) {
const auto provider = EPUtils::StringtoEP(provider_name);
switch (provider) {
case ExecutionProvider::kCPU:
return "CPU";
case ExecutionProvider::kCUDA:
case ExecutionProvider::kWebGPU:
case ExecutionProvider::kTensorRT_RTX:
return "GPU";
case ExecutionProvider::kVitisAI:
case ExecutionProvider::kRyzenAI:
case ExecutionProvider::kQNN:
return "NPU";
Comment thread
selenayang888 marked this conversation as resolved.
default:
return std::nullopt;
}
}

void ApplyTaskDefaults(ModelInfo& info) {
Comment thread
selenayang888 marked this conversation as resolved.
if (info.task == "chat-completion") {
if (!info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR)) {
info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR, "text");
}
if (!info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR)) {
info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR, "text");
}
} else if (info.task == "vision-language-chat") {
if (!info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR)) {
info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR, "text,image");
}
if (!info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR)) {
info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR, "text");
}
} else if (info.task == "automatic-speech-recognition") {
if (!info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR)) {
info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR, "audio");
}
if (!info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR)) {
info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR, "text");
}
} else if (info.task == "embeddings") {
if (!info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR)) {
info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR, "text");
}
if (!info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR)) {
info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR, "embeddings");
}
}
}

ParsedModelId ParseModelId(const std::string& model_id) {
const auto separator = model_id.find(':');
if (separator == std::string::npos || separator == 0 || separator + 1 == model_id.size() ||
Expand Down Expand Up @@ -118,6 +183,25 @@ void RemoveLegacyRegistrationProperties(ModelInfo& info) {
info.int_properties.erase(kLegacyVersionProperty);
}

void RemoveSdkOwnedProperty(ModelInfo& info, std::string_view key) {
if (const auto string_property = info.string_properties.find(key);
string_property != info.string_properties.end()) {
info.string_properties.erase(string_property);
}
if (const auto int_property = info.int_properties.find(key); int_property != info.int_properties.end()) {
info.int_properties.erase(int_property);
}
}

void RemoveSdkOwnedProperties(ModelInfo& info) {
RemoveSdkOwnedProperty(info, FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR);
RemoveSdkOwnedProperty(info, FOUNDRY_LOCAL_MODEL_PROP_ENTITY_TYPE_STR);
RemoveSdkOwnedProperty(info, FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR);
RemoveSdkOwnedProperty(info, FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT);
RemoveSdkOwnedProperty(info, FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT);
RemoveSdkOwnedProperty(info, FOUNDRY_LOCAL_MODEL_PROP_CREATION_TIME_STR);
}

bool IsLegacyIntegerProperty(std::string_view key) {
return key == FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT ||
key == FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT ||
Expand Down Expand Up @@ -217,7 +301,7 @@ Model* LocalModelCatalog::RegisterModel(const std::string& model_path_value, con
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_path must contain a regular genai_config.json file");
}

GenAIConfig::LoadFromFile(config_path.string());
const auto genai_config = GenAIConfig::LoadFromFile(config_path.string());

ListModels();
Registration registration;
Expand All @@ -232,7 +316,8 @@ Model* LocalModelCatalog::RegisterModel(const std::string& model_path_value, con
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_id is already registered: " + model_id);
}

registration = {ResolveMetadata(metadata, model_id, parsed_id.name, parsed_id.version), model_path.string()};
registration = {
ResolveMetadata(metadata, model_id, parsed_id.name, parsed_id.version, genai_config), model_path.string()};

registrations.push_back(registration);
SaveRegistrations(registrations);
Expand Down Expand Up @@ -316,22 +401,50 @@ void LocalModelCatalog::UnregisterModel(const std::string& alias_or_model_id) {
}

ModelInfo LocalModelCatalog::ResolveMetadata(const ModelInfo& metadata, const std::string& model_id,
const std::string& name, int version) const {
const std::string& name, int version,
const GenAIConfig& genai_config) const {
auto resolved = metadata;
RemoveLegacyRegistrationProperties(resolved);
RemoveSdkOwnedProperties(resolved);

resolved.detected_region.clear();
resolved.prompt_templates = {};
resolved.model_settings = {};
Comment thread
selenayang888 marked this conversation as resolved.
if (genai_config.model) {
resolved.prompt_templates.CopyFromMap(genai_config.model->prompt_templates);
}
resolved.alias = DeriveAlias(name);
resolved.name = name;
resolved.version = version;
resolved.model_id = model_id;
resolved.uri.clear();
if (!resolved.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_DISPLAY_NAME_STR)) {
resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_DISPLAY_NAME_STR, name);
}
if (!resolved.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR)) {
resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR, "local");
}
if (!resolved.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_EP_STR)) {
if (const auto provider = ConfigExecutionProvider(genai_config)) {
resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_EP_STR, *provider);
}
}
if (!resolved.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_DEVICE_TYPE_STR)) {
if (const auto* provider = resolved.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_EP_STR)) {
if (const auto device_type = DeviceTypeForExecutionProvider(*provider)) {
resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_DEVICE_TYPE_STR, std::string(*device_type));
}
}
}
ApplyTaskDefaults(resolved);
resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR, "LocalRegistration");
resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_ENTITY_TYPE_STR, "Model");
resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR, "ONNX");

if (genai_config.model && genai_config.model->context_length > 0) {
resolved.SetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT, genai_config.model->context_length);
}

const auto now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
resolved.SetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT, now);
resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_CREATION_TIME_STR, FormatUtcTimestamp(now));
Expand Down
4 changes: 3 additions & 1 deletion sdk_v2/cpp/src/catalog/local_model_catalog.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@

namespace fl {

struct GenAIConfig;

/// Mutable, persistent catalog for models registered from arbitrary local directories.
class LocalModelCatalog final : public BaseModelCatalog {
public:
Expand All @@ -32,7 +34,7 @@ class LocalModelCatalog final : public BaseModelCatalog {

private:
ModelInfo ResolveMetadata(const ModelInfo& metadata, const std::string& model_id,
const std::string& name, int version) const;
const std::string& name, int version, const GenAIConfig& genai_config) const;
std::vector<Registration> LoadRegistrations() const;
void SaveRegistrations(const std::vector<Registration>& registrations) const;
Model CreateModel(const Registration& registration) const;
Expand Down
6 changes: 4 additions & 2 deletions sdk_v2/cpp/src/inferencing/execution_provider.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ struct EPUtils {
return ExecutionProvider::kCUDA;
} else if (ep == "webgpu" ||
ep == "WebGPU" ||
ep == "WebGPUExecutionProvider") {
ep == "WebGPUExecutionProvider" ||
ep == "WebGpuExecutionProvider") {
return ExecutionProvider::kWebGPU;
} else if (ep == "openvino" ||
ep == "OpenVINO" ||
Expand All @@ -50,7 +51,8 @@ struct EPUtils {
ep == "VitisAI" ||
ep == "VitisAIExecutionProvider") {
return ExecutionProvider::kVitisAI;
} else if (ep == "RyzenAI") {
} else if (ep == "RyzenAI" ||
ep == "RyzenAIExecutionProvider") {
return ExecutionProvider::kRyzenAI;
} else if (ep == "qnn" ||
ep == "QNN" ||
Expand Down
12 changes: 11 additions & 1 deletion sdk_v2/cpp/test/internal_api/c_api_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,8 @@ TEST(CApiTest, LocalCatalogRegistersListsAndUnregistersWithoutOwningAssets) {
const auto app_data_path = root.path() / "appdata";
const auto model_cache_path = root.path() / "cache" / "models";
std::filesystem::create_directories(model_path);
std::ofstream(model_path / "genai_config.json") << R"({"model":{"type":"phi3"}})";
std::ofstream(model_path / "genai_config.json")
<< R"({"model":{"type":"phi3","context_length":4096}})";

const flApi* api = GetApi();
const flConfigurationApi* config_api = api->GetConfigurationApi();
Expand Down Expand Up @@ -376,6 +377,7 @@ TEST(CApiTest, LocalCatalogRegistersListsAndUnregistersWithoutOwningAssets) {
ASSERT_FL_OK(api, model_api->Info_SetStringProperty(metadata, FOUNDRY_LOCAL_MODEL_PROP_TASK_STR,
"chat-completion"));
ASSERT_FL_OK(api, model_api->Info_SetIntProperty(metadata, FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT, 17));
ASSERT_FL_OK(api, model_api->Info_SetIntProperty(metadata, FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT, 17));

flModel* registered = nullptr;
ASSERT_FL_OK(api, catalog_api->RegisterModel(catalog, model_path.string().c_str(), "c-api-model:3", metadata,
Expand All @@ -390,6 +392,14 @@ TEST(CApiTest, LocalCatalogRegistersListsAndUnregistersWithoutOwningAssets) {
EXPECT_EQ(model_api->Info_GetVersion(registered_info), 3);
EXPECT_STREQ(model_api->Info_GetTask(registered_info), "chat-completion");
EXPECT_EQ(model_api->Info_GetIntProperty(registered_info, FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT, -1), 17);
EXPECT_EQ(model_api->Info_GetIntProperty(registered_info, FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT, -1), 4096);
EXPECT_STREQ(model_api->Info_GetStringProperty(registered_info, FOUNDRY_LOCAL_MODEL_PROP_DISPLAY_NAME_STR),
"c-api-model");
EXPECT_STREQ(model_api->Info_GetStringProperty(registered_info, FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR), "local");
EXPECT_STREQ(model_api->Info_GetStringProperty(registered_info, FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR),
"text");
EXPECT_STREQ(model_api->Info_GetStringProperty(registered_info, FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR),
"text");

ModelListGuard models_guard(nullptr, [api](flModelList* value) { api->ModelList_Release(value); });
flModelList* models = nullptr;
Expand Down
17 changes: 17 additions & 0 deletions sdk_v2/cpp/test/internal_api/execution_provider_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,20 @@ TEST(ExecutionProviderTest, RoundTripPreservesSupportedNonDefaultProviders) {
EXPECT_EQ(EPUtils::StringtoEP(EPUtils::EPtoGenAI(provider)), provider);
}
}

TEST(ExecutionProviderTest, RegistrationNameRoundTripPreservesSupportedProviders) {
const auto providers = {
ExecutionProvider::kCPU,
ExecutionProvider::kCUDA,
ExecutionProvider::kWebGPU,
ExecutionProvider::kOpenVINO,
ExecutionProvider::kTensorRT_RTX,
ExecutionProvider::kVitisAI,
ExecutionProvider::kRyzenAI,
ExecutionProvider::kQNN,
};

for (auto provider : providers) {
EXPECT_EQ(EPUtils::StringtoEP(EPUtils::EPtoRegistrationName(provider)), provider);
}
}
Loading