From 5c89c17bc0020de8354a32b04dde5dbd8c4f8d9b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:22:40 +0000 Subject: [PATCH 1/6] Initial plan From e07a6798a9ddc78c196fd86db49f496767a6b5a1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:35:57 +0000 Subject: [PATCH 2/6] Implement config monitoring for Thor in containerized mode Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com> --- thorlcr/master/thmastermain.cpp | 80 ++++++++++++++++++++++++++++++++- thorlcr/slave/thslavemain.cpp | 48 +++++++++++++++++++- 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index 98a2640d90a..05a0308b7b8 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -131,6 +131,13 @@ class CThorEndHandler : implements IThreaded static CThorEndHandler *thorEndHandler = nullptr; static StringBuffer cloudJobName; +#ifdef _CONTAINERIZED +// Additional settings that the manager adds to the configuration before sending to workers. +// These need to be re-merged when the configuration is refreshed. +static Owned managerAdditionalSettings; +static CriticalSection managerAdditionalSettingsCrit; +#endif + MODULE_INIT(INIT_PRIORITY_STANDARD) { /* NB: CThorEndHandler starts the thread now, although strictly it is not needed until later. @@ -397,11 +404,59 @@ class CRegistryServer : public CSimpleInterface //Check that nothing has caused the global configuration to be refreshed - otherwise inconsistent values may be used by the slave assertex(globals == getComponentConfigSP()); +#ifdef _CONTAINERIZED + // Create additional settings tree to send to workers + // These settings will be re-merged when config is refreshed + { + CriticalBlock b(managerAdditionalSettingsCrit); + managerAdditionalSettings.setown(createPTree("ThorManagerAdditionalSettings")); + + // Copy properties that the manager added to globals and need to be merged on workers + if (globals->hasProp("@masterBuildTag")) + managerAdditionalSettings->setProp("@masterBuildTag", globals->queryProp("@masterBuildTag")); + if (globals->hasProp("@channelsPerWorker")) + managerAdditionalSettings->setPropInt("@channelsPerWorker", globals->getPropInt("@channelsPerWorker")); + if (globals->hasProp("@name")) + managerAdditionalSettings->setProp("@name", globals->queryProp("@name")); + if (globals->hasProp("@nodeGroup")) + managerAdditionalSettings->setProp("@nodeGroup", globals->queryProp("@nodeGroup")); + if (globals->hasProp("@masterTotalMem")) + managerAdditionalSettings->setPropInt("@masterTotalMem", globals->getPropInt("@masterTotalMem")); + if (globals->hasProp("@thorPath")) + managerAdditionalSettings->setProp("@thorPath", globals->queryProp("@thorPath")); + if (globals->hasProp("@query_so_dir")) + managerAdditionalSettings->setProp("@query_so_dir", globals->queryProp("@query_so_dir")); + if (globals->hasProp("@dllsToSlaves")) + managerAdditionalSettings->setPropBool("@dllsToSlaves", globals->getPropBool("@dllsToSlaves")); + if (globals->hasProp("@thorTempDirectory")) + managerAdditionalSettings->setProp("@thorTempDirectory", globals->queryProp("@thorTempDirectory")); + + // Copy memory settings subtrees + IPropertyTree *workerMemory = globals->queryPropTree("workerMemory"); + if (workerMemory) + managerAdditionalSettings->setPropTree("workerMemory", createPTreeFromIPT(workerMemory)); + + IPropertyTree *managerMemory = globals->queryPropTree("managerMemory"); + if (managerMemory) + managerAdditionalSettings->setPropTree("managerMemory", createPTreeFromIPT(managerMemory)); + } +#endif + PROGLOG("Workers connected, initializing.."); msg.clear(); msg.append(THOR_VERSION_MAJOR).append(THOR_VERSION_MINOR); processGroup->serialize(msg); +#ifdef _CONTAINERIZED + // In containerized mode, workers already have the base config loaded. + // Only send the additional manager settings that need to be merged. + { + CriticalBlock b(managerAdditionalSettingsCrit); + managerAdditionalSettings->serialize(msg); + } +#else + // In bare-metal mode, send the full merged globals as before globals->serialize(msg); +#endif getGlobalConfigSP()->serialize(msg); msg.append(managerWorkerMpTag); msg.append(kjServiceMpTag); @@ -645,10 +700,33 @@ int main( int argc, const char *argv[] ) InitModuleObjects(); NoQuickEditSection xxx; { - bool monitorConfig = false; // Do not allow updates to the config file, otherwise the slave may not be in sync. +#ifdef _CONTAINERIZED + // In containerized mode, enable config monitoring + // Additional manager settings will be sent separately to workers and re-merged on config refresh + bool monitorConfig = true; +#else + // Do not allow updates to the config file in bare-metal, otherwise the slave may not be in sync. //MORE: What about updates to storage planes - they will not be passed through to the slaves + bool monitorConfig = false; +#endif globals.setown(loadConfiguration(thorDefaultConfigYaml, argv, "thor", "THOR", "thor.xml", nullptr, nullptr, monitorConfig)); } + +#ifdef _CONTAINERIZED + // Install config update hook to re-merge manager additional settings when config is refreshed + static CConfigUpdateHook managerConfigHook; + managerConfigHook.installOnce([](const IPropertyTree *oldComponentConfiguration, const IPropertyTree *oldGlobalConfiguration) + { + // Re-merge additional manager settings into refreshed config + CriticalBlock b(managerAdditionalSettingsCrit); + if (managerAdditionalSettings) + { + Owned currentConfig = getComponentConfigSP(); + mergeConfiguration(*currentConfig, *managerAdditionalSettings); + } + }, false); // false = don't call when installed, only on updates +#endif + updateTraceFlags(loadTraceFlags(globals, thorTraceOptions, queryTraceFlags()), true); #ifdef _DEBUG unsigned holdWorker = globals->getPropInt("@holdSlave", NotFound); diff --git a/thorlcr/slave/thslavemain.cpp b/thorlcr/slave/thslavemain.cpp index c6b97a1f54c..dc64b2a4480 100644 --- a/thorlcr/slave/thslavemain.cpp +++ b/thorlcr/slave/thslavemain.cpp @@ -139,7 +139,13 @@ static bool RegisterSelf(SocketEndpoint &masterEp) msg.read(vmajor); msg.read(vminor); Owned processGroup = deserializeIGroup(msg); +#ifdef _CONTAINERIZED + // In containerized mode, receive only the additional manager settings + Owned managerAdditionalSettings = createPTree(msg); +#else + // In bare-metal mode, receive the full merged component config from manager Owned masterComponentConfig = createPTree(msg); +#endif Owned masterGlobalConfig = createPTree(msg); mySlaveNum = (unsigned)processGroup->rank(queryMyNode()); assertex(NotFound != mySlaveNum); @@ -151,6 +157,44 @@ static bool RegisterSelf(SocketEndpoint &masterEp) else assertex(mySlaveNum == configSlaveNum); +#ifdef _CONTAINERIZED + // In containerized mode, merge the additional manager settings into our existing config + Owned mergedComponentConfig = createPTreeFromIPT(globals); + mergeConfiguration(*mergedComponentConfig, *managerAdditionalSettings); + + // Store additional settings for re-merging on config refresh + static Owned workerStoredManagerSettings; + static CriticalSection workerManagerSettingsCrit; + { + CriticalBlock b(workerManagerSettingsCrit); + workerStoredManagerSettings.setown(createPTreeFromIPT(managerAdditionalSettings)); + } + + // Install config update hook to re-merge manager settings when config is refreshed + static CConfigUpdateHook workerConfigHook; + workerConfigHook.installOnce([](const IPropertyTree *oldComponentConfiguration, const IPropertyTree *oldGlobalConfiguration) + { + // Re-merge additional manager settings into refreshed config + CriticalBlock b(workerManagerSettingsCrit); + if (workerStoredManagerSettings) + { + Owned currentConfig = getComponentConfigSP(); + mergeConfiguration(*currentConfig, *workerStoredManagerSettings); + } + }, false); // false = don't call when installed, only on updates + + // Handle logging detail level from manager settings + if (managerAdditionalSettings->hasProp("logging/@thorworkerdetail")) + { + unsigned workerDetailLevel = managerAdditionalSettings->getPropInt("logging/@thorworkerdetail"); + mergedComponentConfig->setPropInt("logging/@detail", workerDetailLevel); + ILogMsgFilter *existingLogFilter = queryLogMsgManager()->queryMonitorFilter(logHandler); + dbgassertex(existingLogFilter); + if (existingLogFilter->queryMaxDetail() != workerDetailLevel) + verifyex(queryLogMsgManager()->changeMonitorFilterOwn(logHandler, getCategoryLogMsgFilter(existingLogFilter->queryAudienceMask(), existingLogFilter->queryClassMask(), workerDetailLevel))); + } +#else + // In bare-metal mode, merge the full master config as before Owned mergedComponentConfig = createPTreeFromIPT(globals); mergeConfiguration(*mergedComponentConfig, *masterComponentConfig); if (masterComponentConfig->hasProp("logging/@thorworkerdetail")) @@ -162,6 +206,7 @@ static bool RegisterSelf(SocketEndpoint &masterEp) if (existingLogFilter->queryMaxDetail() != workerDetailLevel) verifyex(queryLogMsgManager()->changeMonitorFilterOwn(logHandler, getCategoryLogMsgFilter(existingLogFilter->queryAudienceMask(), existingLogFilter->queryClassMask(), workerDetailLevel))); } +#endif replaceComponentConfig(mergedComponentConfig, masterGlobalConfig); globals.set(mergedComponentConfig); #ifdef _DEBUG @@ -415,7 +460,8 @@ int main( int argc, const char *argv[] ) } cmdArgs = argv+1; #ifdef _CONTAINERIZED - globals.setown(loadConfiguration(thorDefaultConfigYaml, argv, "thor", "THOR", nullptr, nullptr, nullptr, false)); + // In containerized mode, enable config monitoring for workers too + globals.setown(loadConfiguration(thorDefaultConfigYaml, argv, "thor", "THOR", nullptr, nullptr, nullptr, true)); // pickup the default logging level from the thor default config yaml if (globals->hasProp("logging/@thorworkerdetail")) { From 5199c5ddd99547cb599da8c04a14d7f3acf34d28 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:38:12 +0000 Subject: [PATCH 3/6] Fix logging detail level handling in containerized mode Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com> --- thorlcr/master/thmastermain.cpp | 4 ++++ thorlcr/slave/thslavemain.cpp | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index 05a0308b7b8..744a4bda4a8 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -431,6 +431,10 @@ class CRegistryServer : public CSimpleInterface if (globals->hasProp("@thorTempDirectory")) managerAdditionalSettings->setProp("@thorTempDirectory", globals->queryProp("@thorTempDirectory")); + // Copy logging settings if present (for worker detail level override) + if (globals->hasProp("logging/@thorworkerdetail")) + managerAdditionalSettings->setPropInt("logging/@thorworkerdetail", globals->getPropInt("logging/@thorworkerdetail")); + // Copy memory settings subtrees IPropertyTree *workerMemory = globals->queryPropTree("workerMemory"); if (workerMemory) diff --git a/thorlcr/slave/thslavemain.cpp b/thorlcr/slave/thslavemain.cpp index dc64b2a4480..4f939ea4fdb 100644 --- a/thorlcr/slave/thslavemain.cpp +++ b/thorlcr/slave/thslavemain.cpp @@ -183,10 +183,12 @@ static bool RegisterSelf(SocketEndpoint &masterEp) } }, false); // false = don't call when installed, only on updates - // Handle logging detail level from manager settings - if (managerAdditionalSettings->hasProp("logging/@thorworkerdetail")) + // Handle logging detail level override if present + // Note: In containerized mode, both manager and worker load the same base config, + // but the manager may have added or modified logging/@thorworkerdetail + if (mergedComponentConfig->hasProp("logging/@thorworkerdetail")) { - unsigned workerDetailLevel = managerAdditionalSettings->getPropInt("logging/@thorworkerdetail"); + unsigned workerDetailLevel = mergedComponentConfig->getPropInt("logging/@thorworkerdetail"); mergedComponentConfig->setPropInt("logging/@detail", workerDetailLevel); ILogMsgFilter *existingLogFilter = queryLogMsgManager()->queryMonitorFilter(logHandler); dbgassertex(existingLogFilter); From eee722c31ab733bc6d64ede098ad9fe88d80b4c8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:41:19 +0000 Subject: [PATCH 4/6] Use ConfigModifyFunc for proper config refresh handling Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com> --- thorlcr/master/thmastermain.cpp | 9 ++++----- thorlcr/slave/thslavemain.cpp | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index 744a4bda4a8..208d6ebf877 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -719,16 +719,15 @@ int main( int argc, const char *argv[] ) #ifdef _CONTAINERIZED // Install config update hook to re-merge manager additional settings when config is refreshed static CConfigUpdateHook managerConfigHook; - managerConfigHook.installOnce([](const IPropertyTree *oldComponentConfiguration, const IPropertyTree *oldGlobalConfiguration) + managerConfigHook.installModifierOnce([](IPropertyTree *newComponentConfiguration, IPropertyTree *newGlobalConfiguration) { - // Re-merge additional manager settings into refreshed config + // Re-merge additional manager settings into refreshed config (before it becomes active) CriticalBlock b(managerAdditionalSettingsCrit); if (managerAdditionalSettings) { - Owned currentConfig = getComponentConfigSP(); - mergeConfiguration(*currentConfig, *managerAdditionalSettings); + mergeConfiguration(*newComponentConfiguration, *managerAdditionalSettings); } - }, false); // false = don't call when installed, only on updates + }, true); // true = thread safe (we're in main thread during init) #endif updateTraceFlags(loadTraceFlags(globals, thorTraceOptions, queryTraceFlags()), true); diff --git a/thorlcr/slave/thslavemain.cpp b/thorlcr/slave/thslavemain.cpp index 4f939ea4fdb..931872fb90d 100644 --- a/thorlcr/slave/thslavemain.cpp +++ b/thorlcr/slave/thslavemain.cpp @@ -172,16 +172,15 @@ static bool RegisterSelf(SocketEndpoint &masterEp) // Install config update hook to re-merge manager settings when config is refreshed static CConfigUpdateHook workerConfigHook; - workerConfigHook.installOnce([](const IPropertyTree *oldComponentConfiguration, const IPropertyTree *oldGlobalConfiguration) + workerConfigHook.installModifierOnce([](IPropertyTree *newComponentConfiguration, IPropertyTree *newGlobalConfiguration) { - // Re-merge additional manager settings into refreshed config + // Re-merge additional manager settings into refreshed config (before it becomes active) CriticalBlock b(workerManagerSettingsCrit); if (workerStoredManagerSettings) { - Owned currentConfig = getComponentConfigSP(); - mergeConfiguration(*currentConfig, *workerStoredManagerSettings); + mergeConfiguration(*newComponentConfiguration, *workerStoredManagerSettings); } - }, false); // false = don't call when installed, only on updates + }, true); // true = thread safe (we're in RegisterSelf, single-threaded at this point) // Handle logging detail level override if present // Note: In containerized mode, both manager and worker load the same base config, From 2b93986e2fa1db086e7eca3dbdc14c8d56a835a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:46:21 +0000 Subject: [PATCH 5/6] Add documentation for Thor config monitoring implementation Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com> --- thorlcr/THOR_CONFIG_MONITORING.md | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 thorlcr/THOR_CONFIG_MONITORING.md diff --git a/thorlcr/THOR_CONFIG_MONITORING.md b/thorlcr/THOR_CONFIG_MONITORING.md new file mode 100644 index 00000000000..7017c0d12b2 --- /dev/null +++ b/thorlcr/THOR_CONFIG_MONITORING.md @@ -0,0 +1,59 @@ +# Thor Config Monitoring Implementation + +## Overview +This implementation enables configuration monitoring for Thor (manager and workers) in containerized deployments, allowing "soft" configuration changes (e.g., logging levels) to be applied without restarting Thor. + +## Problem Solved +Previously, Thor components loaded configuration without monitoring (monitor=false), meaning any configuration changes required a restart. While other components could auto-reload config, Thor couldn't because: +- The manager sends additional settings to workers during registration +- If workers auto-reloaded config, they would lose these manager-provided settings + +## Solution +The solution reorganizes how Thor handles configuration in containerized mode: + +1. **Separate additional settings**: Manager-specific settings are extracted into a dedicated IPropertyTree +2. **Send settings separately**: Only additional settings are sent to workers (not the full merged config) +3. **Re-merge on refresh**: Both manager and workers install hooks to re-merge additional settings when config is refreshed +4. **Enable monitoring**: Config monitoring is enabled for both manager and workers in containerized mode + +## Technical Details + +### Containerized Mode Changes + +#### Manager (thmastermain.cpp) +- Enables config monitoring: `loadConfiguration(..., monitor=true)` +- Creates `managerAdditionalSettings` tree with manager-specific settings: + - `@masterBuildTag`, `@channelsPerWorker`, `@name`, `@nodeGroup` + - `@masterTotalMem`, `@thorPath`, `@query_so_dir`, `@dllsToSlaves`, `@thorTempDirectory` + - `logging/@thorworkerdetail` + - `workerMemory/*`, `managerMemory/*` +- Sends only `managerAdditionalSettings` to workers (not full globals) +- Installs `ConfigModifyFunc` hook to re-merge settings on config refresh + +#### Worker (thslavemain.cpp) +- Enables config monitoring: `loadConfiguration(..., monitor=true)` +- Receives `managerAdditionalSettings` from manager +- Merges additional settings into its own config +- Stores additional settings for re-use +- Installs `ConfigModifyFunc` hook to re-merge settings on config refresh + +### Bare-Metal Mode (Unchanged) +- Config monitoring disabled: `loadConfiguration(..., monitor=false)` +- Manager sends full merged globals to workers +- Workers use master config as before +- Complete backward compatibility + +## Files Modified +- `thorlcr/master/thmastermain.cpp` (+83 lines) +- `thorlcr/slave/thslavemain.cpp` (+49 lines) + +## Benefits +1. **No restart required**: Soft config changes applied automatically +2. **Preserved settings**: Manager-provided settings maintained across refreshes +3. **Backward compatible**: Bare-metal mode unchanged +4. **Clean design**: Clear separation of base config and additional settings + +## Related Code +- Config update hooks: `system/jlib/jptree.cpp` (ConfigModifyFunc, ConfigUpdateFunc) +- Config loading: `system/jlib/jptree.cpp` (loadConfiguration) +- Config merging: `system/jlib/jptree.cpp` (mergeConfiguration) From 5811547011653b5f9183c8483d74fcdd777318e9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 17:21:45 +0000 Subject: [PATCH 6/6] Address code review feedback: optimize settings, use isContainerized(), move statics Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com> --- thorlcr/master/thmastermain.cpp | 88 ++++++++++---------------- thorlcr/slave/thslavemain.cpp | 109 ++++++++++++++++---------------- 2 files changed, 87 insertions(+), 110 deletions(-) diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index 208d6ebf877..9d24b246af7 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -131,12 +131,11 @@ class CThorEndHandler : implements IThreaded static CThorEndHandler *thorEndHandler = nullptr; static StringBuffer cloudJobName; -#ifdef _CONTAINERIZED // Additional settings that the manager adds to the configuration before sending to workers. -// These need to be re-merged when the configuration is refreshed. +// These need to be re-merged when the configuration is refreshed (containerized mode only). static Owned managerAdditionalSettings; static CriticalSection managerAdditionalSettingsCrit; -#endif +static CConfigUpdateHook managerConfigHook; MODULE_INIT(INIT_PRIORITY_STANDARD) { @@ -404,26 +403,19 @@ class CRegistryServer : public CSimpleInterface //Check that nothing has caused the global configuration to be refreshed - otherwise inconsistent values may be used by the slave assertex(globals == getComponentConfigSP()); -#ifdef _CONTAINERIZED - // Create additional settings tree to send to workers - // These settings will be re-merged when config is refreshed + if (isContainerized()) { + // Create additional settings tree to send to workers + // These settings will be re-merged when config is refreshed + // Only include settings that are actually SET by the manager (not those from config) CriticalBlock b(managerAdditionalSettingsCrit); managerAdditionalSettings.setown(createPTree("ThorManagerAdditionalSettings")); - // Copy properties that the manager added to globals and need to be merged on workers - if (globals->hasProp("@masterBuildTag")) - managerAdditionalSettings->setProp("@masterBuildTag", globals->queryProp("@masterBuildTag")); - if (globals->hasProp("@channelsPerWorker")) - managerAdditionalSettings->setPropInt("@channelsPerWorker", globals->getPropInt("@channelsPerWorker")); - if (globals->hasProp("@name")) - managerAdditionalSettings->setProp("@name", globals->queryProp("@name")); - if (globals->hasProp("@nodeGroup")) - managerAdditionalSettings->setProp("@nodeGroup", globals->queryProp("@nodeGroup")); - if (globals->hasProp("@masterTotalMem")) - managerAdditionalSettings->setPropInt("@masterTotalMem", globals->getPropInt("@masterTotalMem")); - if (globals->hasProp("@thorPath")) - managerAdditionalSettings->setProp("@thorPath", globals->queryProp("@thorPath")); + // Properties that the manager dynamically sets and workers need + managerAdditionalSettings->setProp("@masterBuildTag", globals->queryProp("@masterBuildTag")); + managerAdditionalSettings->setPropInt("@masterTotalMem", globals->getPropInt("@masterTotalMem")); + managerAdditionalSettings->setProp("@thorPath", globals->queryProp("@thorPath")); + if (globals->hasProp("@query_so_dir")) managerAdditionalSettings->setProp("@query_so_dir", globals->queryProp("@query_so_dir")); if (globals->hasProp("@dllsToSlaves")) @@ -431,36 +423,28 @@ class CRegistryServer : public CSimpleInterface if (globals->hasProp("@thorTempDirectory")) managerAdditionalSettings->setProp("@thorTempDirectory", globals->queryProp("@thorTempDirectory")); - // Copy logging settings if present (for worker detail level override) - if (globals->hasProp("logging/@thorworkerdetail")) - managerAdditionalSettings->setPropInt("logging/@thorworkerdetail", globals->getPropInt("logging/@thorworkerdetail")); - - // Copy memory settings subtrees + // Copy worker memory settings that manager computed IPropertyTree *workerMemory = globals->queryPropTree("workerMemory"); if (workerMemory) managerAdditionalSettings->setPropTree("workerMemory", createPTreeFromIPT(workerMemory)); - - IPropertyTree *managerMemory = globals->queryPropTree("managerMemory"); - if (managerMemory) - managerAdditionalSettings->setPropTree("managerMemory", createPTreeFromIPT(managerMemory)); } -#endif PROGLOG("Workers connected, initializing.."); msg.clear(); msg.append(THOR_VERSION_MAJOR).append(THOR_VERSION_MINOR); processGroup->serialize(msg); -#ifdef _CONTAINERIZED - // In containerized mode, workers already have the base config loaded. - // Only send the additional manager settings that need to be merged. + if (isContainerized()) { + // In containerized mode, workers already have the base config loaded. + // Only send the additional manager settings that need to be merged. CriticalBlock b(managerAdditionalSettingsCrit); managerAdditionalSettings->serialize(msg); } -#else - // In bare-metal mode, send the full merged globals as before - globals->serialize(msg); -#endif + else + { + // In bare-metal mode, send the full merged globals as before + globals->serialize(msg); + } getGlobalConfigSP()->serialize(msg); msg.append(managerWorkerMpTag); msg.append(kjServiceMpTag); @@ -704,31 +688,23 @@ int main( int argc, const char *argv[] ) InitModuleObjects(); NoQuickEditSection xxx; { -#ifdef _CONTAINERIZED - // In containerized mode, enable config monitoring - // Additional manager settings will be sent separately to workers and re-merged on config refresh - bool monitorConfig = true; -#else - // Do not allow updates to the config file in bare-metal, otherwise the slave may not be in sync. - //MORE: What about updates to storage planes - they will not be passed through to the slaves - bool monitorConfig = false; -#endif + bool monitorConfig = isContainerized(); // Enable monitoring in containerized mode only globals.setown(loadConfiguration(thorDefaultConfigYaml, argv, "thor", "THOR", "thor.xml", nullptr, nullptr, monitorConfig)); } -#ifdef _CONTAINERIZED - // Install config update hook to re-merge manager additional settings when config is refreshed - static CConfigUpdateHook managerConfigHook; - managerConfigHook.installModifierOnce([](IPropertyTree *newComponentConfiguration, IPropertyTree *newGlobalConfiguration) + if (isContainerized()) { - // Re-merge additional manager settings into refreshed config (before it becomes active) - CriticalBlock b(managerAdditionalSettingsCrit); - if (managerAdditionalSettings) + // Install config update hook to re-merge manager additional settings when config is refreshed + managerConfigHook.installModifierOnce([](IPropertyTree *newComponentConfiguration, IPropertyTree *newGlobalConfiguration) { - mergeConfiguration(*newComponentConfiguration, *managerAdditionalSettings); - } - }, true); // true = thread safe (we're in main thread during init) -#endif + // Re-merge additional manager settings into refreshed config (before it becomes active) + CriticalBlock b(managerAdditionalSettingsCrit); + if (managerAdditionalSettings) + { + mergeConfiguration(*newComponentConfiguration, *managerAdditionalSettings); + } + }, true); // true = thread safe (we're in main thread during init) + } updateTraceFlags(loadTraceFlags(globals, thorTraceOptions, queryTraceFlags()), true); #ifdef _DEBUG diff --git a/thorlcr/slave/thslavemain.cpp b/thorlcr/slave/thslavemain.cpp index 931872fb90d..b76424809ce 100644 --- a/thorlcr/slave/thslavemain.cpp +++ b/thorlcr/slave/thslavemain.cpp @@ -79,6 +79,11 @@ static const unsigned defaultForceNumStrands = 0; static const char **cmdArgs; static ILogMsgHandler *logHandler = nullptr; +// Additional settings from manager that need to be re-merged on config refresh (containerized mode only) +static Owned workerStoredManagerSettings; +static CriticalSection workerManagerSettingsCrit; +static CConfigUpdateHook workerConfigHook; + static void replyError(unsigned errorCode, const char *errorMsg) { SocketEndpoint myEp = queryMyNode()->endpoint(); @@ -139,13 +144,17 @@ static bool RegisterSelf(SocketEndpoint &masterEp) msg.read(vmajor); msg.read(vminor); Owned processGroup = deserializeIGroup(msg); -#ifdef _CONTAINERIZED - // In containerized mode, receive only the additional manager settings - Owned managerAdditionalSettings = createPTree(msg); -#else - // In bare-metal mode, receive the full merged component config from manager - Owned masterComponentConfig = createPTree(msg); -#endif + Owned masterComponentConfig; + if (isContainerized()) + { + // In containerized mode, receive only the additional manager settings + masterComponentConfig.setown(createPTree(msg)); + } + else + { + // In bare-metal mode, receive the full merged component config from manager + masterComponentConfig.setown(createPTree(msg)); + } Owned masterGlobalConfig = createPTree(msg); mySlaveNum = (unsigned)processGroup->rank(queryMyNode()); assertex(NotFound != mySlaveNum); @@ -157,34 +166,36 @@ static bool RegisterSelf(SocketEndpoint &masterEp) else assertex(mySlaveNum == configSlaveNum); -#ifdef _CONTAINERIZED - // In containerized mode, merge the additional manager settings into our existing config Owned mergedComponentConfig = createPTreeFromIPT(globals); - mergeConfiguration(*mergedComponentConfig, *managerAdditionalSettings); - - // Store additional settings for re-merging on config refresh - static Owned workerStoredManagerSettings; - static CriticalSection workerManagerSettingsCrit; - { - CriticalBlock b(workerManagerSettingsCrit); - workerStoredManagerSettings.setown(createPTreeFromIPT(managerAdditionalSettings)); - } - - // Install config update hook to re-merge manager settings when config is refreshed - static CConfigUpdateHook workerConfigHook; - workerConfigHook.installModifierOnce([](IPropertyTree *newComponentConfiguration, IPropertyTree *newGlobalConfiguration) + if (isContainerized()) { - // Re-merge additional manager settings into refreshed config (before it becomes active) - CriticalBlock b(workerManagerSettingsCrit); - if (workerStoredManagerSettings) + // In containerized mode, merge the additional manager settings into our existing config + mergeConfiguration(*mergedComponentConfig, *masterComponentConfig); + + // Store additional settings for re-merging on config refresh { - mergeConfiguration(*newComponentConfiguration, *workerStoredManagerSettings); + CriticalBlock b(workerManagerSettingsCrit); + workerStoredManagerSettings.setown(createPTreeFromIPT(masterComponentConfig)); } - }, true); // true = thread safe (we're in RegisterSelf, single-threaded at this point) + + // Install config update hook to re-merge manager settings when config is refreshed + workerConfigHook.installModifierOnce([](IPropertyTree *newComponentConfiguration, IPropertyTree *newGlobalConfiguration) + { + // Re-merge additional manager settings into refreshed config (before it becomes active) + CriticalBlock b(workerManagerSettingsCrit); + if (workerStoredManagerSettings) + { + mergeConfiguration(*newComponentConfiguration, *workerStoredManagerSettings); + } + }, true); // true = thread safe (we're in RegisterSelf, single-threaded at this point) + } + else + { + // In bare-metal mode, merge the full master config as before + mergeConfiguration(*mergedComponentConfig, *masterComponentConfig); + } // Handle logging detail level override if present - // Note: In containerized mode, both manager and worker load the same base config, - // but the manager may have added or modified logging/@thorworkerdetail if (mergedComponentConfig->hasProp("logging/@thorworkerdetail")) { unsigned workerDetailLevel = mergedComponentConfig->getPropInt("logging/@thorworkerdetail"); @@ -194,20 +205,6 @@ static bool RegisterSelf(SocketEndpoint &masterEp) if (existingLogFilter->queryMaxDetail() != workerDetailLevel) verifyex(queryLogMsgManager()->changeMonitorFilterOwn(logHandler, getCategoryLogMsgFilter(existingLogFilter->queryAudienceMask(), existingLogFilter->queryClassMask(), workerDetailLevel))); } -#else - // In bare-metal mode, merge the full master config as before - Owned mergedComponentConfig = createPTreeFromIPT(globals); - mergeConfiguration(*mergedComponentConfig, *masterComponentConfig); - if (masterComponentConfig->hasProp("logging/@thorworkerdetail")) - { - unsigned workerDetailLevel = masterComponentConfig->getPropInt("logging/@thorworkerdetail"); - mergedComponentConfig->setPropInt("logging/@detail", workerDetailLevel); - ILogMsgFilter *existingLogFilter = queryLogMsgManager()->queryMonitorFilter(logHandler); - dbgassertex(existingLogFilter); - if (existingLogFilter->queryMaxDetail() != workerDetailLevel) - verifyex(queryLogMsgManager()->changeMonitorFilterOwn(logHandler, getCategoryLogMsgFilter(existingLogFilter->queryAudienceMask(), existingLogFilter->queryClassMask(), workerDetailLevel))); - } -#endif replaceComponentConfig(mergedComponentConfig, masterGlobalConfig); globals.set(mergedComponentConfig); #ifdef _DEBUG @@ -460,19 +457,23 @@ int main( int argc, const char *argv[] ) return 1; } cmdArgs = argv+1; -#ifdef _CONTAINERIZED - // In containerized mode, enable config monitoring for workers too - globals.setown(loadConfiguration(thorDefaultConfigYaml, argv, "thor", "THOR", nullptr, nullptr, nullptr, true)); - // pickup the default logging level from the thor default config yaml - if (globals->hasProp("logging/@thorworkerdetail")) + if (isContainerized()) { - unsigned workerDetailLevel = globals->getPropInt("logging/@thorworkerdetail"); - globals->setPropInt("logging/@detail", workerDetailLevel); - // NB: may be overridden by Thor config settings during RegisterSelf + // In containerized mode, enable config monitoring for workers + globals.setown(loadConfiguration(thorDefaultConfigYaml, argv, "thor", "THOR", nullptr, nullptr, nullptr, true)); + // pickup the default logging level from the thor default config yaml + if (globals->hasProp("logging/@thorworkerdetail")) + { + unsigned workerDetailLevel = globals->getPropInt("logging/@thorworkerdetail"); + globals->setPropInt("logging/@detail", workerDetailLevel); + // NB: may be overridden by Thor config settings during RegisterSelf + } + } + else + { + // In bare-metal mode, no monitoring + globals.setown(loadConfiguration(globals, nullptr, argv, "thor", "THOR", nullptr, nullptr, nullptr, false)); } -#else - globals.setown(loadConfiguration(globals, nullptr, argv, "thor", "THOR", nullptr, nullptr, nullptr, false)); -#endif // NB: the thor configuration is serialized from the manager and only available after RegisterSelf // Until that point, only properties on the command line are available.