Add dynamic shapes for inputs and outputs

CHANGELOG.md

@@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Features
 
+- Added dynamic shapes for inputs and outputs.  Disciplines can now
+  declare variables with `dynamic_shape=True` in `add_input` /
+  `add_output`, indicating that the client is allowed to set the
+  variable's shape at runtime.  A new `SetVariableShapes` gRPC RPC
+  lets clients send resolved shapes after querying variable definitions.
+  The OpenMDAO bindings automatically map dynamic-shape variables to
+  `shape_by_conn=True` and send resolved shapes back to the server
+  (MDO-Standards/Philote-MDO#6).
 - Added support for struct (dict) options via the new `kStruct` DataType enum
   value, enabling complex nested data to be declared and passed as discipline
   options (#49).

philote_mdo/examples/__init__.py

@@ -27,6 +27,7 @@
 # the linked websites, of the information, products, or services contained
 # therein. The DoD does not exercise any editorial, security, or other
 # control over the information you may find at these locations.
+from .flexible import FlexibleDiscipline
 from .paraboloid import Paraboloid
 from .quadratic import QuadradicImplicit
 from .rosenbrock import Rosenbrock

philote_mdo/examples/flexible.py

@@ -0,0 +1,55 @@
+# Philote-Python
+#
+# Copyright 2022-2025 Christopher A. Lupp
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# This work has been cleared for public release, distribution unlimited, case
+# number: AFRL-2023-5713.
+#
+# The views expressed are those of the authors and do not reflect the
+# official guidance or position of the United States Government, the
+# Department of Defense or of the United States Air Force.
+#
+# Statement from DoD: The Appearance of external hyperlinks does not
+# constitute endorsement by the United States Department of Defense (DoD) of
+# the linked websites, of the information, products, or services contained
+# therein. The DoD does not exercise any editorial, security, or other
+# control over the information you may find at these locations.
+import numpy as np
+import philote_mdo.general as pmdo
+
+
+class FlexibleDiscipline(pmdo.ExplicitDiscipline):
+    """
+    Example explicit discipline with dynamic shapes.
+
+    This discipline doubles every element of the input vector.  The input
+    and output shapes are not fixed by the server — the client is
+    expected to set them via ``SetVariableShapes`` before computation.
+    """
+
+    def setup(self):
+        self.add_input("x", dynamic_shape=True, units="m")
+        self.add_output("y", dynamic_shape=True, units="m")
+
+    def setup_partials(self):
+        self.declare_partials("y", "x")
+
+    def compute(self, inputs, outputs):
+        outputs["y"] = 2.0 * inputs["x"]
+
+    def compute_partials(self, inputs, partials):
+        n = inputs["x"].size
+        partials["y", "x"] = 2.0 * np.eye(n)

philote_mdo/general/discipline.py

@@ -86,7 +86,7 @@ def add_option(self, name, type):
             )
         self.options_list[name] = type
 
-    def add_input(self, name, shape=(1,), units=""):
+    def add_input(self, name, shape=(1,), units="", dynamic_shape=False):
         """
         Define a continuous input.
 
@@ -95,12 +95,16 @@ def add_input(self, name, shape=(1,), units=""):
         name : string
             the name of the input variable
         shape : tuple
-            the shape of the input variable
+            the shape of the input variable (ignored when dynamic_shape
+            is True)
         units : string
             the unit definition for the input variable
+        dynamic_shape : bool
+            when True, the client is allowed to set this variable's shape
         """
         validate_name(name, "add_input")
-        validate_shape(shape, "add_input")
+        if not dynamic_shape:
+            validate_shape(shape, "add_input")
         validate_units(units, "add_input")
         if any(v.name == name and v.type == data.VariableType.kInput for v in self._var_meta):
             raise PhiloteValidationError(
@@ -109,8 +113,10 @@ def add_input(self, name, shape=(1,), units=""):
         meta = data.VariableMetaData()
         meta.type = data.VariableType.kInput
         meta.name = name
-        meta.shape.extend(shape)
+        if not dynamic_shape:
+            meta.shape.extend(shape)
         meta.units = units
+        meta.dynamic_shape = dynamic_shape
         self._var_meta += [meta]
 
     def add_discrete_input(self, name, default=None):
@@ -167,7 +173,7 @@ def add_discrete_output(self, name, default=None):
         meta.name = name
         self._discrete_var_meta += [meta]
 
-    def add_output(self, name, shape=(1,), units=""):
+    def add_output(self, name, shape=(1,), units="", dynamic_shape=False):
         """
         Defines a continuous output.
 
@@ -176,12 +182,16 @@ def add_output(self, name, shape=(1,), units=""):
         name : string
             the name of the output variable
         shape : tuple
-            the shape of the output variable
+            the shape of the output variable (ignored when dynamic_shape
+            is True)
         units : string
             the unit definition for the output variable
+        dynamic_shape : bool
+            when True, the client is allowed to set this variable's shape
         """
         validate_name(name, "add_output")
-        validate_shape(shape, "add_output")
+        if not dynamic_shape:
+            validate_shape(shape, "add_output")
         validate_units(units, "add_output")
         if any(v.name == name and v.type == data.VariableType.kOutput for v in self._var_meta):
             raise PhiloteValidationError(
@@ -190,17 +200,21 @@ def add_output(self, name, shape=(1,), units=""):
         out_meta = data.VariableMetaData()
         out_meta.type = data.VariableType.kOutput
         out_meta.name = name
-        out_meta.shape.extend(shape)
+        if not dynamic_shape:
+            out_meta.shape.extend(shape)
         out_meta.units = units
+        out_meta.dynamic_shape = dynamic_shape
         self._var_meta += [out_meta]
 
         if self._is_implicit:
             res_meta = data.VariableMetaData()
             res_meta.type = data.VariableType.kOutput
             res_meta.name = name
-            res_meta.shape.extend(shape)
+            if not dynamic_shape:
+                res_meta.shape.extend(shape)
             res_meta.units = units
             res_meta.type = data.VariableType.kResidual
+            res_meta.dynamic_shape = dynamic_shape
             self._var_meta += [res_meta]
 
     def declare_partials(self, func, var):

philote_mdo/general/discipline_client.py

@@ -154,6 +154,71 @@ def get_partials_definitions(self):
             if message.name not in self._partials_meta:
                 self._partials_meta += [message]
 
+    def get_dynamic_variables(self):
+        """
+        Returns a list of variable metadata entries that have
+        ``dynamic_shape`` set to ``True``.
+        """
+        return [v for v in self._var_meta if v.dynamic_shape]
+
+    def set_variable_shape(self, name, shape, var_type=data.VariableType.kInput):
+        """
+        Creates a ``VariableMetaData`` message for setting a dynamic
+        variable's shape.
+
+        Parameters
+        ----------
+        name : str
+            the name of the variable
+        shape : tuple
+            the desired shape
+        var_type : VariableType
+            the variable type (kInput or kOutput)
+
+        Returns
+        -------
+        VariableMetaData
+            protobuf message ready for ``send_variable_shapes``
+        """
+        meta = data.VariableMetaData()
+        meta.type = var_type
+        meta.name = name
+        meta.shape.extend(shape)
+        return meta
+
+    def send_variable_shapes(self, variable_metadata):
+        """
+        Sends shapes for variables flagged as ``dynamic_shape``.
+
+        Call after ``get_variable_definitions()`` and before compute
+        calls.
+
+        Parameters
+        ----------
+        variable_metadata : list of VariableMetaData
+            shapes for dynamic variables
+        """
+        self._disc_stub.SetVariableShapes(iter(variable_metadata))
+
+        # update local metadata to reflect the new shapes
+        for meta in variable_metadata:
+            for var in self._var_meta:
+                if var.name == meta.name and var.type == meta.type:
+                    var.shape[:] = []
+                    var.shape.extend(meta.shape)
+                    break
+
+            # for implicit outputs, also update the matching residual
+            if meta.type == data.VariableType.kOutput:
+                for var in self._var_meta:
+                    if (
+                        var.name == meta.name
+                        and var.type == data.VariableType.kResidual
+                    ):
+                        var.shape[:] = []
+                        var.shape.extend(meta.shape)
+                        break
+
     def _assemble_input_messages(
         self, inputs, outputs=None, discrete_inputs=None, discrete_outputs=None
     ):

philote_mdo/general/discipline_server.py

@@ -35,7 +35,7 @@
 from google.protobuf.empty_pb2 import Empty
 from google.protobuf import struct_pb2
 from philote_mdo.utils import PairDict, get_flattened_view
-from philote_mdo.utils.validation import PhiloteValidationError
+from philote_mdo.utils.validation import PhiloteValidationError, validate_shape
 
 
 class DisciplineServer(disc.DisciplineService):
@@ -170,6 +170,57 @@ def GetPartialDefinitions(self, request, context):
         for jac in self._discipline._partials_meta:
             yield jac
 
+    def SetVariableShapes(self, request_iterator, context):
+        """
+        Receives client-defined shapes for variables flagged as
+        dynamic_shape.
+
+        The client must call this RPC after GetVariableDefinitions and
+        before any compute RPCs for disciplines that contain variables
+        with dynamic shapes.
+        """
+        try:
+            for meta in request_iterator:
+                validate_shape(tuple(meta.shape), "SetVariableShapes")
+
+                # find the matching variable and update its shape
+                for var in self._discipline._var_meta:
+                    if var.name == meta.name and var.type == meta.type:
+                        if not var.dynamic_shape:
+                            raise PhiloteValidationError(
+                                f"Variable '{meta.name}' does not allow "
+                                f"dynamic shapes."
+                            )
+                        var.shape[:] = []
+                        var.shape.extend(meta.shape)
+                        break
+                else:
+                    raise PhiloteValidationError(
+                        f"SetVariableShapes: variable '{meta.name}' "
+                        f"not found."
+                    )
+
+                # if the variable is an output on an implicit discipline,
+                # also update the matching residual entry
+                if meta.type == data.VariableType.kOutput:
+                    for var in self._discipline._var_meta:
+                        if (
+                            var.name == meta.name
+                            and var.type == data.VariableType.kResidual
+                            and var.dynamic_shape
+                        ):
+                            var.shape[:] = []
+                            var.shape.extend(meta.shape)
+                            break
+
+            return Empty()
+        except PhiloteValidationError as e:
+            context.abort(grpc.StatusCode.INVALID_ARGUMENT, str(e))
+        except Exception as e:
+            context.abort(
+                grpc.StatusCode.INTERNAL, f"SetVariableShapes failed: {e}"
+            )
+
     def preallocate_inputs(self, inputs, flat_inputs, outputs=None, flat_outputs=None):
         """
         Preallocates the inputs before receiving data from the client.
@@ -178,6 +229,14 @@ def preallocate_inputs(self, inputs, flat_inputs, outputs=None, flat_outputs=Non
         inputs to evaluate the residuals and the partials of the residuals.
         """
         for var in self._discipline._var_meta:
+            # validate that dynamic-shape variables have been resolved
+            if var.dynamic_shape and len(var.shape) == 0:
+                raise PhiloteValidationError(
+                    f"Variable '{var.name}' has dynamic_shape=True but "
+                    f"no shape has been set. Call SetVariableShapes "
+                    f"before computing."
+                )
+
             if var.type == data.kInput:
                 inputs[var.name] = np.zeros(var.shape)
                 flat_inputs[var.name] = get_flattened_view(inputs[var.name])

philote_mdo/generated/data_pb2.py

@@ -7,17 +7,17 @@
 _runtime_version.ValidateProtobufRuntimeVersion(_runtime_version.Domain.PUBLIC, 5, 27, 2, '', 'data.proto')
 _sym_db = _symbol_database.Default()
 from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\ndata.proto\x12\x07philote\x1a\x1cgoogle/protobuf/struct.proto"}\n\x14DisciplineProperties\x12\x12\n\ncontinuous\x18\x01 \x01(\x08\x12\x16\n\x0edifferentiable\x18\x02 \x01(\x08\x12\x1a\n\x12provides_gradients\x18\x03 \x01(\x08\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0f\n\x07version\x18\x05 \x01(\t"#\n\rStreamOptions\x12\x12\n\nnum_double\x18\x01 \x01(\x03"?\n\x0bOptionsList\x12\x0f\n\x07options\x18\x01 \x03(\t\x12\x1f\n\x04type\x18\x02 \x03(\x0e2\x11.philote.DataType"=\n\x11DisciplineOptions\x12(\n\x07options\x18\x01 \x01(\x0b2\x17.google.protobuf.Struct"c\n\x10VariableMetaData\x12#\n\x04type\x18\x01 \x01(\x0e2\x15.philote.VariableType\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\r\n\x05shape\x18\x04 \x03(\x03\x12\r\n\x05units\x18\x05 \x01(\t"@\n\x10PartialsMetaData\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07subname\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x03"u\n\x05Array\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07subname\x18\x02 \x01(\t\x12\r\n\x05start\x18\x03 \x01(\x03\x12\x0b\n\x03end\x18\x04 \x01(\x03\x12#\n\x04type\x18\x05 \x01(\x0e2\x15.philote.VariableType\x12\x0c\n\x04data\x18\x06 \x03(\x01"l\n\x10DiscreteVariable\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x04type\x18\x02 \x01(\x0e2\x15.philote.VariableType\x12%\n\x05value\x18\x03 \x01(\x0b2\x16.google.protobuf.Value"q\n\x0fVariableMessage\x12$\n\ncontinuous\x18\x01 \x01(\x0b2\x0e.philote.ArrayH\x00\x12-\n\x08discrete\x18\x02 \x01(\x0b2\x19.philote.DiscreteVariableH\x00B\t\n\x07payload*F\n\x08DataType\x12\t\n\x05kBool\x10\x00\x12\x08\n\x04kInt\x10\x01\x12\x0b\n\x07kDouble\x10\x02\x12\x0b\n\x07kString\x10\x03\x12\x0b\n\x07kStruct\x10\x04*m\n\x0cVariableType\x12\n\n\x06kInput\x10\x00\x12\x12\n\x0ekDiscreteInput\x10\x01\x12\r\n\tkResidual\x10\x02\x12\x0b\n\x07kOutput\x10\x03\x12\x13\n\x0fkDiscreteOutput\x10\x04\x12\x0c\n\x08kPartial\x10\x05B\x11\n\x0forg.philote.mdob\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\ndata.proto\x12\x07philote\x1a\x1cgoogle/protobuf/struct.proto"}\n\x14DisciplineProperties\x12\x12\n\ncontinuous\x18\x01 \x01(\x08\x12\x16\n\x0edifferentiable\x18\x02 \x01(\x08\x12\x1a\n\x12provides_gradients\x18\x03 \x01(\x08\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0f\n\x07version\x18\x05 \x01(\t"#\n\rStreamOptions\x12\x12\n\nnum_double\x18\x01 \x01(\x03"?\n\x0bOptionsList\x12\x0f\n\x07options\x18\x01 \x03(\t\x12\x1f\n\x04type\x18\x02 \x03(\x0e2\x11.philote.DataType"=\n\x11DisciplineOptions\x12(\n\x07options\x18\x01 \x01(\x0b2\x17.google.protobuf.Struct"z\n\x10VariableMetaData\x12#\n\x04type\x18\x01 \x01(\x0e2\x15.philote.VariableType\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\r\n\x05shape\x18\x04 \x03(\x03\x12\r\n\x05units\x18\x05 \x01(\t\x12\x15\n\rdynamic_shape\x18\x06 \x01(\x08"@\n\x10PartialsMetaData\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07subname\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x03"u\n\x05Array\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07subname\x18\x02 \x01(\t\x12\r\n\x05start\x18\x03 \x01(\x03\x12\x0b\n\x03end\x18\x04 \x01(\x03\x12#\n\x04type\x18\x05 \x01(\x0e2\x15.philote.VariableType\x12\x0c\n\x04data\x18\x06 \x03(\x01"l\n\x10DiscreteVariable\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x04type\x18\x02 \x01(\x0e2\x15.philote.VariableType\x12%\n\x05value\x18\x03 \x01(\x0b2\x16.google.protobuf.Value"q\n\x0fVariableMessage\x12$\n\ncontinuous\x18\x01 \x01(\x0b2\x0e.philote.ArrayH\x00\x12-\n\x08discrete\x18\x02 \x01(\x0b2\x19.philote.DiscreteVariableH\x00B\t\n\x07payload*F\n\x08DataType\x12\t\n\x05kBool\x10\x00\x12\x08\n\x04kInt\x10\x01\x12\x0b\n\x07kDouble\x10\x02\x12\x0b\n\x07kString\x10\x03\x12\x0b\n\x07kStruct\x10\x04*m\n\x0cVariableType\x12\n\n\x06kInput\x10\x00\x12\x12\n\x0ekDiscreteInput\x10\x01\x12\r\n\tkResidual\x10\x02\x12\x0b\n\x07kOutput\x10\x03\x12\x13\n\x0fkDiscreteOutput\x10\x04\x12\x0c\n\x08kPartial\x10\x05B\x11\n\x0forg.philote.mdob\x06proto3')
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
 _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'data_pb2', _globals)
 if not _descriptor._USE_C_DESCRIPTORS:
     _globals['DESCRIPTOR']._loaded_options = None
     _globals['DESCRIPTOR']._serialized_options = b'\n\x0forg.philote.mdo'
-    _globals['_DATATYPE']._serialized_start = 856
-    _globals['_DATATYPE']._serialized_end = 926
-    _globals['_VARIABLETYPE']._serialized_start = 928
-    _globals['_VARIABLETYPE']._serialized_end = 1037
+    _globals['_DATATYPE']._serialized_start = 879
+    _globals['_DATATYPE']._serialized_end = 949
+    _globals['_VARIABLETYPE']._serialized_start = 951
+    _globals['_VARIABLETYPE']._serialized_end = 1060
     _globals['_DISCIPLINEPROPERTIES']._serialized_start = 53
     _globals['_DISCIPLINEPROPERTIES']._serialized_end = 178
     _globals['_STREAMOPTIONS']._serialized_start = 180
@@ -27,12 +27,12 @@
     _globals['_DISCIPLINEOPTIONS']._serialized_start = 282
     _globals['_DISCIPLINEOPTIONS']._serialized_end = 343
     _globals['_VARIABLEMETADATA']._serialized_start = 345
-    _globals['_VARIABLEMETADATA']._serialized_end = 444
-    _globals['_PARTIALSMETADATA']._serialized_start = 446
-    _globals['_PARTIALSMETADATA']._serialized_end = 510
-    _globals['_ARRAY']._serialized_start = 512
-    _globals['_ARRAY']._serialized_end = 629
-    _globals['_DISCRETEVARIABLE']._serialized_start = 631
-    _globals['_DISCRETEVARIABLE']._serialized_end = 739
-    _globals['_VARIABLEMESSAGE']._serialized_start = 741
-    _globals['_VARIABLEMESSAGE']._serialized_end = 854
+    _globals['_VARIABLEMETADATA']._serialized_end = 467
+    _globals['_PARTIALSMETADATA']._serialized_start = 469
+    _globals['_PARTIALSMETADATA']._serialized_end = 533
+    _globals['_ARRAY']._serialized_start = 535
+    _globals['_ARRAY']._serialized_end = 652
+    _globals['_DISCRETEVARIABLE']._serialized_start = 654
+    _globals['_DISCRETEVARIABLE']._serialized_end = 762
+    _globals['_VARIABLEMESSAGE']._serialized_start = 764
+    _globals['_VARIABLEMESSAGE']._serialized_end = 877