diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d88967b767..9d8bcd26d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -666,14 +666,14 @@ gcc49_cxx98_debug_hdf5_mpi_regent: variables: <<: [*gcc49, *terra38, *debug, *cxx98_normal, *hdf5, *mpi, *regent] # Multi-node Legion Spy -gcc49_cxx98_debug_spy_gasnet_regent: - <<: [*linux_compute, *image, *tests] - variables: - <<: [*gcc49, *terra38, *cxx98_normal, *spy, *gasnet, *regent] -gcc49_cxx98_debug_spy_mpi_regent: - <<: [*linux_compute, *image, *tests] - variables: - <<: [*gcc49, *terra38, *cxx98_normal, *spy, *mpi, *regent] +#gcc49_cxx98_debug_spy_gasnet_regent: +# <<: [*linux_compute, *image, *tests] +# variables: +# <<: [*gcc49, *terra38, *cxx98_normal, *spy, *gasnet, *regent] +#gcc49_cxx98_debug_spy_mpi_regent: +# <<: [*linux_compute, *image, *tests] +# variables: +# <<: [*gcc49, *terra38, *cxx98_normal, *spy, *mpi, *regent] # * Different architectures gcc49_cxx98_32bit_debug_legion: <<: [*linux, *image, *tests] diff --git a/bindings/python/legion_top.py b/bindings/python/legion_top.py index 59612318fd..b243181711 100644 --- a/bindings/python/legion_top.py +++ b/bindings/python/legion_top.py @@ -263,7 +263,39 @@ def import_global(module, check_depth=True, block=True): # not safe to use with control replication so this will give them a way # to check whether they are running in a safe context or not def is_control_replicated(): - return False + try: + # We should only be doing something for this if we're the top-level task + return c.legion_context_get_num_shards(top_level.runtime[0], + top_level.context[0], True) > 1 + except AttributeError: + raise RuntimeError('"is_control_replicated" must be called in a legion_python task') + + +# Helper class for deduplicating output streams with control replication +class LegionOutputStream(object): + def __init__(self, shard_id, stream): + self.shard_id = shard_id + # This is the original stream + self.stream = stream + + def close(self): + self.stream.close() + + def flush(self): + self.stream.flush() + + def write(self, string): + # Only do the write if we are shard 0 + if self.shard_id == 0: + self.stream.write(string) + + def writelines(self, sequence): + # Only do the write if we are shard 0 + if self.shard_id == 0: + self.stream.writelines(sequence) + + def isatty(self): + return self.stream.isatty() def legion_python_main(raw_args, user_data, proc): @@ -282,6 +314,11 @@ def legion_python_main(raw_args, user_data, proc): top_level.runtime, top_level.context, top_level.task = runtime, context, task + # If we're control replicated, deduplicate stdout + if is_control_replicated(): + shard_id = c.legion_context_get_shard_id(runtime[0], context[0], True) + sys.stdout = LegionOutputStream(shard_id, sys.stdout) + # Run user's script. args = input_args(True) start = 1 diff --git a/bindings/python/main.cc b/bindings/python/main.cc index cee1325b8b..97597af5e5 100644 --- a/bindings/python/main.cc +++ b/bindings/python/main.cc @@ -27,10 +27,21 @@ static bool control_replicate = true; static const char * const unique_name = "legion_python"; static const VariantID vid = 1; +class LegionPyShardingFunctor : public ShardingFunctor { +public: + LegionPyShardingFunctor(void) { } + virtual ~LegionPyShardingFunctor(void) { } +public: + virtual ShardID shard(const DomainPoint &point, + const Domain &full_space, + const size_t total_shards); +}; + // Special mapper just for mapping the top-level Python tasks class LegionPyMapper : public Legion::Mapping::NullMapper { public: - LegionPyMapper(MapperRuntime *runtime, Machine machine, TaskID top_task_id); + LegionPyMapper(MapperRuntime *runtime, Machine machine, + TaskID top_task_id, ShardingID sharding_id); virtual ~LegionPyMapper(void); public: static AddressSpaceID get_local_node(void); @@ -57,6 +68,11 @@ class LegionPyMapper : public Legion::Mapping::NullMapper { const MapTaskInput& input, const MapTaskOutput& default_output, MapReplicateTaskOutput& output); + virtual void select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); virtual void select_tunable_value(const MapperContext ctx, const Task& task, const SelectTunableInput& input, @@ -81,6 +97,7 @@ class LegionPyMapper : public Legion::Mapping::NullMapper { const size_t total_nodes; const char *const mapper_name; const TaskID top_task_id; + const ShardingID sharding_id; protected: std::vector local_pys; // Python processors }; @@ -125,11 +142,14 @@ static void python_main_callback(Machine machine, Runtime *runtime, runtime->register_task_variant(registrar, code_desc, NULL, 0, task_name, vid); runtime->attach_name(top_task_id+2, task_name, false/*mutable*/, true/*local only*/); } + // Register our sharding function for any global import tasks + const ShardingID sharding_id = runtime->generate_library_sharding_ids(unique_name, 1); + runtime->register_sharding_functor(sharding_id, new LegionPyShardingFunctor()); // Register our mapper for the top-level task const MapperID top_mapper_id = runtime->generate_library_mapper_ids(unique_name, 1); runtime->set_top_level_task_mapper_id(top_mapper_id); runtime->add_mapper(top_mapper_id, - new LegionPyMapper(runtime->get_mapper_runtime(), machine, top_task_id)); + new LegionPyMapper(runtime->get_mapper_runtime(), machine, top_task_id, sharding_id)); } int main(int argc, char **argv) @@ -197,10 +217,10 @@ int main(int argc, char **argv) return Runtime::start(argc, argv); } -LegionPyMapper::LegionPyMapper(MapperRuntime *rt, Machine m, TaskID top_id) +LegionPyMapper::LegionPyMapper(MapperRuntime *rt, Machine m, TaskID top_id, ShardingID sid) : NullMapper(rt, m), local_node(get_local_node()), total_nodes(get_total_nodes(m)), mapper_name(create_name(local_node)), - top_task_id(top_id) + top_task_id(top_id), sharding_id(sid) { Machine::ProcessorQuery py_procs(machine); py_procs.local_address_space(); @@ -378,6 +398,28 @@ void LegionPyMapper::map_top_level_task(const MapperContext ctx, output.chosen_variant = vid; } +ShardID LegionPyShardingFunctor::shard(const DomainPoint &point, + const Domain &full_domain, + size_t total_shards) +{ + Point<1> p = point; + Rect<1> bounds = full_domain; + const size_t volume = bounds.volume(); + assert((volume % total_shards) == 0); + const size_t pernode = volume / total_shards; + return (p[0] / pernode); +} + +void LegionPyMapper::select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) +{ + assert(task.task_id == (top_task_id+1)); + output.chosen_functor = sharding_id; +} + void LegionPyMapper::select_tunable_value(const MapperContext ctx, const Task& task, const SelectTunableInput& input, diff --git a/bindings/python/projection_functor_example.c b/bindings/python/projection_functor_example.c new file mode 100644 index 0000000000..300c60a276 --- /dev/null +++ b/bindings/python/projection_functor_example.c @@ -0,0 +1,9 @@ +#include "legion.h" + +void proj_functor(legion_runtime_t runtime, + legion_logical_partition_t parent, + legion_domain_point_t point, + legion_domain_t launch_domain) +{ + legion_projection_functor_logical_partition_print_arguments_2(parent); +} diff --git a/bindings/python/projection_functor_example.py b/bindings/python/projection_functor_example.py new file mode 100644 index 0000000000..46c0a2c9c9 --- /dev/null +++ b/bindings/python/projection_functor_example.py @@ -0,0 +1,67 @@ +from __future__ import print_function + +import pygion +from pygion import ( + index_launch, + task, + Domain, + ID, + IndexLaunch, + R, + Region, + Partition, + ProjectionFunctor, +) + +from typing import cast, Callable + +import subprocess +import petra as pt + +f = ProjectionFunctor.create(1 + ID) + + +@task(privileges=[R]) +def hello(R, i, num): + print("hello from point %s (region %s)" % (i, R.ispace.bounds)) + assert int(R.ispace.bounds[0, 0]) == int(i + num) + + +@task +def main(): + R = Region([4], {"x": pygion.float64}) + P = Partition.equal(R, [4]) + for i in range(4): + print( + "python region %s is %s %s %s" + % ( + i, + P[i].handle[0].tree_id, + P[i].handle[0].index_space.tid, + P[i].handle[0].index_space.id, + ) + ) + pygion.fill(R, "x", 0) + + for i in IndexLaunch([3]): + hello(P[f(i)], i, 1) + + for i in IndexLaunch([3]): + hello(P[i], i, 0) + + for i in IndexLaunch([2]): + hello(P[i + 2], i, 2) + + for i in IndexLaunch([2]): + hello(P[i + 2], i, 2) + + index_launch([3], hello, P[ID], ID, 0) + + # This Seg Fault when running all tests but not when it's the only test: + index_launch([3], hello, P[f(ID)], ID, 1) + + index_launch([2], hello, P[ID + 2], ID, 2) + + +if __name__ == "__main__": + main() diff --git a/bindings/python/pygion.py b/bindings/python/pygion.py index eafac55a05..6c46ae7c52 100644 --- a/bindings/python/pygion.py +++ b/bindings/python/pygion.py @@ -1,2493 +1,3512 @@ -#!/usr/bin/env python - -# Copyright 2020 Stanford University -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from __future__ import absolute_import, division, print_function, unicode_literals - -try: - import cPickle as pickle -except ImportError: - import pickle -import collections -import contextlib -from io import StringIO -import itertools -import math -import numpy -import os -import re -import subprocess -import sys -import threading -import weakref - -# Python 3.x compatibility: -try: - long # Python 2 -except NameError: - long = int # Python 3 - -try: - basestring # Python 2 -except NameError: - basestring = str # Python 3 - -try: - xrange # Python 2 -except NameError: - xrange = range # Python 3 - -try: - imap = itertools.imap # Python 2 -except: - imap = map # Python 3 - -try: - zip_longest = itertools.izip_longest # Python 2 -except: - zip_longest = itertools.zip_longest # Python 3 - -_pickle_version = pickle.HIGHEST_PROTOCOL # Use latest Pickle protocol - -import legion_top -from legion_cffi import ffi, lib as c - -_max_dim = None -for dim in range(1, 9): - try: - getattr(c, 'legion_domain_get_rect_{}d'.format(dim)) - except AttributeError: - break - _max_dim = dim -assert _max_dim is not None, 'Unable to detect LEGION_MAX_DIM' - -AUTO_GENERATE_ID = c.legion_auto_generate_id() - -# Duplicate enum values from legion_config.h since CFFI isn't smart -# enough to parse them directly. - -EXTERNAL_HDF5_FILE = 1 - -NO_ACCESS = 0x00000000 -READ_PRIV = 0x00000001 -READ_ONLY = 0x00000001 # READ_PRIV -WRITE_PRIV = 0x00000002 -REDUCE_PRIV = 0x00000004 -REDUCE = 0x00000004 # REDUCE_PRIV -READ_WRITE = 0x00000007 # READ_PRIV | WRITE_PRIV | REDUCE_PRIV -DISCARD_MASK = 0x10000000 # For marking we don't need inputs -WRITE_ONLY = 0x10000002 # WRITE_PRIV | DISCARD_MASK -WRITE_DISCARD = 0x10000007 # READ_WRITE | DISCARD_MASK - -# Note: don't use __file__ here, it may return either .py or .pyc and cause -# non-deterministic failures. -library_name = "pygion.py" -max_legion_python_tasks = 1000000 -next_legion_task_id = c.legion_runtime_generate_library_task_ids( - c.legion_runtime_get_runtime(), - library_name.encode('utf-8'), - max_legion_python_tasks) -max_legion_task_id = next_legion_task_id + max_legion_python_tasks - -# Returns true if this module is running inside of a Legion -# executable. If false, then other Legion functionality should not be -# expected to work. -def inside_legion_executable(): - try: - c.legion_get_current_time_in_micros() - except AttributeError: - return False - else: - return True - -input_args = legion_top.input_args - -def execute_as_script(): - args = input_args(True) - if len(args) < 1: - return False, False # no idea what's going on here, just return - if os.path.basename(args[0]) != 'legion_python': - return False, False # not in legion_python - if len(args) < 2 or args[1].startswith('-'): - return True, False # argument is a flag - # If it has an extension, we're going to guess that it was - # intended to be a script. - return True, len(os.path.splitext(args[1])[1]) > 1 - -is_legion_python, is_script = execute_as_script() - -# The Legion context is stored in thread-local storage. This assumes -# that the Python processor maintains the invariant that every task -# corresponds to one and only one thread. -_my = threading.local() - -global_task_registration_barrier = None - -class Context(object): - __slots__ = ['context_root', 'context', 'runtime_root', 'runtime', - 'task_root', 'task', 'regions', - 'owned_objects', 'current_launch', 'next_trace_id'] - def __init__(self, context_root, runtime_root, task_root, regions): - self.context_root = context_root - self.context = self.context_root[0] - self.runtime_root = runtime_root - self.runtime = self.runtime_root[0] - self.task_root = task_root - self.task = self.task_root[0] - self.regions = regions - self.owned_objects = [] - self.current_launch = None - self.next_trace_id = 0 - def track_object(self, obj): - self.owned_objects.append(weakref.ref(obj)) - def begin_launch(self, launch): - assert self.current_launch == None - self.current_launch = launch - def end_launch(self, launch): - assert self.current_launch == launch - self.current_launch = None - -# Hack: Can't pickle static methods. -def _DomainPoint_unpickle(values): - return DomainPoint(values) - -class DomainPoint(object): - __slots__ = [ - 'handle', - '_point', # cached properties - ] - def __init__(self, values, **kwargs): - def parse_kwargs(_handle=None): - return _handle - handle = parse_kwargs(**kwargs) - - if values is not None: - assert handle is None - try: - len(values) - except TypeError: - values = [values] - assert 1 <= len(values) <= _max_dim - self.handle = ffi.new('legion_domain_point_t *') - self.handle[0].dim = len(values) - for i, value in enumerate(values): - self.handle[0].point_data[i] = value - else: - # Important: Copy handle. Do NOT assume ownership. - assert handle is not None - self.handle = ffi.new('legion_domain_point_t *', handle) - - self._point = None - - def __reduce__(self): - return (_DomainPoint_unpickle, - ([self.handle[0].point_data[i] for i in xrange(self.dim)],)) - - def __int__(self): - assert self.dim == 1 - return self.handle[0].point_data[0] - - def __index__(self): - return self.__int__() - - def __getitem__(self, i): - assert 0 <= i < self.dim - return self.handle[0].point_data[i] - - def __eq__(self, other): - if not isinstance(other, DomainPoint): - return NotImplemented - return numpy.array_equal(self.point, other.point) - - def __str__(self): - if self.dim == 1: - return str(int(self)) - return str(self.point) - - def __repr__(self): - return 'DomainPoint({})'.format(self.point) - - @property - def dim(self): - return self.handle[0].dim - - @property - def point(self): - if self._point is None: - self._point = self.asarray() - return self._point - - @staticmethod - def coerce(value): - if not isinstance(value, DomainPoint): - return DomainPoint(value) - return value - - def raw_value(self): - return self.handle[0] - - def asarray(self): - return numpy.frombuffer( - ffi.buffer(self.handle[0].point_data), - count=self.dim, - dtype=numpy.int64) - -class Domain(object): - __slots__ = [ - 'handle', - '_bounds', # cached properties - ] - def __init__(self, extent, start=None, **kwargs): - def parse_kwargs(_handle=None): - return _handle - handle = parse_kwargs(**kwargs) - - if extent is not None: - assert handle is None - try: - len(extent) - except TypeError: - extent = [extent] - if start is not None: - try: - len(start) - except TypeError: - start = [start] - assert len(start) == len(extent) - else: - start = [0 for _ in extent] - assert 1 <= len(extent) <= _max_dim - rect = ffi.new('legion_rect_{}d_t *'.format(len(extent))) - for i in xrange(len(extent)): - rect[0].lo.x[i] = start[i] - rect[0].hi.x[i] = start[i] + extent[i] - 1 - handle = getattr(c, 'legion_domain_from_rect_{}d'.format(len(extent)))(rect[0]) - - # Important: Copy handle. Do NOT assume ownership. - assert handle is not None - self.handle = ffi.new('legion_domain_t *', handle) - self._bounds = None - - @property - def dim(self): - return self.handle[0].dim - - @property - def volume(self): - return c.legion_domain_get_volume(self.handle[0]) - - @property - def bounds(self): - if self._bounds is None: - self._bounds = self.asarray() - return self._bounds - - @property - def extent(self): - bounds = self.bounds - return bounds[1] - bounds[0] + 1 - - @property - def start(self): - return self.bounds[0] - - @staticmethod - def coerce(value): - if not isinstance(value, Domain): - return Domain(value) - return value - - def __iter__(self): - return imap( - DomainPoint, - itertools.product( - *[xrange( - self.handle[0].rect_data[i], - self.handle[0].rect_data[i+self.dim] + 1) - for i in xrange(self.dim)])) - - def raw_value(self): - return self.handle[0] - - def asarray(self): - return numpy.frombuffer( - ffi.buffer(self.handle[0].rect_data), - count=self.dim * 2, - dtype=numpy.int64 - ).reshape((2, self.dim)) - -class DomainTransform(object): - __slots__ = ['handle'] - def __init__(self, matrix, **kwargs): - def parse_kwargs(_handle=None): - return _handle - handle = parse_kwargs(**kwargs) - - if matrix is not None: - assert handle is None - matrix = numpy.asarray(matrix, dtype=numpy.int64) - transform = ffi.new('legion_transform_{}x{}_t *'.format(*matrix.shape)) - ffi.buffer(transform[0].trans)[:] = matrix - handle = getattr(c, 'legion_domain_transform_from_{}x{}'.format(*matrix.shape))(transform[0]) - - # Important: Copy handle. Do NOT assume ownership. - assert handle is not None - self.handle = ffi.new('legion_domain_transform_t *', handle) - - @staticmethod - def coerce(value): - if not isinstance(value, DomainTransform): - return DomainTransform(value) - return value - - def raw_value(self): - return self.handle[0] - -class Future(object): - __slots__ = ['handle', 'value_type', 'argument_number'] - def __init__(self, value, value_type=None, argument_number=None): - if value is None: - self.handle = None - elif isinstance(value, Future): - value.resolve_handle() - self.handle = c.legion_future_copy(value.handle) - if value_type is None: - value_type = value.value_type - elif value_type is not None: - if value_type.size > 0: - value_ptr = ffi.new(ffi.getctype(value_type.cffi_type, '*'), value) - else: - assert value is None - value_ptr = ffi.NULL - value_size = value_type.size - self.handle = c.legion_future_from_untyped_pointer(_my.ctx.runtime, value_ptr, value_size) - else: - value_str = pickle.dumps(value, protocol=_pickle_version) - value_size = len(value_str) - value_ptr = ffi.new('char[]', value_size) - ffi.buffer(value_ptr, value_size)[:] = value_str - self.handle = c.legion_future_from_untyped_pointer(_my.ctx.runtime, value_ptr, value_size) - - self.value_type = value_type - self.argument_number = argument_number - - @staticmethod - def from_cdata(value, *args, **kwargs): - result = Future(None, *args, **kwargs) - result.handle = c.legion_future_copy(value) - return result - - @staticmethod - def from_buffer(value, *args, **kwargs): - result = Future(None, *args, **kwargs) - result.handle = c.legion_future_from_untyped_pointer(_my.ctx.runtime, ffi.from_buffer(value), len(value)) - return result - - def __del__(self): - if self.handle is not None: - c.legion_future_destroy(self.handle) - - def __reduce__(self): - if self.argument_number is None: - raise Exception('Cannot pickle a Future except when used as a task argument') - return (Future, (None, self.value_type, self.argument_number)) - - def resolve_handle(self): - if self.handle is None and self.argument_number is not None: - self.handle = c.legion_future_copy( - c.legion_task_get_future(_my.ctx.task, self.argument_number)) - - def get(self): - self.resolve_handle() - - if self.handle is None: - return - if self.value_type is None: - value_ptr = c.legion_future_get_untyped_pointer(self.handle) - value_size = c.legion_future_get_untyped_size(self.handle) - assert value_size > 0 - value_str = ffi.unpack(ffi.cast('char *', value_ptr), value_size) - value = pickle.loads(value_str) - return value - elif self.value_type.size == 0: - c.legion_future_get_void_result(self.handle) - else: - expected_size = ffi.sizeof(self.value_type.cffi_type) - - value_ptr = c.legion_future_get_untyped_pointer(self.handle) - value_size = c.legion_future_get_untyped_size(self.handle) - assert value_size == expected_size - value = ffi.cast(ffi.getctype(self.value_type.cffi_type, '*'), value_ptr)[0] - # Hack: Use closure to keep self alive as long as the value is live. - if isinstance(value, ffi.CData): - return ffi.gc(value, lambda x: self) - return value - - def get_buffer(self): - self.resolve_handle() - - if self.handle is None: - return - value_ptr = c.legion_future_get_untyped_pointer(self.handle) - value_size = c.legion_future_get_untyped_size(self.handle) - return ffi.buffer(value_ptr, value_size) - -class FutureMap(object): - __slots__ = ['handle', 'value_type'] - def __init__(self, handle, value_type=None): - self.handle = c.legion_future_map_copy(handle) - self.value_type = value_type - - def __del__(self): - c.legion_future_map_destroy(self.handle) - - def __getitem__(self, point): - point = DomainPoint.coerce(point) - return Future.from_cdata( - c.legion_future_map_get_future(self.handle, point.raw_value()), - value_type=self.value_type) - - def wait_all_results(self): - c.legion_future_map_wait_all_results(self.handle) - -_type_cache = {} - -class Type(object): - __slots__ = ['numpy_type', 'cffi_type', 'size'] - - def __new__(cls, numpy_type, cffi_type): - if cffi_type in _type_cache: - return _type_cache[cffi_type] - obj = super(Type, cls).__new__(cls) - _type_cache[cffi_type] = obj - return obj - - def __init__(self, numpy_type, cffi_type): - assert (numpy_type is None) == (cffi_type is None) - self.numpy_type = numpy_type - self.cffi_type = cffi_type - self.size = ffi.sizeof(cffi_type) if cffi_type is not None else 0 - - def __reduce__(self): - return (Type, (self.numpy_type, self.cffi_type)) - -# Pre-defined Types -void = Type(None, None) -bool_ = Type(numpy.bool_, 'bool') -complex64 = Type(numpy.complex64, 'float _Complex') -complex128 = Type(numpy.complex128, 'double _Complex') -float32 = Type(numpy.float32, 'float') -float64 = Type(numpy.float64, 'double') -int8 = Type(numpy.int8, 'int8_t') -int16 = Type(numpy.int16, 'int16_t') -int32 = Type(numpy.int32, 'int32_t') -int64 = Type(numpy.int64, 'int64_t') -uint8 = Type(numpy.uint8, 'uint8_t') -uint16 = Type(numpy.uint16, 'uint16_t') -uint32 = Type(numpy.uint32, 'uint32_t') -uint64 = Type(numpy.uint64, 'uint64_t') - -_rect_types = [] -for dim in xrange(1, _max_dim + 1): - globals()["int{}d".format(dim)] = Type( - numpy.dtype([('x', numpy.int64, (dim,))], align=True), - 'legion_point_{}d_t'.format(dim)) - rtype = Type( - numpy.dtype([('lo', numpy.int64, (dim,)), ('hi', numpy.int64, (dim,))], align=True), - 'legion_rect_{}d_t'.format(dim)) - globals()["rect{}d".format(dim)] = rtype - _rect_types.append(rtype) -_rect_types = frozenset(_rect_types) - -def is_rect_type(t): - return t in _rect_types - -_redop_ids = {} -def _fill_redop_ids(): - operators = ['+', '-', '*', '/', 'max', 'min'] - types = [bool_, int8, int16, int32, int64, uint8, uint16, uint32, uint64, None, float32, float64, None, complex64, complex128] - next_id = 1048576 - for operator in operators: - _redop_ids[operator] = {} - for type in types: - if type is not None: - _redop_ids[operator][type] = next_id - next_id += 1 -_fill_redop_ids() - -class Privilege(object): - __slots__ = ['read', 'write', 'discard', 'reduce', 'fields'] - - def __init__(self, read=False, write=False, discard=False, reduce=False, fields=None): - self.read = read - self.write = write - self.discard = discard - self.reduce = reduce - self.fields = fields - - if self.fields is not None: - assert len(self.fields) > 0 - - if self.discard: - assert self.write - - def _fields(self): - return (self.read, self.write, self.discard, self.reduce, self.fields) - - def __eq__(self, other): - if not isinstance(other, Privilege): - return NotImplemented - return self._fields() == other._fields() - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return hash(self._fields()) - - def __call__(self, *fields): - assert self.fields is None - return Privilege(self.read, self.write, self.discard, self.reduce, fields) - - def __add__(self, other): - return PrivilegeComposite([self, other]) - - def __repr__(self): - return str(self) - - def __str__(self): - if self.discard: - priv = 'WD' - elif self.write: - priv = 'RW' - elif self.read: - priv = 'R' - elif self.reduce: - priv = 'Reduce(%s)' % self.reduce - else: - priv = 'N' - if self.fields is not None: - return '%s(%s)' % (priv, ', '.join(self.fields)) - return priv - - def _legion_privilege(self): - bits = NO_ACCESS - if self.reduce: - assert False - else: - if self.write: bits = READ_WRITE - elif self.read: bits = READ_ONLY - if self.discard: - bits |= DISCARD_MASK - return bits - - def _legion_grouped_privileges(self, fspace): - if self.fields: - if not set(self.fields) <= set(fspace.keys()): - raise Exception( - 'Privilege fields ({}) are not a subset of fspace fields ({})'.format( - ' '.join(self.fields), ' '.join(fspace.keys()))) - fields = fspace.keys() if self.fields is None else self.fields - if self.reduce: - return [ - (self, None, self._legion_redop_id(fspace.field_types[field_name]), (field_name,)) - for field_name in fields] - else: - return [(self, self._legion_privilege(), None, fields if self.read or self.write or self.reduce else [])] - - def _legion_redop_id(self, field_type): - return _redop_ids[self.reduce][field_type] - -class PrivilegeComposite(object): - __slots__ = ['privileges'] - - def __init__(self, privileges): - self.privileges = self.normalize(privileges) - - @staticmethod - def normalize(privileges): - fields = collections.OrderedDict() - read_set = set() - write_set = set() - discard_set = set() - reduce_sets = collections.OrderedDict() - - for privilege in privileges: - privilege_fields = privilege.fields if privilege.fields is not None else [None] - fields.update([(x, True) for x in privilege_fields]) - if privilege.read: - read_set.update(privilege_fields) - if privilege.write: - write_set.update(privilege_fields) - if privilege.discard: - discard_set.update(privilege_fields) - if privilege.reduce: - if privilege.reduce not in reduce_sets: - reduce_sets[privilege.reduce] = set() - reduce_sets[privilege.reduce].update(privilege_fields) - - # Reductions combine with read/reduce privileges to upgrade to read-write. - for op, reduce_set in reduce_sets.items(): - write_set.update(reduce_set & read_set) - if None in read_set: - write_set.update(reduce_set) - for op2, reduce_set2 in reduce_sets.items(): - if op != op2: - write_set.update(reduce_set & reduce_set2) - if None in reduce_set2: - write_set.update(reduce_set) - - # Read/write/discard shadow reduction privileges. - if None in read_set or None in write_set or None in discard_set: - reduce_sets = collections.OrderedDict() - else: - for reduce_set in reduce_sets.values(): - reduce_set.difference_update(read_set, write_set, discard_set) - - # Discard shadows read/write. - if None in discard_set: - read_set = set() - write_set = set() - discard_set = set([None]) - else: - read_set -= discard_set - write_set -= discard_set - - # Write shadows read. - if None in write_set: - read_set = set() - write_set = set([None]) - else: - read_set -= write_set - - def filter_set(ctor, field_set): - if None in field_set: - return ctor - return ctor(*filter(lambda x: x in field_set, fields.keys())) - - return tuple( - ([filter_set(R, read_set)] if len(read_set) > 0 else []) + - ([filter_set(RW, write_set)] if len(write_set) > 0 else []) + - ([filter_set(WD, discard_set)] if len(discard_set) > 0 else []) + - [filter_set(Reduce(op), reduce_set) for op, reduce_set in reduce_sets.items()]) - - def __eq__(self, other): - if len(self.privileges) == 1: - return other == self.privileges[0] - - if not isinstance(other, PrivilegeComposite): - return NotImplemented - return self.privileges == other.privileges - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return hash(self.privileges) - - def __add__(self, other): - return PrivilegeComposite(self.privileges + (other,)) - - def __repr__(self): - return str(self) - - def __str__(self): - return ' + '.join(map(str, self.privileges)) - - def _legion_grouped_privileges(self, fspace): - return [x for privilege in self.privileges for x in privilege._legion_grouped_privileges(fspace)] - -# Pre-defined Privileges -N = Privilege() -R = Privilege(read=True) -RO = Privilege(read=True) -RW = Privilege(read=True, write=True) -WD = Privilege(write=True, discard=True) - -def Reduce(operator, *fields): - return Privilege(reduce=operator, fields=fields if len(fields) > 0 else None) - -class Disjointness(object): - __slots__ = ['kind', 'value'] - - def __init__(self, kind, value): - self.kind = kind - self.value = value - - def __eq__(self, other): - return isinstance(other, Disjointness) and self.value == other.value - - def __cmp__(self, other): - assert isinstance(other, Disjointness) - return self.value.__cmp__(other.value) - - def __hash__(self): - return hash(self.value) - - def __str__(self): - return self.kind - -disjoint = Disjointness('disjoint', 0) -aliased = Disjointness('aliased', 1) -compute = Disjointness('compute', 2) -disjoint_complete = Disjointness('disjoint_complete', 3) -aliased_complete = Disjointness('aliased_complete', 4) -compute_complete = Disjointness('compute_complete', 5) -disjoint_incomplete = Disjointness('disjoint_incomplete', 6) -aliased_incomplete = Disjointness('aliased_incomplete', 7) -compute_incomplete = Disjointness('compute_incomplete', 8) - -class FileMode(object): - __slots__ = ['kind', 'value'] - - def __init__(self, kind, value): - self.kind = kind - self.value = value - - def __eq__(self, other): - return isinstance(other, FileMode) and self.value == other.value - - def __cmp__(self, other): - assert isinstance(other, FileMode) - return self.value.__cmp__(other.value) - - def __hash__(self): - return hash(self.value) - - def __str__(self): - return self.kind - -file_read_only = FileMode('read_only', 0) -file_read_write = FileMode('read_write', 1) -file_create = FileMode('create', 2) - -# Hack: Can't pickle static methods. -def _Ispace_unpickle(ispace_tid, ispace_id, ispace_type_tag, owned): - handle = ffi.new('legion_index_space_t *') - handle[0].tid = ispace_tid - handle[0].id = ispace_id - handle[0].type_tag = ispace_type_tag - return Ispace(None, _handle=handle[0], _owned=owned) - -class Ispace(object): - __slots__ = [ - 'handle', 'owned', 'escaped', - '_domain', # cached properties - '__weakref__', # allow weak references - ] - - def __init__(self, extent, start=None, name=None, **kwargs): - def parse_kwargs(_handle=None, _owned=False): - return _handle, _owned - handle, owned = parse_kwargs(**kwargs) - - if extent is not None: - assert handle is None - domain = Domain(extent, start=start).raw_value() - handle = c.legion_index_space_create_domain(_my.ctx.runtime, _my.ctx.context, domain) - if name is not None: - c.legion_index_space_attach_name(_my.ctx.runtime, handle, name.encode('utf-8'), False) - owned = True - - # Important: Copy handle. Do NOT assume ownership. - assert handle is not None - self.handle = ffi.new('legion_index_space_t *', handle) - self.owned = owned - self.escaped = False - self._domain = None - - if self.owned: - _my.ctx.track_object(self) - - def __del__(self): - if self.owned and not self.escaped: - self.destroy() - - def __reduce__(self): - return (_Ispace_unpickle, - (self.handle[0].tid, - self.handle[0].id, - self.handle[0].type_tag, - self.owned and self.escaped)) - - def __iter__(self): - return self.domain.__iter__() - - @property - def domain(self): - if self._domain is None: - self._domain = Domain(None, _handle=c.legion_index_space_get_domain(_my.ctx.runtime, self.handle[0])) - return self._domain - - @property - def dim(self): - return self.domain.dim - - @property - def volume(self): - return self.domain.volume - - @property - def bounds(self): - return self.domain.bounds - - @staticmethod - def coerce(value): - if not isinstance(value, Ispace): - return Ispace(value) - return value - - def destroy(self): - assert self.owned and not self.escaped - - # This is not something you want to have happen in a - # destructor, since fspaces may outlive the lifetime of the handle. - c.legion_index_space_destroy( - _my.ctx.runtime, _my.ctx.context, self.handle[0]) - # Clear out references. Technically unnecessary but avoids abuse. - del self.handle - - def raw_value(self): - return self.handle[0] - -# Hack: Can't pickle static methods. -def _Fspace_unpickle(fspace_id, field_ids, field_types, owned): - handle = ffi.new('legion_field_space_t *') - handle[0].id = fspace_id - return Fspace(None, _handle=handle[0], _field_ids=field_ids, _field_types=field_types, _owned=owned) - -class Fspace(object): - __slots__ = [ - 'handle', 'field_ids', 'field_types', - 'owned', 'escaped', - '__weakref__', # allow weak references - ] - - def __init__(self, fields, name=None, **kwargs): - def parse_kwargs(_handle=None, _field_ids=None, _field_types=None, _owned=False): - return _handle, _field_ids, _field_types, _owned - handle, field_ids, field_types, owned = parse_kwargs(**kwargs) - - if fields is not None: - assert handle is None and field_ids is None and field_types is None - handle = c.legion_field_space_create(_my.ctx.runtime, _my.ctx.context) - if name is not None: - c.legion_field_space_attach_name(_my.ctx.runtime, handle, name.encode('utf-8'), False) - alloc = c.legion_field_allocator_create( - _my.ctx.runtime, _my.ctx.context, handle) - field_ids = collections.OrderedDict() - field_types = collections.OrderedDict() - for field_name, field_entry in fields.items(): - try: - field_type, field_id = field_entry - except TypeError: - field_type = field_entry - field_id = ffi.cast('legion_field_id_t', AUTO_GENERATE_ID) - field_id = c.legion_field_allocator_allocate_field( - alloc, field_type.size, field_id) - c.legion_field_id_attach_name( - _my.ctx.runtime, handle, field_id, field_name.encode('utf-8'), False) - field_ids[field_name] = field_id - field_types[field_name] = field_type - c.legion_field_allocator_destroy(alloc) - owned = True - - # Important: Copy handle. Do NOT assume ownership. - assert handle is not None and field_ids is not None and field_types is not None - self.handle = ffi.new('legion_field_space_t *', handle) - self.field_ids = field_ids - self.field_types = field_types - self.owned = owned - self.escaped = False - - if owned: - _my.ctx.track_object(self) - - def __del__(self): - if self.owned and not self.escaped: - self.destroy() - - def __reduce__(self): - return (_Fspace_unpickle, - (self.handle[0].id, - self.field_ids, - self.field_types, - self.owned and self.escaped)) - - @staticmethod - def coerce(value): - if not isinstance(value, Fspace): - return Fspace(value) - return value - - def destroy(self): - assert self.owned and not self.escaped - - # This is not something you want to have happen in a - # destructor, since fspaces may outlive the lifetime of the handle. - c.legion_field_space_destroy( - _my.ctx.runtime, _my.ctx.context, self.handle[0]) - # Clear out references. Technically unnecessary but avoids abuse. - del self.handle - del self.field_ids - del self.field_types - - def raw_value(self): - return self.handle[0] - - def keys(self): - return self.field_ids.keys() - -# Hack: Can't pickle static methods. -def _Region_unpickle(ispace, fspace, tree_id, owned): - handle = ffi.new('legion_logical_region_t *') - handle[0].tree_id = tree_id - handle[0].index_space = ispace.handle[0] - handle[0].field_space = fspace.handle[0] - - return Region(ispace, fspace, _handle=handle[0], _owned=owned) - -class Region(object): - __slots__ = [ - 'handle', 'ispace', 'fspace', 'parent', - 'instances', 'privileges', 'instance_wrappers', - 'owned', 'escaped', - '__weakref__', # allow weak references - ] - - # Make this speak the Type interface - numpy_type = None - cffi_type = 'legion_logical_region_t' - size = ffi.sizeof(cffi_type) - - def __init__(self, ispace, fspace, name=None, **kwargs): - def parse_kwargs(_handle=None, _parent=None, _owned=False): - return _handle, _parent, _owned - handle, parent, owned = parse_kwargs(**kwargs) - - if handle is None: - assert parent is None - ispace = Ispace.coerce(ispace) - fspace = Fspace.coerce(fspace) - handle = c.legion_logical_region_create( - _my.ctx.runtime, _my.ctx.context, ispace.raw_value(), fspace.raw_value(), False) - if name is not None: - c.legion_logical_region_attach_name(_my.ctx.runtime, handle, name.encode('utf-8'), False) - owned = True - - # Important: Copy handle. Do NOT assume ownership. - assert handle is not None - self.handle = ffi.new('legion_logical_region_t *', handle) - self.ispace = ispace - self.fspace = fspace - self.parent = parent - self.owned = owned - self.escaped = False - self.instances = {} - self.privileges = {} - self.instance_wrappers = {} - - if owned: - _my.ctx.track_object(self) - for field_name in fspace.field_ids.keys(): - self._set_privilege(field_name, RW) - - def __del__(self): - if self.owned and not self.escaped: - self.destroy() - - def __reduce__(self): - return (_Region_unpickle, - (self.ispace, - self.fspace, - self.handle[0].tree_id, - self.owned and self.escaped)) - - def destroy(self): - assert self.owned and not self.escaped - - # This is not something you want to have happen in a - # destructor, since regions may outlive the lifetime of the handle. - c.legion_logical_region_destroy( - _my.ctx.runtime, _my.ctx.context, self.handle[0]) - # Clear out references. Technically unnecessary but avoids abuse. - del self.parent - del self.instance_wrappers - del self.instances - del self.handle - del self.ispace - del self.fspace - - def raw_value(self): - return self.handle[0] - - def keys(self): - return self.fspace.keys() - - def values(self): - for key in self.keys(): - if key in self.privileges and self.privileges[key] is not None: - yield getattr(self, key) - - def items(self): - for key in self.keys(): - if key in self.privileges and self.privileges[key] is not None: - yield key, getattr(self, key) - - def _set_privilege(self, field_name, privilege): - assert self.parent is None # not supported on subregions - assert field_name not in self.privileges - self.privileges[field_name] = privilege - - def _set_instance(self, field_name, instance, privilege=None): - assert self.parent is None # not supported on subregions - assert field_name not in self.instances - self.instances[field_name] = instance - if privilege is not None: - self._set_privilege(field_name, privilege) - - def _clear_instance(self, field_name): - assert self.parent is None # not supported on subregions - if field_name in self.instances: - # FIXME: need to determine when it is safe to destroy the - # associated instance (may or may not be inline mapped) - del self.instances[field_name] - - def _map_inline(self): - assert self.parent is None # FIXME: support inline mapping subregions - - fields_by_privilege = collections.defaultdict(set) - for field_name, privilege in self.privileges.items(): - fields_by_privilege[privilege].add(field_name) - for privilege, field_names in fields_by_privilege.items(): - launcher = c.legion_inline_launcher_create_logical_region( - self.handle[0], - privilege._legion_privilege(), 0, # EXCLUSIVE - self.handle[0], - 0, False, 0, 0) - for field_name in field_names: - c.legion_inline_launcher_add_field( - launcher, self.fspace.field_ids[field_name], True) - instance = c.legion_inline_launcher_execute( - _my.ctx.runtime, _my.ctx.context, launcher) - for field_name in field_names: - self._set_instance(field_name, instance) - - def __getattr__(self, field_name): - if field_name in self.fspace.field_ids: - if field_name not in self.instances: - if self.privileges[field_name] is None: - raise Exception('Invalid attempt to access field "%s" without privileges' % field_name) - self._map_inline() - if field_name not in self.instance_wrappers: - self.instance_wrappers[field_name] = RegionField( - self, field_name) - return self.instance_wrappers[field_name] - else: - raise AttributeError() - -class RegionField(numpy.ndarray): - # NumPy requires us to implement __new__ for subclasses of ndarray: - # https://docs.scipy.org/doc/numpy/user/basics.subclassing.html - def __new__(cls, region, field_name): - accessor = RegionField._get_accessor(region, field_name) - initializer = RegionField._get_array_initializer(region, field_name, accessor) - if initializer is None: - obj = numpy.empty(tuple(0 for i in xrange(region.ispace.dim))).view( - dtype=region.fspace.field_types[field_name].numpy_type, - type=cls) - else: - obj = numpy.asarray(initializer).view( - dtype=region.fspace.field_types[field_name].numpy_type, - type=cls) - - obj.accessor = accessor - return obj - - @staticmethod - def _get_accessor(region, field_name): - # Note: the accessor needs to be kept alive, to make sure to - # save the result of this function in an instance variable. - instance = region.instances[field_name] - dim = region.ispace.dim - get_accessor = getattr(c, 'legion_physical_region_get_field_accessor_array_{}d'.format(dim)) - return get_accessor(instance, region.fspace.field_ids[field_name]) - - @staticmethod - def _get_base_and_stride(region, field_name, accessor): - domain = region.ispace.domain - dim = domain.dim - if domain.volume < 1: - return None, None, None - - rect = getattr(c, 'legion_domain_get_rect_{}d'.format(dim))(domain.raw_value()) - subrect = ffi.new('legion_rect_{}d_t *'.format(dim)) - offsets = ffi.new('legion_byte_offset_t[]', dim) - - base_ptr = getattr(c, 'legion_accessor_array_{}d_raw_rect_ptr'.format(dim))( - accessor, rect, subrect, offsets) - assert base_ptr - for i in xrange(dim): - assert subrect[0].lo.x[i] == rect.lo.x[i] - assert subrect[0].hi.x[i] == rect.hi.x[i] - assert offsets[0].offset == region.fspace.field_types[field_name].size - - shape = tuple(rect.hi.x[i] - rect.lo.x[i] + 1 for i in xrange(dim)) - strides = tuple(offsets[i].offset for i in xrange(dim)) - - return base_ptr, shape, strides - - @staticmethod - def _get_array_initializer(region, field_name, accessor): - base_ptr, shape, strides = RegionField._get_base_and_stride( - region, field_name, accessor) - if base_ptr is None: - return None - - field_type = region.fspace.field_types[field_name] - - # Numpy doesn't know about CFFI pointers, so we have to cast - # this to a Python long before we can hand it off to Numpy. - base_ptr = long(ffi.cast("size_t", base_ptr)) - - return _RegionNdarray(shape, field_type, base_ptr, strides, False) - -# This is a dummy object that is only used as an initializer for the -# RegionField object above. It is thrown away as soon as the -# RegionField is constructed. -class _RegionNdarray(object): - __slots__ = ['__array_interface__'] - def __init__(self, shape, field_type, base_ptr, strides, read_only): - # See: https://docs.scipy.org/doc/numpy/reference/arrays.interface.html - self.__array_interface__ = { - 'version': 3, - 'shape': shape, - 'typestr': numpy.dtype(field_type.numpy_type).str, - 'data': (base_ptr, read_only), - 'strides': strides, - } - -def fill(region, field_names, value): - assert(isinstance(region, Region)) - if isinstance(field_names, basestring): - field_names = [field_names] - - for field_name in field_names: - field_id = region.fspace.field_ids[field_name] - field_type = region.fspace.field_types[field_name] - raw_value = ffi.new('{} *'.format(field_type.cffi_type), value) - c.legion_runtime_fill_field( - _my.ctx.runtime, _my.ctx.context, - region.raw_value(), region.parent.raw_value() if region.parent is not None else region.raw_value(), - field_id, raw_value, field_type.size, - c.legion_predicate_true()) - -def copy(src_region, src_field_names, dst_region, dst_field_names, redop=None): - assert(isinstance(src_region, Region)) - assert(isinstance(dst_region, Region)) - - if isinstance(src_field_names, basestring): - src_field_names = [src_field_names] - if isinstance(dst_field_names, basestring): - dst_field_names = [dst_field_names] - - launcher = c.legion_copy_launcher_create(c.legion_predicate_true(), 0, 0) - - if redop is None: - src_groups = [src_field_names] - dst_groups = [dst_field_names] - add_dst_requirement = c.legion_copy_launcher_add_dst_region_requirement_logical_region - else: - src_groups = zip(src_field_names) - dst_groups = zip(dst_field_names) - add_dst_requirement = c.legion_copy_launcher_add_dst_region_requirement_logical_region_reduction - - for idx, group in enumerate(src_groups): - c.legion_copy_launcher_add_src_region_requirement_logical_region( - launcher, - src_region.raw_value(), - R._legion_privilege(), 0, # EXCLUSIVE - src_region.parent.raw_value() if src_region.parent is not None else src_region.raw_value(), - 0, False) - for src_field_name in group: - src_field_id = src_region.fspace.field_ids[src_field_name] - c.legion_copy_launcher_add_src_field(launcher, idx, src_field_id, True) - - for idx, group in enumerate(dst_groups): - if redop is None: - dst_privilege = RW._legion_privilege() - else: - dst_field_type = dst_region.fspace.field_types[group[0]] - dst_privilege = Reduce(redop, [group[0]])._legion_redop_id(dst_field_type) - add_dst_requirement( - launcher, - dst_region.raw_value(), - dst_privilege, 0, # EXCLUSIVE - dst_region.parent.raw_value() if dst_region.parent is not None else dst_region.raw_value(), - 0, False) - for dst_field_name in group: - dst_field_id = dst_region.fspace.field_ids[dst_field_name] - c.legion_copy_launcher_add_dst_field(launcher, idx, dst_field_id, True) - - c.legion_copy_launcher_execute(_my.ctx.runtime, _my.ctx.context, launcher) - - c.legion_copy_launcher_destroy(launcher) - -@contextlib.contextmanager -def attach_hdf5(region, filename, field_map, mode, restricted=True, mapped=False): - assert(isinstance(region, Region)) - - assert(isinstance(filename, basestring)) - filename = filename.encode('utf-8') - - raw_field_map = c.legion_field_map_create() - encoded_values = [] # make sure these don't get deleted before the launcher - for field_name, value in field_map.items(): - encoded_value = value.encode('utf-8') - encoded_values.append(encoded_value) - c.legion_field_map_insert(raw_field_map, region.fspace.field_ids[field_name], encoded_value) - region._clear_instance(field_name) - - assert(isinstance(mode, FileMode)) - - launcher = c.legion_attach_launcher_create( - region.raw_value(), - region.parent.raw_value() if region.parent is not None else region.raw_value(), - EXTERNAL_HDF5_FILE) - - c.legion_attach_launcher_attach_hdf5(launcher, filename, raw_field_map, mode.value) - c.legion_attach_launcher_set_restricted(launcher, restricted) - c.legion_attach_launcher_set_mapped(launcher, mapped) - - instance = c.legion_attach_launcher_execute( - _my.ctx.runtime, _my.ctx.context, launcher) - - c.legion_attach_launcher_destroy(launcher) - c.legion_field_map_destroy(raw_field_map) - - yield - - c.legion_detach_external_resource( - _my.ctx.runtime, _my.ctx.context, instance) - -@contextlib.contextmanager -def acquire(region, field_names): - assert(isinstance(region, Region)) - - launcher = c.legion_acquire_launcher_create( - region.raw_value(), - region.parent.raw_value() if region.parent is not None else region.raw_value(), - c.legion_predicate_true(), 0, 0) - - for field_name in field_names: - c.legion_acquire_launcher_add_field(launcher, region.fspace.field_ids[field_name]) - - c.legion_acquire_launcher_execute(_my.ctx.runtime, _my.ctx.context, launcher) - c.legion_acquire_launcher_destroy(launcher) - - yield - - launcher = c.legion_release_launcher_create( - region.raw_value(), - region.parent.raw_value() if region.parent is not None else region.raw_value(), - c.legion_predicate_true(), 0, 0) - - for field_name in field_names: - c.legion_release_launcher_add_field(launcher, region.fspace.field_ids[field_name]) - - c.legion_release_launcher_execute(_my.ctx.runtime, _my.ctx.context, launcher) - c.legion_release_launcher_destroy(launcher) - -# Hack: Can't pickle static methods. -def _Ipartition_unpickle(tid, id, type_tag, parent, color_space): - handle = ffi.new('legion_index_partition_t *') - handle[0].tid = tid - handle[0].id = id - handle[0].type_tag = type_tag - - return Ipartition(handle[0], parent, color_space) - -class Ipartition(object): - __slots__ = ['handle', 'parent', 'color_space'] - - # Make this speak the Type interface - numpy_type = None - cffi_type = 'legion_index_partition_t' - size = ffi.sizeof(cffi_type) - - def __init__(self, handle, parent, color_space): - # Important: Copy handle. Do NOT assume ownership. - self.handle = ffi.new('legion_index_partition_t *', handle) - self.parent = parent - self.color_space = color_space - - def __reduce__(self): - return (_Ipartition_unpickle, - (self.handle[0].tid, self.handle[0].id, self.handle[0].type_tag, self.parent, self.color_space)) - - def __getitem__(self, point): - if isinstance(point, SymbolicExpr): - return SymbolicIndexAccess(self, point) - point = DomainPoint.coerce(point) - subspace = c.legion_index_partition_get_index_subspace_domain_point( - _my.ctx.runtime, self.handle[0], point.raw_value()) - return Ispace(None, _handle=subspace) - - def __iter__(self): - for point in self.color_space: - yield self[point] - - @staticmethod - def equal(ispace, color_space, granularity=1, color=AUTO_GENERATE_ID): - assert isinstance(ispace, Ispace) - color_space = Ispace.coerce(color_space) - handle = c.legion_index_partition_create_equal( - _my.ctx.runtime, _my.ctx.context, - ispace.raw_value(), color_space.raw_value(), granularity, color) - return Ipartition(handle, ispace, color_space) - - @staticmethod - def by_field(region, field, color_space, color=AUTO_GENERATE_ID): - assert isinstance(region, Region) - color_space = Ispace.coerce(color_space) - handle = c.legion_index_partition_create_by_field( - _my.ctx.runtime, _my.ctx.context, - region.raw_value(), - region.parent.raw_value() if region.parent is not None else region.raw_value(), - region.fspace.field_ids[field], - color_space.raw_value(), color, 0, 0, disjoint.value) - return Ipartition(handle, region.ispace, color_space) - - @staticmethod - def image(ispace, projection, field, color_space, - part_kind=compute, color=AUTO_GENERATE_ID): - assert isinstance(ispace, Ispace) - assert isinstance(projection, Partition) - assert isinstance(part_kind, Disjointness) - color_space = Ispace.coerce(color_space) - parent = projection.parent - if is_rect_type(parent.fspace.field_types[field]): - create_by_image = c.legion_index_partition_create_by_image_range - else: - create_by_image = c.legion_index_partition_create_by_image - handle = create_by_image( - _my.ctx.runtime, _my.ctx.context, - ispace.raw_value(), projection.raw_value(), - parent.parent.raw_value() if parent.parent is not None else parent.raw_value(), - parent.fspace.field_ids[field], - color_space.raw_value(), part_kind.value, color, 0, 0) - return Ipartition(handle, parent.ispace, color_space) - - @staticmethod - def preimage(projection, region, field, color_space, - part_kind=compute, color=AUTO_GENERATE_ID): - assert isinstance(projection, Ipartition) - assert isinstance(region, Region) - assert isinstance(part_kind, Disjointness) - color_space = Ispace.coerce(color_space) - if is_rect_type(region.fspace.field_types[field]): - create_by_preimage = c.legion_index_partition_create_by_preimage_range - else: - create_by_preimage = c.legion_index_partition_create_by_preimage - handle = create_by_preimage( - _my.ctx.runtime, _my.ctx.context, - projection.raw_value(), region.raw_value(), - region.parent.raw_value() if region.parent is not None else region.raw_value(), - region.fspace.field_ids[field], - color_space.raw_value(), part_kind.value, color, 0, 0) - return Ipartition(handle, region.ispace, color_space) - - @staticmethod - def restrict(ispace, color_space, transform, extent, - part_kind=compute, color=AUTO_GENERATE_ID): - assert isinstance(ispace, Ispace) - assert isinstance(part_kind, Disjointness) - color_space = Ispace.coerce(color_space) - transform = DomainTransform.coerce(transform) - extent = Domain.coerce(extent) - handle = c.legion_index_partition_create_by_restriction( - _my.ctx.runtime, _my.ctx.context, - ispace.raw_value(), color_space.raw_value(), transform.raw_value(), extent.raw_value(), part_kind.value, color) - return Ipartition(handle, ispace, color_space) - - @staticmethod - def pending(ispace, color_space, - part_kind=compute, color=AUTO_GENERATE_ID): - assert isinstance(ispace, Ispace) - assert isinstance(part_kind, Disjointness) - color_space = Ispace.coerce(color_space) - handle = c.legion_index_partition_create_pending_partition( - _my.ctx.runtime, _my.ctx.context, - ispace.raw_value(), color_space.raw_value(), part_kind.value, color) - return Ipartition(handle, ispace, color_space) - - # The following methods are for pending partitions only: - def union(self, color, ispaces): - color = DomainPoint.coerce(color) - - handles = ffi.new('legion_index_space_t[]', [ispace.raw_value() for ispace in ispaces]) - c.legion_index_partition_create_index_space_union_spaces( - _my.ctx.runtime, _my.ctx.context, - self.handle[0], color.raw_value(), handles, len(ispaces)) - - def destroy(self): - # This is not something you want to have happen in a - # destructor, since partitions may outlive the lifetime of the handle. - c.legion_index_partition_destroy( - _my.ctx.runtime, _my.ctx.context, self.handle[0]) - # Clear out references. Technically unnecessary but avoids abuse. - del self.handle - del self.parent - del self.color_space - - def raw_value(self): - return self.handle[0] - -# Hack: Can't pickle static methods. -def _Partition_unpickle(parent, ipartition): - handle = ffi.new('legion_logical_partition_t *') - handle[0].tree_id = parent.raw_value().tree_id - handle[0].index_partition = ipartition.raw_value() - handle[0].field_space = parent.fspace.raw_value() - - return Partition(parent, ipartition, _handle=handle[0]) - -class Partition(object): - __slots__ = ['handle', 'parent', 'ipartition'] - - # Make this speak the Type interface - numpy_type = None - cffi_type = 'legion_logical_partition_t' - size = ffi.sizeof(cffi_type) - - def __init__(self, parent, ipartition, **kwargs): - def parse_kwargs(_handle=None): - return _handle - handle = parse_kwargs(**kwargs) - - if handle is None: - assert isinstance(parent, Region) - assert isinstance(ipartition, Ipartition) - handle = c.legion_logical_partition_create( - _my.ctx.runtime, _my.ctx.context, parent.raw_value(), ipartition.raw_value()) - - # Important: Copy handle. Do NOT assume ownership. - assert handle is not None - self.handle = ffi.new('legion_logical_partition_t *', handle) - self.parent = parent - self.ipartition = ipartition - - def __reduce__(self): - return (_Partition_unpickle, - (self.parent, - self.ipartition)) - - def __getitem__(self, point): - if isinstance(point, SymbolicExpr): - return SymbolicIndexAccess(self, point) - point = DomainPoint.coerce(point) - subspace = self.ipartition[point] - subregion = c.legion_logical_partition_get_logical_subregion_by_color_domain_point( - _my.ctx.runtime, self.handle[0], point.raw_value()) - return Region(subspace, self.parent.fspace, _handle=subregion, - _parent=self.parent.parent if self.parent.parent is not None else self.parent) - - def __iter__(self): - for point in self.color_space: - yield self[point] - - @property - def color_space(self): - return self.ipartition.color_space - - @staticmethod - def equal(region, color_space, granularity=1, color=AUTO_GENERATE_ID): - assert isinstance(region, Region) - ipartition = Ipartition.equal(region.ispace, color_space, granularity, color) - return Partition(region, ipartition) - - @staticmethod - def by_field(region, field, color_space, color=AUTO_GENERATE_ID): - assert isinstance(region, Region) - ipartition = Ipartition.by_field( - region, field, color_space, color) - return Partition(region, ipartition) - - @staticmethod - def image(region, projection, field, color_space, - part_kind=compute, color=AUTO_GENERATE_ID): - assert isinstance(region, Region) - ipartition = Ipartition.image( - region.ispace, projection, field, color_space, part_kind, color) - return Partition(region, ipartition) - - @staticmethod - def preimage(projection, region, field, color_space, - part_kind=compute, color=AUTO_GENERATE_ID): - assert isinstance(projection, Partition) - ipartition = Ipartition.preimage( - projection.ipartition, region, field, color_space, part_kind, color) - return Partition(region, ipartition) - - @staticmethod - def restrict(region, color_space, transform, extent, - part_kind=compute, color=AUTO_GENERATE_ID): - assert isinstance(region, Region) - ipartition = Ipartition.restrict( - region.ispace, color_space, transform, extent, part_kind, color) - return Partition(region, ipartition) - - @staticmethod - def pending(region, color_space, part_kind=compute, color=AUTO_GENERATE_ID): - assert isinstance(region, Region) - ipartition = Ipartition.pending( - region.ispace, color_space, part_kind, color) - return Partition(region, ipartition) - - def union(self, color, regions): - ispaces = [region.ispace for region in regions] - self.ipartition.union(color, ispaces) - - def destroy(self): - # This is not something you want to have happen in a - # destructor, since partitions may outlive the lifetime of the handle. - c.legion_logical_partition_destroy( - _my.ctx.runtime, _my.ctx.context, self.handle[0]) - # Clear out references. Technically unnecessary but avoids abuse. - del self.handle - del self.parent - del self.ipartition - - def raw_value(self): - return self.handle[0] - -def define_regent_argument_struct(task_id, argument_types, privileges, return_type, arguments): - if argument_types is None: - raise Exception('Arguments must be typed in extern Regent tasks') - - struct_name = 'task_args_%s' % task_id - - n_fields = int(math.ceil(len(argument_types)/64.)) - - fields = ['uint64_t %s[%s];' % ('__map', n_fields)] - for i, arg_type in enumerate(argument_types): - arg_name = '__arg_%s' % i - fields.append('%s %s;' % (arg_type.cffi_type, arg_name)) - for i, arg in enumerate(arguments): - if isinstance(arg, Region): - fields.append('legion_field_id_t __arg_%s_fields[%s];' % (i, len(arg.fspace.field_types))) - - struct = 'typedef struct %s { %s } %s;' % (struct_name, ' '.join(fields), struct_name) - ffi.cdef(struct) - - return struct_name - -class ExternTask(object): - __slots__ = ['argument_types', 'privileges', 'return_type', - 'calling_convention', 'task_id', '_argument_struct'] - - def __init__(self, task_id, argument_types=None, privileges=None, - return_type=void, calling_convention=None): - self.argument_types = argument_types - self.privileges = privileges - self.return_type = return_type - self.calling_convention = calling_convention - assert isinstance(task_id, int) - self.task_id = task_id - self._argument_struct = None - - def argument_struct(self, args): - if self.calling_convention == 'regent' and self._argument_struct is None: - self._argument_struct = define_regent_argument_struct( - self.task_id, self.argument_types, self.privileges, self.return_type, args) - return self._argument_struct - - def __call__(self, *args): - return self.spawn_task(*args) - - def spawn_task(self, *args, **kwargs): - if _my.ctx.current_launch: - return _my.ctx.current_launch.spawn_task(self, *args, **kwargs) - return TaskLaunch().spawn_task(self, *args, **kwargs) - -def extern_task(**kwargs): - return ExternTask(**kwargs) - -class ExternTaskWrapper(object): - # Note: Can't use __slots__ for this class because __qualname__ - # conflicts with the class variable. - def __init__(self, thunk, name): - self.thunk = thunk - self.__name__ = name - self.__qualname__ = name - def __call__(self, *args, **kwargs): - f = self.thunk(*args, **kwargs) - if f.value_type != void: - return f.get() - -_next_wrapper_id = 1000 -def extern_task_wrapper(privileges=None, return_type=void, **kwargs): - global _next_wrapper_id - extern = extern_task(privileges=privileges, return_type=return_type, **kwargs) - wrapper_name = str('wrapper_task_%s' % _next_wrapper_id) - _next_wrapper_id += 1 - wrapper = ExternTaskWrapper(extern, wrapper_name) - task_wrapper = task(wrapper, privileges=privileges, return_type=return_type, inner=True) - setattr(sys.modules[__name__], wrapper_name, task_wrapper) - return task_wrapper - -def get_qualname(fn): - # Python >= 3.3 only - try: - return fn.__qualname__.split('.') - except AttributeError: - pass - - # Python < 3.3 - try: - import qualname - return qualname.qualname(fn).split('.') - except ImportError: - pass - - # Hack: Issue error if we're wrapping a class method and failed to - # get the qualname - import inspect - context = [x[0].f_code.co_name for x in inspect.stack() - if '__module__' in x[0].f_code.co_names and - inspect.getmodule(x[0].f_code).__name__ != __name__] - if len(context) > 0: - raise Exception('To use a task defined in a class, please upgrade to Python >= 3.3 or install qualname (e.g. pip install qualname)') - - return [fn.__name__] - -def _postprocess(arg, point): - if hasattr(arg, '_legion_postprocess_task_argument'): - return arg._legion_postprocess_task_argument(point) - return arg - -class Task (object): - __slots__ = ['body', 'privileges', 'return_type', - 'leaf', 'inner', 'idempotent', 'replicable', - 'calling_convention', 'argument_struct', - 'task_id', 'registered'] - - def __init__(self, body, privileges=None, return_type=None, - leaf=False, inner=False, idempotent=False, replicable=False, - register=True, task_id=None, top_level=False): - self.body = body - self.privileges = privileges - self.return_type = return_type - self.leaf = bool(leaf) - self.inner = bool(inner) - self.idempotent = bool(idempotent) - self.replicable = bool(replicable) - self.calling_convention = 'python' - self.argument_struct = None - self.task_id = None - if register: - self.register(task_id, top_level) - - def __call__(self, *args, **kwargs): - # Hack: This entrypoint needs to be able to handle both being - # called in user code (to launch a task) and as the task - # wrapper when the task itself executes. Unfortunately isn't a - # good way to disentangle these. Detect if we're in the task - # wrapper case by checking the number and types of arguments. - if len(args) == 3 and \ - isinstance(args[0], bytearray) and \ - isinstance(args[1], bytearray) and \ - isinstance(args[2], long): - return self.execute_task(*args, **kwargs) - else: - return self.spawn_task(*args, **kwargs) - - def spawn_task(self, *args, **kwargs): - if _my.ctx.current_launch: - return _my.ctx.current_launch.spawn_task(self, *args, **kwargs) - return TaskLaunch().spawn_task(self, *args, **kwargs) - - def execute_task(self, raw_args, user_data, proc): - raw_arg_ptr = ffi.new('char[]', bytes(raw_args)) - raw_arg_size = len(raw_args) - - # Execute preamble to obtain Legion API context. - task = ffi.new('legion_task_t *') - raw_regions = ffi.new('legion_physical_region_t **') - num_regions = ffi.new('unsigned *') - context = ffi.new('legion_context_t *') - runtime = ffi.new('legion_runtime_t *') - c.legion_task_preamble( - raw_arg_ptr, raw_arg_size, proc, - task, raw_regions, num_regions, context, runtime) - - # Decode arguments from Pickle format. - arg_ptr = ffi.cast('char *', c.legion_task_get_args(task[0])) - arg_size = c.legion_task_get_arglen(task[0]) - if c.legion_task_get_is_index_space(task[0]) and arg_size == 0: - arg_ptr = ffi.cast('char *', c.legion_task_get_local_args(task[0])) - arg_size = c.legion_task_get_local_arglen(task[0]) - - if arg_size > 0 and c.legion_task_get_depth(task[0]) > 0: - args = pickle.loads(ffi.unpack(arg_ptr, arg_size)) - else: - args = () - - # Unpack regions. - regions = [] - for i in xrange(num_regions[0]): - regions.append(raw_regions[0][i]) - - # Build context. - ctx = Context(context, runtime, task, regions) - - # Ensure that we're not getting tangled up in another - # thread. There should be exactly one thread per task. - try: - _my.ctx - except AttributeError: - pass - else: - raise Exception('thread-local context already set') - - # Store context in thread-local storage. - _my.ctx = ctx - - # Postprocess arguments. - point = DomainPoint(None, _handle=c.legion_task_get_index_point(task[0])) - args = tuple(_postprocess(arg, point) for arg in args) - - # Unpack physical regions. - if self.privileges is not None: - req = 0 - for i, arg in zip(range(len(args)), args): - if isinstance(arg, Region): - assert i < len(self.privileges) - groups = self.privileges[i]._legion_grouped_privileges(arg.fspace) - for priv, _, _, fields in groups: - assert req < num_regions[0] - instance = raw_regions[0][req] - req += 1 - - for field in fields: - arg._set_instance(field, instance, priv) - assert req == num_regions[0] - - # Execute task body. - result = self.body(*args) - - # Mark any remaining objects as escaped. - for ref in ctx.owned_objects: - obj = ref() - if obj is not None: - obj.escaped = True - - # Encode result. - if not self.return_type: - result_str = pickle.dumps(result, protocol=_pickle_version) - result_size = len(result_str) - result_ptr = ffi.new('char[]', result_size) - ffi.buffer(result_ptr, result_size)[:] = result_str - else: - if self.return_type.size > 0: - result_ptr = ffi.new(ffi.getctype(self.return_type.cffi_type, '*'), result) - else: - result_ptr = ffi.NULL - result_size = self.return_type.size - - # Execute postamble. - c.legion_task_postamble(runtime[0], context[0], result_ptr, result_size) - - # Clear thread-local storage. - del _my.ctx - - def register(self, task_id, top_level_task): - assert(self.task_id is None) - - if not task_id: - global next_legion_task_id - task_id = next_legion_task_id - next_legion_task_id += 1 - # If we ever hit this then we need to allocate more task IDs - assert task_id < max_legion_task_id - - execution_constraints = c.legion_execution_constraint_set_create() - c.legion_execution_constraint_set_add_processor_constraint( - execution_constraints, c.PY_PROC) - - layout_constraints = c.legion_task_layout_constraint_set_create() - # FIXME: Add layout constraints - - options = ffi.new('legion_task_config_options_t *') - options[0].leaf = self.leaf - options[0].inner = self.inner - options[0].idempotent = self.idempotent - options[0].replicable = self.replicable - - qualname = get_qualname(self.body) - task_name = ('%s.%s' % (self.body.__module__, '.'.join(qualname))) - - c_qualname_comps = [ffi.new('char []', comp.encode('utf-8')) for comp in qualname] - c_qualname = ffi.new('char *[]', c_qualname_comps) - - global global_task_registration_barrier - if global_task_registration_barrier is not None: - c.legion_phase_barrier_arrive(_my.ctx.runtime, _my.ctx.context, global_task_registration_barrier, 1) - global_task_registration_barrier = c.legion_phase_barrier_advance(_my.ctx.runtime, _my.ctx.context, global_task_registration_barrier) - c.legion_runtime_enable_scheduler_lock() - c.legion_phase_barrier_wait(_my.ctx.runtime, _my.ctx.context, global_task_registration_barrier) - # Need to hold this through the end of registration. - # c.legion_runtime_disable_scheduler_lock() - - c.legion_runtime_register_task_variant_python_source_qualname( - c.legion_runtime_get_runtime(), - task_id, - task_name.encode('utf-8'), - True, # self.replicable or not is_script, # Global - execution_constraints, - layout_constraints, - options[0], - self.body.__module__.encode('utf-8'), - c_qualname, - len(qualname), - ffi.NULL, - 0) - # If we're the top-level task then tell the runtime about our ID - if top_level_task: - c.legion_runtime_set_top_level_task_id(task_id) - if global_task_registration_barrier is not None: - c.legion_phase_barrier_arrive(_my.ctx.runtime, _my.ctx.context, global_task_registration_barrier, 1) - global_task_registration_barrier = c.legion_phase_barrier_advance(_my.ctx.runtime, _my.ctx.context, global_task_registration_barrier) - # c.legion_runtime_enable_scheduler_lock() - c.legion_phase_barrier_wait(_my.ctx.runtime, _my.ctx.context, global_task_registration_barrier) - c.legion_runtime_disable_scheduler_lock() - - c.legion_execution_constraint_set_destroy(execution_constraints) - c.legion_task_layout_constraint_set_destroy(layout_constraints) - - self.task_id = task_id - return self - -def task(body=None, **kwargs): - if body is None: - return lambda body: task(body, **kwargs) - return Task(body, **kwargs) - -class _TaskLauncher(object): - __slots__ = ['task'] - - def __init__(self, task): - self.task = task - - def preprocess_args(self, args): - return [ - arg._legion_preprocess_task_argument() - if hasattr(arg, '_legion_preprocess_task_argument') else arg - for arg in args] - - def gather_futures(self, args): - normal = [] - futures = [] - for arg in args: - if isinstance(arg, Future): - arg = Future(arg, argument_number=len(futures)) - futures.append(arg) - normal.append(arg) - return normal, futures - - def encode_args(self, args): - task_args = ffi.new('legion_task_argument_t *') - task_args_buffer = None - if self.task.calling_convention == 'python': - arg_str = pickle.dumps(args, protocol=_pickle_version) - task_args_buffer = ffi.new('char[]', arg_str) - task_args[0].args = task_args_buffer - task_args[0].arglen = len(arg_str) - elif self.task.calling_convention == 'regent': - arg_struct = self.task.argument_struct(args) - task_args_buffer = ffi.new('%s*' % arg_struct) - # Note: ffi.new returns zeroed memory - for i, arg in enumerate(args): - if isinstance(arg, Future): - getattr(task_args_buffer, '__map')[i // 64] |= 1 << (i % 64) - for i, arg in enumerate(args): - if not isinstance(arg, Future): - arg_name = '__arg_%s' % i - arg_value = arg - if hasattr(arg, 'handle') and not isinstance(arg, DomainPoint): - arg_value = arg.handle[0] - setattr(task_args_buffer, arg_name, arg_value) - for i, arg in enumerate(args): - if isinstance(arg, Region): - arg_name = '__arg_%s_fields' % i - arg_slot = getattr(task_args_buffer, arg_name) - for j, field_id in enumerate(arg.fspace.field_ids.values()): - arg_slot[j] = field_id - task_args[0].args = task_args_buffer - task_args[0].arglen = ffi.sizeof(arg_struct) - else: - # FIXME: External tasks need a dedicated calling - # convention to permit the passing of task arguments. - task_args[0].args = ffi.NULL - task_args[0].arglen = 0 - # WARNING: Need to return the interior buffer or else it will be GC'd - return task_args, task_args_buffer - - def attach_region_requirements(self, launcher, args, is_index_launch): - if is_index_launch: - def add_region_normal(launcher, handle, *args): - return c.legion_index_launcher_add_region_requirement_logical_region( - launcher, handle, 0, # projection - *args) - def add_region_reduction(launcher, handle, *args): - return c.legion_index_launcher_add_region_requirement_logical_region_reduction( - launcher, handle, 0, # projection - *args) - add_partition_normal = c.legion_index_launcher_add_region_requirement_logical_partition - add_partition_reduction = c.legion_index_launcher_add_region_requirement_logical_partition_reduction - add_field = c.legion_index_launcher_add_field - else: - add_region_normal = c.legion_task_launcher_add_region_requirement_logical_region - add_region_reduction = c.legion_task_launcher_add_region_requirement_logical_region_reduction - add_field = c.legion_task_launcher_add_field - - for i, arg in zip(range(len(args)), args): - if isinstance(arg, Region) or (isinstance(arg, SymbolicExpr) and arg.is_region()): - if self.task.privileges is None or i >= len(self.task.privileges): - raise Exception('Privileges are required on all Region arguments') - groups = self.task.privileges[i]._legion_grouped_privileges(arg.fspace) - if isinstance(arg, Region): - parent = arg.parent if arg.parent is not None else arg - for _, priv, redop, fields in groups: - if redop is None: - req = add_region_normal( - launcher, arg.raw_value(), - priv, 0, # EXCLUSIVE - parent.raw_value(), 0, False) - else: - req = add_region_reduction( - launcher, arg.raw_value(), - redop, 0, # EXCLUSIVE - parent.raw_value(), 0, False) - for field in fields: - add_field( - launcher, req, arg.fspace.field_ids[field], True) - elif isinstance(arg, SymbolicExpr): - # FIXME: Support non-trivial projection functors - assert isinstance(arg, SymbolicIndexAccess) and (isinstance(arg.index, SymbolicLoopIndex) or isinstance(arg.index, ConcreteLoopIndex)) - proj_id = 0 - - parent = arg.parent if arg.parent is not None else arg - parent = parent.parent if parent.parent is not None else parent - for _, priv, redop, fields in groups: - if redop is None: - req = add_partition_normal( - launcher, arg.raw_value(), proj_id, - priv, 0, # EXCLUSIVE - parent.raw_value(), 0, False) - else: - req = add_partition_reduction( - launcher, arg.raw_value(), proj_id, - redop, 0, # EXCLUSIVE - parent.raw_value(), 0, False) - for field in fields: - add_field( - launcher, req, arg.fspace.field_ids[field], True) - elif self.task.privileges is not None and i < len(self.task.privileges) and self.task.privileges[i]: - raise TypeError('Privileges can only be specified for Region arguments, got %s' % type(arg)) - - def spawn_task(self, *args, **kwargs): - # Hack: workaround for Python 2 not having keyword-only arguments - def validate_spawn_task_args(point=None, mapper=0, tag=0): - return point, mapper, tag - point, mapper, tag = validate_spawn_task_args(**kwargs) - - assert(isinstance(_my.ctx, Context)) - - args, futures = self.gather_futures(args) - args = self.preprocess_args(args) - task_args, task_args_root = self.encode_args(args) - - # Construct the task launcher. - launcher = c.legion_task_launcher_create( - self.task.task_id, task_args[0], c.legion_predicate_true(), mapper, tag) - if point is not None: - point = DomainPoint.coerce(point) - c.legion_task_launcher_set_point(launcher, point.raw_value()) - self.attach_region_requirements(launcher, args, False) - for i, arg in zip(range(len(args)), args): - if self.task.privileges is not None and i < len(self.task.privileges) and self.task.privileges[i] and not isinstance(arg, Region): - raise TypeError('Privileges can only be specified for Region arguments, got %s' % type(arg)) - if isinstance(arg, Region): - pass # Already attached above - elif isinstance(arg, Future): - c.legion_task_launcher_add_future(launcher, arg.handle) - elif self.task.calling_convention is None: - # FIXME: Task arguments aren't being encoded AT ALL; - # at least throw an exception so that the user knows - raise Exception('External tasks do not support non-region arguments') - - # Launch the task. - if _my.ctx.current_launch is not None: - return _my.ctx.current_launch.attach_task_launcher( - launcher, point, root=task_args_root) - - result = c.legion_task_launcher_execute( - _my.ctx.runtime, _my.ctx.context, launcher) - c.legion_task_launcher_destroy(launcher) - - # Build future of result. - future = Future.from_cdata(result, value_type=self.task.return_type) - c.legion_future_destroy(result) - return future - -class _IndexLauncher(_TaskLauncher): - __slots__ = ['task', 'domain', 'mapper', 'tag', - 'global_args', 'local_args', 'region_args', 'future_args', - 'reduction_op', 'future_map'] - - def __init__(self, task, domain, mapper, tag): - super(_IndexLauncher, self).__init__(task) - self.domain = domain - self.mapper = mapper - self.tag = tag - self.global_args = None - self.local_args = c.legion_argument_map_create() - self.region_args = None - self.future_args = [] - self.reduction_op = None - self.future_map = None - - def __del__(self): - c.legion_argument_map_destroy(self.local_args) - - def spawn_task(self, *args, **kwargs): - raise Exception('IndexLaunch does not support spawn_task') - - def attach_local_args(self, index, *args): - task_args, _ = self.encode_args(self.preprocess_args(args)) - c.legion_argument_map_set_point( - self.local_args, index.value.raw_value(), task_args[0], False) - - def attach_global_args(self, *args): - assert self.global_args is None - self.global_args = args - - def attach_region_args(self, *args): - self.region_args = args - - def attach_future_args(self, *args): - self.future_args = args - - def set_reduction_op(self, op): - self.reduction_op = op - - def launch(self): - # Encode global args (if any). - if self.global_args is not None: - global_args, global_args_root = self.encode_args(self.preprocess_args(self.global_args)) - else: - global_args = ffi.new('legion_task_argument_t *') - global_args[0].args = ffi.NULL - global_args[0].arglen = 0 - global_args_root = None - - # Construct the task launcher. - launcher = c.legion_index_launcher_create( - self.task.task_id, self.domain.raw_value(), - global_args[0], self.local_args, - c.legion_predicate_true(), False, self.mapper, self.tag) - - assert (self.global_args is not None) != (self.region_args is not None) - if self.global_args is not None: - self.attach_region_requirements(launcher, self.global_args, True) - if self.region_args is not None: - self.attach_region_requirements(launcher, self.region_args, True) - - for arg in self.future_args: - c.legion_index_launcher_add_future(launcher, arg.handle) - - # Launch the task. - if _my.ctx.current_launch is not None: - return _my.ctx.current_launch.attach_index_launcher( - launcher, root=global_args_root) - - launch = c.legion_index_launcher_execute - redop = [] - if self.reduction_op is not None: - assert self.task.return_type is not None - launch = c.legion_index_launcher_execute_reduction - redop = [_redop_ids[self.reduction_op][self.task.return_type]] - - result = launch( - _my.ctx.runtime, _my.ctx.context, launcher, *redop) - c.legion_index_launcher_destroy(launcher) - - # Build future (map) of result. - if self.reduction_op is not None: - self.future_map = Future.from_cdata(result, value_type=self.task.return_type) - c.legion_future_destroy(result) - else: - self.future_map = FutureMap(result, value_type=self.task.return_type) - c.legion_future_map_destroy(result) - -class _MustEpochLauncher(object): - __slots__ = ['domain', 'launcher', 'roots', 'has_sublaunchers'] - - def __init__(self, domain=None): - self.domain = domain - self.launcher = c.legion_must_epoch_launcher_create(0, 0) - if self.domain is not None: - c.legion_must_epoch_launcher_set_launch_domain( - self.launcher, self.domain.raw_value()) - self.roots = [] - self.has_sublaunchers = False - - def __del__(self): - c.legion_must_epoch_launcher_destroy(self.launcher) - - def spawn_task(self, *args, **kwargs): - raise Exception('MustEpochLaunch does not support spawn_task') - - def attach_task_launcher(self, task_launcher, point, root=None): - if point is None: - raise Exception('MustEpochLauncher requires a point for each task') - if root is not None: - self.roots.append(root) - c.legion_must_epoch_launcher_add_single_task( - self.launcher, point.raw_value(), task_launcher) - self.has_sublaunchers = True - - def attach_index_launcher(self, index_launcher, root=None): - if root is not None: - self.roots.append(root) - c.legion_must_epoch_launcher_add_index_task( - self.launcher, index_launcher) - self.has_sublaunchers = True - - def launch(self): - if not self.has_sublaunchers: - raise Exception('MustEpochLaunch requires at least one point task to be executed') - result = c.legion_must_epoch_launcher_execute( - _my.ctx.runtime, _my.ctx.context, self.launcher) - c.legion_future_map_destroy(result) - -class TaskLaunch(object): - __slots__ = [] - def spawn_task(self, task, *args, **kwargs): - launcher = _TaskLauncher(task=task) - return launcher.spawn_task(*args, **kwargs) - -class _FuturePoint(object): - __slots__ = ['launcher', 'point', 'future'] - def __init__(self, launcher, point): - self.launcher = launcher - self.point = point - self.future = None - def get(self): - if self.future is not None: - return self.future.get() - - if self.launcher.future_map is None: - raise Exception('Cannot retrieve a future from an index launch until the launch is complete') - - self.future = self.launcher.future_map[self.point] - - # Clear launcher and point - del self.launcher - del self.point - - return self.future.get() - -class SymbolicExpr(object): - def is_region(self): - return False - -class SymbolicIndexAccess(SymbolicExpr): - __slots__ = ['value', 'index'] - def __init__(self, value, index): - self.value = value - self.index = index - def __str__(self): - return '%s[%s]' % (self.value, self.index) - def __repr__(self): - return '%s[%s]' % (self.value, self.index) - def _legion_preprocess_task_argument(self): - if isinstance(self.index, ConcreteLoopIndex): - return self.value[self.index._legion_preprocess_task_argument()] - return self - def _legion_postprocess_task_argument(self, point): - result = _postprocess(self.value, point)[_postprocess(self.index, point)] - # FIXME: Clear parent field of regions being used as projection requirements - if isinstance(result, Region): - result.parent = None - return result - def is_region(self): - return isinstance(self.value, Partition) - @property - def parent(self): - if self.is_region(): - return self.value.parent - assert False - @property - def fspace(self): - if self.is_region(): - return self.value.parent.fspace - assert False - def raw_value(self): - if self.is_region(): - return self.value.raw_value() - assert False - -class SymbolicLoopIndex(SymbolicExpr): - __slots__ = ['name'] - def __init__(self, name): - self.name = name - def __str__(self): - return self.name - def __repr__(self): - return self.name - def _legion_postprocess_task_argument(self, point): - return point - -ID = SymbolicLoopIndex('ID') - -class ConcreteLoopIndex(SymbolicExpr): - __slots__ = ['value'] - def __init__(self, value): - self.value = value - def __int__(self): - return self.value.__int__() - def __index__(self): - return self.value.__index__() - def __str__(self): - return str(self.value) - def __repr__(self): - return repr(self.value) - def _legion_preprocess_task_argument(self): - return self.value - -def index_launch(domain, task, *args, **kwargs): - def parse_kwargs(reduce=None, mapper=0, tag=0): - return reduce, mapper, tag - reduce, mapper, tag = parse_kwargs(**kwargs) - - if isinstance(domain, Domain): - domain = domain - elif isinstance(domain, Ispace): - domain = domain.domain - else: - domain = Domain(domain) - launcher = _IndexLauncher(task=task, domain=domain, mapper=mapper, tag=tag) - args, futures = launcher.gather_futures(args) - launcher.attach_global_args(*args) - launcher.attach_future_args(*futures) - launcher.set_reduction_op(reduce) - launcher.launch() - return launcher.future_map - -class IndexLaunch(object): - __slots__ = ['domain', 'mapper', 'tag', 'launcher', 'point', - 'saved_task', 'saved_args'] - - def __init__(self, domain, **kwargs): - # Hack: workaround for Python 2 not having keyword-only arguments - def validate_spawn_task_args(mapper=0, tag=0): - return mapper, tag - mapper, tag = validate_spawn_task_args(**kwargs) - - if isinstance(domain, Domain): - self.domain = domain - elif isinstance(domain, Ispace): - self.domain = domain.domain - else: - self.domain = Domain(domain) - self.mapper = mapper - self.tag = tag - self.launcher = None - self.point = None - self.saved_task = None - self.saved_args = None - - def __iter__(self): - _my.ctx.begin_launch(self) - self.point = ConcreteLoopIndex(None) - for i in self.domain: - self.point.value = i - yield self.point - _my.ctx.end_launch(self) - self.launch() - - def ensure_launcher(self, task): - if self.launcher is None: - self.launcher = _IndexLauncher( - task=task, domain=self.domain, mapper=self.mapper, tag=self.tag) - - def check_compatibility(self, task, *args): - # The tasks in a launch must conform to the following constraints: - # * Only one task can be launched. - # * The arguments must be compatible: - # * At a given argument position, the value must always - # be a special value, or always not. - # * Special values include: regions and futures. - # * If a region, the value must be symbolic (i.e. able - # to be analyzed as a function of the index expression). - # * If a future, the values must be literally identical - # (i.e. each argument slot in the launch can only - # accept a single future value.) - - if self.saved_task is None: - self.saved_task = task - if task != self.saved_task: - raise Exception('An IndexLaunch may contain only one task launch') - - if self.saved_args is None: - self.saved_args = args - for arg, saved_arg in zip_longest(args, self.saved_args): - # TODO: Add support for region arguments - if isinstance(arg, Region) or isinstance(arg, RegionField): - if arg != saved_arg: - raise Exception('Region argument to IndexLaunch does not match previous value at this position') - elif isinstance(arg, Future): - if arg != saved_arg: - raise Exception('Future argument to IndexLaunch does not match previous value at this position') - - def spawn_task(self, task, *args): - self.ensure_launcher(task) - self.check_compatibility(task, *args) - args, futures = self.launcher.gather_futures(args) - self.launcher.attach_local_args(self.point, *args) - self.launcher.attach_region_args(*args) - self.launcher.attach_future_args(*futures) - return _FuturePoint(self.launcher, self.point.value) - - def launch(self): - self.launcher.launch() - -class MustEpochLaunch(object): - __slots__ = ['domain', 'launcher'] - - def __init__(self, domain=None): - if isinstance(domain, Domain): - self.domain = domain - elif isinstance(domain, Ispace): - self.domain = ispace.domain - elif domain is not None: - self.domain = Domain(domain) - else: - self.domain = None - self.launcher = None - - def __enter__(self): - self.launcher = _MustEpochLauncher(domain=self.domain) - _my.ctx.begin_launch(self) - - def __exit__(self, exc_type, exc_value, tb): - _my.ctx.end_launch(self) - if exc_value is None: - self.launch() - del self.launcher - - def spawn_task(self, *args, **kwargs): - # TODO: Support index launches - TaskLaunch().spawn_task(*args, **kwargs) - - # TODO: Support return values - - def attach_task_launcher(self, *args, **kwargs): - self.launcher.attach_task_launcher(*args, **kwargs) - - def attach_index_launcher(self, *args, **kwargs): - self.launcher.attach_index_launcher(*args, **kwargs) - - def launch(self): - self.launcher.launch() - -def execution_fence(block=False, future=False): - f = Future.from_cdata( - c.legion_runtime_issue_execution_fence(_my.ctx.runtime, _my.ctx.context), - value_type=void) - if block or future: - if block: - f.get() - if future: - return f - -def print_once(*args, **kwargs): - fd = (kwargs['file'] if 'file' in kwargs else sys.stdout).fileno() - message = StringIO() - kwargs['file'] = message - print(*args, **kwargs) - c.legion_runtime_print_once_fd(_my.ctx.runtime, _my.ctx.context, fd, 'w'.encode('utf-8'), message.getvalue().encode('utf-8')) - -class Tunable(object): - # FIXME: Deduplicate this with DefaultMapper::DefaultTunables - NODE_COUNT = 0 - LOCAL_CPUS = 1 - LOCAL_GPUS = 2 - LOCAL_IOS = 3 - LOCAL_OMPS = 4 - LOCAL_PYS = 5 - GLOBAL_CPUS = 6 - GLOBAL_GPUS = 7 - GLOBAL_IOS = 8 - GLOBAL_OMPS = 9 - GLOBAL_PYS = 10 - - @staticmethod - def select(tunable_id): - result = c.legion_runtime_select_tunable_value( - _my.ctx.runtime, _my.ctx.context, tunable_id, 0, 0) - future = Future.from_cdata(result, value_type=uint64) - c.legion_future_destroy(result) - return future - -class Trace(object): - __slots__ = ['trace_id'] - - def __init__(self): - self.trace_id = _my.ctx.next_trace_id - _my.ctx.next_trace_id += 1 - - def __enter__(self): - c.legion_runtime_begin_trace(_my.ctx.runtime, _my.ctx.context, self.trace_id, True) - - def __exit__(self, exc_type, exc_value, tb): - c.legion_runtime_end_trace(_my.ctx.runtime, _my.ctx.context, self.trace_id) - -if is_script: - _my.ctx = Context( - legion_top.top_level.context, - legion_top.top_level.runtime, - legion_top.top_level.task, - []) - - def _cleanup(): - del _my.ctx - - legion_top.cleanup_items.append(_cleanup) - - # FIXME: Really this should be the number of control replicated shards at this level - c.legion_runtime_enable_scheduler_lock() - num_procs = Tunable.select(Tunable.GLOBAL_PYS).get() - c.legion_runtime_disable_scheduler_lock() - - global_task_registration_barrier = c.legion_phase_barrier_create(_my.ctx.runtime, _my.ctx.context, num_procs) -elif is_legion_python: - print('WARNING: Executing Python modules via legion_python has been deprecated.') - print('It is now recommended to run the script directly by passing the path') - print('to legion_python.') - print() +#!/usr/bin/env python + +# Copyright 2020 Stanford University +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import absolute_import, division, print_function, unicode_literals + +try: + import cPickle as pickle +except ImportError: + import pickle +import collections +import contextlib +from io import StringIO +import itertools +import math +import numpy +import os +import re +import subprocess +import sys +import threading +import weakref + +# Python 3.x compatibility: +try: + long # Python 2 +except NameError: + long = int # Python 3 + +try: + basestring # Python 2 +except NameError: + basestring = str # Python 3 + +try: + xrange # Python 2 +except NameError: + xrange = range # Python 3 + +try: + imap = itertools.imap # Python 2 +except: + imap = map # Python 3 + +try: + zip_longest = itertools.izip_longest # Python 2 +except: + zip_longest = itertools.zip_longest # Python 3 + +_pickle_version = pickle.HIGHEST_PROTOCOL # Use latest Pickle protocol + +import legion_top +from legion_cffi import ffi, lib as c + +_max_dim = None +for dim in range(1, 9): + try: + getattr(c, "legion_domain_get_rect_{}d".format(dim)) + except AttributeError: + break + _max_dim = dim +assert _max_dim is not None, "Unable to detect LEGION_MAX_DIM" + +AUTO_GENERATE_ID = c.legion_auto_generate_id() + +# Duplicate enum values from legion_config.h since CFFI isn't smart +# enough to parse them directly. + +EXTERNAL_HDF5_FILE = 1 + +NO_ACCESS = 0x00000000 +READ_PRIV = 0x00000001 +READ_ONLY = 0x00000001 # READ_PRIV +WRITE_PRIV = 0x00000002 +REDUCE_PRIV = 0x00000004 +REDUCE = 0x00000004 # REDUCE_PRIV +READ_WRITE = 0x00000007 # READ_PRIV | WRITE_PRIV | REDUCE_PRIV +DISCARD_MASK = 0x10000000 # For marking we don't need inputs +WRITE_ONLY = 0x10000002 # WRITE_PRIV | DISCARD_MASK +WRITE_DISCARD = 0x10000007 # READ_WRITE | DISCARD_MASK + +# Note: don't use __file__ here, it may return either .py or .pyc and cause +# non-deterministic failures. +library_name = "pygion.py" +max_legion_python_tasks = 1000000 +next_legion_task_id = c.legion_runtime_generate_library_task_ids( + c.legion_runtime_get_runtime(), + library_name.encode("utf-8"), + max_legion_python_tasks, +) +max_legion_task_id = next_legion_task_id + max_legion_python_tasks + +# Returns true if this module is running inside of a Legion +# executable. If false, then other Legion functionality should not be +# expected to work. +def inside_legion_executable(): + try: + c.legion_get_current_time_in_micros() + except AttributeError: + return False + else: + return True + + +input_args = legion_top.input_args + + +def execute_as_script(): + args = input_args(True) + if len(args) < 1: + return False, False # no idea what's going on here, just return + if os.path.basename(args[0]) != "legion_python": + return False, False # not in legion_python + if len(args) < 2 or args[1].startswith("-"): + return True, False # argument is a flag + # If it has an extension, we're going to guess that it was + # intended to be a script. + return True, len(os.path.splitext(args[1])[1]) > 1 + + +is_legion_python, is_script = execute_as_script() + +# The Legion context is stored in thread-local storage. This assumes +# that the Python processor maintains the invariant that every task +# corresponds to one and only one thread. +_my = threading.local() + +global_task_registration_barrier = None + + +class Context(object): + __slots__ = [ + "context_root", + "context", + "runtime_root", + "runtime", + "task_root", + "task", + "regions", + "owned_objects", + "current_launch", + "next_trace_id", + ] + + def __init__(self, context_root, runtime_root, task_root, regions): + self.context_root = context_root + self.context = self.context_root[0] + self.runtime_root = runtime_root + self.runtime = self.runtime_root[0] + self.task_root = task_root + self.task = self.task_root[0] + self.regions = regions + self.owned_objects = [] + self.current_launch = None + self.next_trace_id = 0 + + def track_object(self, obj): + self.owned_objects.append(weakref.ref(obj)) + + def begin_launch(self, launch): + assert self.current_launch == None + self.current_launch = launch + + def end_launch(self, launch): + assert self.current_launch == launch + self.current_launch = None + + +# Hack: Can't pickle static methods. +def _DomainPoint_unpickle(values): + return DomainPoint(values) + + +import numpy as np + + +class DomainPoint(object): + __slots__ = [ + "handle", + "_point", # cached properties + ] + + def __init__(self, values, **kwargs): + def parse_kwargs(_handle=None): + return _handle + + handle = parse_kwargs(**kwargs) + + if values is not None: + assert handle is None + try: + len(values) + except TypeError: + values = [values] + assert 1 <= len(values) <= _max_dim + self.handle = ffi.new("legion_domain_point_t *") + self.handle[0].dim = len(values) + for i, value in enumerate(values): + self.handle[0].point_data[i] = value + else: + # Important: Copy handle. Do NOT assume ownership. + assert handle is not None + self.handle = ffi.new("legion_domain_point_t *", handle) + + self._point = None + + def __add__(self, other): + output = self.point + other + return self.coerce(output) + + def __radd__(self, other): + output = other + self.point + return self.coerce(output) + + def __sub__(self, other): + output = self.point - other + return self.coerce(output) + + def __rsub__(self, other): + output = other - self.point + return self.coerce(output) + + def __mul__(self, other): + output = self.point * other + return self.coerce(output) + + def __rmul__(self, other): + output = other * self.point + return self.coerce(output) + + def __div__(self, other): + output = self.point / other + return self.coerce(output) + + def __rdiv__(self, other): + output = other / self.point + return self.coerce(output) + + def __reduce__(self): + return ( + _DomainPoint_unpickle, + ([self.handle[0].point_data[i] for i in xrange(self.dim)],), + ) + + def __int__(self): + assert self.dim == 1 + return self.handle[0].point_data[0] + + def __index__(self): + return self.__int__() + + def __getitem__(self, i): + assert 0 <= i < self.dim + return self.handle[0].point_data[i] + + def __eq__(self, other): + if not isinstance(other, DomainPoint): + return NotImplemented + return numpy.array_equal(self.point, other.point) + + def __hash__(self): + return hash(tuple(self.point)) + + def __str__(self): + if self.dim == 1: + return str(int(self)) + return str(self.point) + + def __repr__(self): + return "DomainPoint({})".format(self.point) + + @property + def dim(self): + return self.handle[0].dim + + @property + def point(self): + if self._point is None: + self._point = self.asarray() + return self._point + + @staticmethod + def coerce(value): + if not isinstance(value, DomainPoint): + return DomainPoint(value) + return value + + def raw_value(self): + return self.handle[0] + + def asarray(self): + return numpy.frombuffer( + ffi.buffer(self.handle[0].point_data), count=self.dim, dtype=numpy.int64 + ) + + +class Domain(object): + __slots__ = [ + "handle", + "_bounds", # cached properties + ] + + def __init__(self, extent, start=None, **kwargs): + def parse_kwargs(_handle=None): + return _handle + + handle = parse_kwargs(**kwargs) + + if extent is not None: + assert handle is None + try: + len(extent) + except TypeError: + extent = [extent] + if start is not None: + try: + len(start) + except TypeError: + start = [start] + assert len(start) == len(extent) + else: + start = [0 for _ in extent] + assert 1 <= len(extent) <= _max_dim + rect = ffi.new("legion_rect_{}d_t *".format(len(extent))) + for i in xrange(len(extent)): + rect[0].lo.x[i] = start[i] + rect[0].hi.x[i] = start[i] + extent[i] - 1 + handle = getattr(c, "legion_domain_from_rect_{}d".format(len(extent)))( + rect[0] + ) + + # Important: Copy handle. Do NOT assume ownership. + assert handle is not None + self.handle = ffi.new("legion_domain_t *", handle) + self._bounds = None + + @property + def dim(self): + return self.handle[0].dim + + @property + def volume(self): + return c.legion_domain_get_volume(self.handle[0]) + + @property + def bounds(self): + if self._bounds is None: + self._bounds = self.asarray() + return self._bounds + + @property + def extent(self): + bounds = self.bounds + return bounds[1] - bounds[0] + 1 + + @property + def start(self): + return self.bounds[0] + + @staticmethod + def coerce(value): + if not isinstance(value, Domain): + return Domain(value) + return value + + def __iter__(self): + return imap( + DomainPoint, + itertools.product( + *[ + xrange( + self.handle[0].rect_data[i], + self.handle[0].rect_data[i + self.dim] + 1, + ) + for i in xrange(self.dim) + ] + ), + ) + + def raw_value(self): + return self.handle[0] + + def asarray(self): + return numpy.frombuffer( + ffi.buffer(self.handle[0].rect_data), count=self.dim * 2, dtype=numpy.int64 + ).reshape((2, self.dim)) + + +class DomainTransform(object): + __slots__ = ["handle"] + + def __init__(self, matrix, **kwargs): + def parse_kwargs(_handle=None): + return _handle + + handle = parse_kwargs(**kwargs) + + if matrix is not None: + assert handle is None + matrix = numpy.asarray(matrix, dtype=numpy.int64) + transform = ffi.new("legion_transform_{}x{}_t *".format(*matrix.shape)) + ffi.buffer(transform[0].trans)[:] = matrix + handle = getattr( + c, "legion_domain_transform_from_{}x{}".format(*matrix.shape) + )(transform[0]) + + # Important: Copy handle. Do NOT assume ownership. + assert handle is not None + self.handle = ffi.new("legion_domain_transform_t *", handle) + + @staticmethod + def coerce(value): + if not isinstance(value, DomainTransform): + return DomainTransform(value) + return value + + def raw_value(self): + return self.handle[0] + + +class Future(object): + __slots__ = ["handle", "value_type", "argument_number"] + + def __init__(self, value, value_type=None, argument_number=None): + if value is None: + self.handle = None + elif isinstance(value, Future): + value.resolve_handle() + self.handle = c.legion_future_copy(value.handle) + if value_type is None: + value_type = value.value_type + elif value_type is not None: + if value_type.size > 0: + value_ptr = ffi.new(ffi.getctype(value_type.cffi_type, "*"), value) + else: + assert value is None + value_ptr = ffi.NULL + value_size = value_type.size + self.handle = c.legion_future_from_untyped_pointer( + _my.ctx.runtime, value_ptr, value_size + ) + else: + value_str = pickle.dumps(value, protocol=_pickle_version) + value_size = len(value_str) + value_ptr = ffi.new("char[]", value_size) + ffi.buffer(value_ptr, value_size)[:] = value_str + self.handle = c.legion_future_from_untyped_pointer( + _my.ctx.runtime, value_ptr, value_size + ) + + self.value_type = value_type + self.argument_number = argument_number + + @staticmethod + def from_cdata(value, *args, **kwargs): + result = Future(None, *args, **kwargs) + result.handle = c.legion_future_copy(value) + return result + + @staticmethod + def from_buffer(value, *args, **kwargs): + result = Future(None, *args, **kwargs) + result.handle = c.legion_future_from_untyped_pointer( + _my.ctx.runtime, ffi.from_buffer(value), len(value) + ) + return result + + def __del__(self): + if self.handle is not None: + c.legion_future_destroy(self.handle) + + def __reduce__(self): + if self.argument_number is None: + raise Exception( + "Cannot pickle a Future except when used as a task argument" + ) + return (Future, (None, self.value_type, self.argument_number)) + + def resolve_handle(self): + if self.handle is None and self.argument_number is not None: + self.handle = c.legion_future_copy( + c.legion_task_get_future(_my.ctx.task, self.argument_number) + ) + + def get(self): + self.resolve_handle() + + if self.handle is None: + return + if self.value_type is None: + value_ptr = c.legion_future_get_untyped_pointer(self.handle) + value_size = c.legion_future_get_untyped_size(self.handle) + assert value_size > 0 + value_str = ffi.unpack(ffi.cast("char *", value_ptr), value_size) + value = pickle.loads(value_str) + return value + elif self.value_type.size == 0: + c.legion_future_get_void_result(self.handle) + else: + expected_size = ffi.sizeof(self.value_type.cffi_type) + + value_ptr = c.legion_future_get_untyped_pointer(self.handle) + value_size = c.legion_future_get_untyped_size(self.handle) + assert value_size == expected_size + value = ffi.cast(ffi.getctype(self.value_type.cffi_type, "*"), value_ptr)[0] + # Hack: Use closure to keep self alive as long as the value is live. + if isinstance(value, ffi.CData): + return ffi.gc(value, lambda x: self) + return value + + def get_buffer(self): + self.resolve_handle() + + if self.handle is None: + return + value_ptr = c.legion_future_get_untyped_pointer(self.handle) + value_size = c.legion_future_get_untyped_size(self.handle) + return ffi.buffer(value_ptr, value_size) + + +class FutureMap(object): + __slots__ = ["handle", "value_type"] + + def __init__(self, handle, value_type=None): + self.handle = c.legion_future_map_copy(handle) + self.value_type = value_type + + def __del__(self): + c.legion_future_map_destroy(self.handle) + + def __getitem__(self, point): + point = DomainPoint.coerce(point) + return Future.from_cdata( + c.legion_future_map_get_future(self.handle, point.raw_value()), + value_type=self.value_type, + ) + + def wait_all_results(self): + c.legion_future_map_wait_all_results(self.handle) + + +_type_cache = {} + + +class Type(object): + __slots__ = ["numpy_type", "cffi_type", "size"] + + def __new__(cls, numpy_type, cffi_type): + if cffi_type in _type_cache: + return _type_cache[cffi_type] + obj = super(Type, cls).__new__(cls) + _type_cache[cffi_type] = obj + return obj + + def __init__(self, numpy_type, cffi_type): + assert (numpy_type is None) == (cffi_type is None) + self.numpy_type = numpy_type + self.cffi_type = cffi_type + self.size = ffi.sizeof(cffi_type) if cffi_type is not None else 0 + + def __reduce__(self): + return (Type, (self.numpy_type, self.cffi_type)) + + +# Pre-defined Types +void = Type(None, None) +bool_ = Type(numpy.bool_, "bool") +complex64 = Type(numpy.complex64, "float _Complex") +complex128 = Type(numpy.complex128, "double _Complex") +float32 = Type(numpy.float32, "float") +float64 = Type(numpy.float64, "double") +int8 = Type(numpy.int8, "int8_t") +int16 = Type(numpy.int16, "int16_t") +int32 = Type(numpy.int32, "int32_t") +int64 = Type(numpy.int64, "int64_t") +uint8 = Type(numpy.uint8, "uint8_t") +uint16 = Type(numpy.uint16, "uint16_t") +uint32 = Type(numpy.uint32, "uint32_t") +uint64 = Type(numpy.uint64, "uint64_t") + +_rect_types = [] +for dim in xrange(1, _max_dim + 1): + globals()["int{}d".format(dim)] = Type( + numpy.dtype([("x", numpy.int64, (dim,))], align=True), + "legion_point_{}d_t".format(dim), + ) + rtype = Type( + numpy.dtype( + [("lo", numpy.int64, (dim,)), ("hi", numpy.int64, (dim,))], align=True + ), + "legion_rect_{}d_t".format(dim), + ) + globals()["rect{}d".format(dim)] = rtype + _rect_types.append(rtype) +_rect_types = frozenset(_rect_types) + + +def is_rect_type(t): + return t in _rect_types + + +_redop_ids = {} + + +def _fill_redop_ids(): + operators = ["+", "-", "*", "/", "max", "min"] + types = [ + bool_, + int8, + int16, + int32, + int64, + uint8, + uint16, + uint32, + uint64, + None, + float32, + float64, + None, + complex64, + complex128, + ] + next_id = 1048576 + for operator in operators: + _redop_ids[operator] = {} + for type in types: + if type is not None: + _redop_ids[operator][type] = next_id + next_id += 1 + + +_fill_redop_ids() + + +class Privilege(object): + __slots__ = ["read", "write", "discard", "reduce", "fields"] + + def __init__( + self, read=False, write=False, discard=False, reduce=False, fields=None + ): + self.read = read + self.write = write + self.discard = discard + self.reduce = reduce + self.fields = fields + + if self.fields is not None: + assert len(self.fields) > 0 + + if self.discard: + assert self.write + + def _fields(self): + return (self.read, self.write, self.discard, self.reduce, self.fields) + + def __eq__(self, other): + if not isinstance(other, Privilege): + return NotImplemented + return self._fields() == other._fields() + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self._fields()) + + def __call__(self, *fields): + assert self.fields is None + return Privilege(self.read, self.write, self.discard, self.reduce, fields) + + def __add__(self, other): + return PrivilegeComposite([self, other]) + + def __repr__(self): + return str(self) + + def __str__(self): + if self.discard: + priv = "WD" + elif self.write: + priv = "RW" + elif self.read: + priv = "R" + elif self.reduce: + priv = "Reduce(%s)" % self.reduce + else: + priv = "N" + if self.fields is not None: + return "%s(%s)" % (priv, ", ".join(self.fields)) + return priv + + def _legion_privilege(self): + bits = NO_ACCESS + if self.reduce: + assert False + else: + if self.write: + bits = READ_WRITE + elif self.read: + bits = READ_ONLY + if self.discard: + bits |= DISCARD_MASK + return bits + + def _legion_grouped_privileges(self, fspace): + if self.fields: + if not set(self.fields) <= set(fspace.keys()): + raise Exception( + "Privilege fields ({}) are not a subset of fspace fields ({})".format( + " ".join(self.fields), " ".join(fspace.keys()) + ) + ) + fields = fspace.keys() if self.fields is None else self.fields + if self.reduce: + return [ + ( + self, + None, + self._legion_redop_id(fspace.field_types[field_name]), + (field_name,), + ) + for field_name in fields + ] + else: + return [ + ( + self, + self._legion_privilege(), + None, + fields if self.read or self.write or self.reduce else [], + ) + ] + + def _legion_redop_id(self, field_type): + return _redop_ids[self.reduce][field_type] + + +class PrivilegeComposite(object): + __slots__ = ["privileges"] + + def __init__(self, privileges): + self.privileges = self.normalize(privileges) + + @staticmethod + def normalize(privileges): + fields = collections.OrderedDict() + read_set = set() + write_set = set() + discard_set = set() + reduce_sets = collections.OrderedDict() + + for privilege in privileges: + privilege_fields = ( + privilege.fields if privilege.fields is not None else [None] + ) + fields.update([(x, True) for x in privilege_fields]) + if privilege.read: + read_set.update(privilege_fields) + if privilege.write: + write_set.update(privilege_fields) + if privilege.discard: + discard_set.update(privilege_fields) + if privilege.reduce: + if privilege.reduce not in reduce_sets: + reduce_sets[privilege.reduce] = set() + reduce_sets[privilege.reduce].update(privilege_fields) + + # Reductions combine with read/reduce privileges to upgrade to read-write. + for op, reduce_set in reduce_sets.items(): + write_set.update(reduce_set & read_set) + if None in read_set: + write_set.update(reduce_set) + for op2, reduce_set2 in reduce_sets.items(): + if op != op2: + write_set.update(reduce_set & reduce_set2) + if None in reduce_set2: + write_set.update(reduce_set) + + # Read/write/discard shadow reduction privileges. + if None in read_set or None in write_set or None in discard_set: + reduce_sets = collections.OrderedDict() + else: + for reduce_set in reduce_sets.values(): + reduce_set.difference_update(read_set, write_set, discard_set) + + # Discard shadows read/write. + if None in discard_set: + read_set = set() + write_set = set() + discard_set = set([None]) + else: + read_set -= discard_set + write_set -= discard_set + + # Write shadows read. + if None in write_set: + read_set = set() + write_set = set([None]) + else: + read_set -= write_set + + def filter_set(ctor, field_set): + if None in field_set: + return ctor + return ctor(*filter(lambda x: x in field_set, fields.keys())) + + return tuple( + ([filter_set(R, read_set)] if len(read_set) > 0 else []) + + ([filter_set(RW, write_set)] if len(write_set) > 0 else []) + + ([filter_set(WD, discard_set)] if len(discard_set) > 0 else []) + + [ + filter_set(Reduce(op), reduce_set) + for op, reduce_set in reduce_sets.items() + ] + ) + + def __eq__(self, other): + if len(self.privileges) == 1: + return other == self.privileges[0] + + if not isinstance(other, PrivilegeComposite): + return NotImplemented + return self.privileges == other.privileges + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self.privileges) + + def __add__(self, other): + return PrivilegeComposite(self.privileges + (other,)) + + def __repr__(self): + return str(self) + + def __str__(self): + return " + ".join(map(str, self.privileges)) + + def _legion_grouped_privileges(self, fspace): + return [ + x + for privilege in self.privileges + for x in privilege._legion_grouped_privileges(fspace) + ] + + +# Pre-defined Privileges +N = Privilege() +R = Privilege(read=True) +RO = Privilege(read=True) +RW = Privilege(read=True, write=True) +WD = Privilege(write=True, discard=True) + + +def Reduce(operator, *fields): + return Privilege(reduce=operator, fields=fields if len(fields) > 0 else None) + + +class Disjointness(object): + __slots__ = ["kind", "value"] + + def __init__(self, kind, value): + self.kind = kind + self.value = value + + def __eq__(self, other): + return isinstance(other, Disjointness) and self.value == other.value + + def __cmp__(self, other): + assert isinstance(other, Disjointness) + return self.value.__cmp__(other.value) + + def __hash__(self): + return hash(self.value) + + def __str__(self): + return self.kind + + +disjoint = Disjointness("disjoint", 0) +aliased = Disjointness("aliased", 1) +compute = Disjointness("compute", 2) +disjoint_complete = Disjointness("disjoint_complete", 3) +aliased_complete = Disjointness("aliased_complete", 4) +compute_complete = Disjointness("compute_complete", 5) +disjoint_incomplete = Disjointness("disjoint_incomplete", 6) +aliased_incomplete = Disjointness("aliased_incomplete", 7) +compute_incomplete = Disjointness("compute_incomplete", 8) + + +class FileMode(object): + __slots__ = ["kind", "value"] + + def __init__(self, kind, value): + self.kind = kind + self.value = value + + def __eq__(self, other): + return isinstance(other, FileMode) and self.value == other.value + + def __cmp__(self, other): + assert isinstance(other, FileMode) + return self.value.__cmp__(other.value) + + def __hash__(self): + return hash(self.value) + + def __str__(self): + return self.kind + + +file_read_only = FileMode("read_only", 0) +file_read_write = FileMode("read_write", 1) +file_create = FileMode("create", 2) + +# Hack: Can't pickle static methods. +def _Ispace_unpickle(ispace_tid, ispace_id, ispace_type_tag, owned): + handle = ffi.new("legion_index_space_t *") + handle[0].tid = ispace_tid + handle[0].id = ispace_id + handle[0].type_tag = ispace_type_tag + return Ispace(None, _handle=handle[0], _owned=owned) + + +class Ispace(object): + __slots__ = [ + "handle", + "owned", + "escaped", + "_domain", # cached properties + "__weakref__", # allow weak references + ] + + def __init__(self, extent, start=None, name=None, **kwargs): + def parse_kwargs(_handle=None, _owned=False): + return _handle, _owned + + handle, owned = parse_kwargs(**kwargs) + + if extent is not None: + assert handle is None + domain = Domain(extent, start=start).raw_value() + handle = c.legion_index_space_create_domain( + _my.ctx.runtime, _my.ctx.context, domain + ) + if name is not None: + c.legion_index_space_attach_name( + _my.ctx.runtime, handle, name.encode("utf-8"), False + ) + owned = True + + # Important: Copy handle. Do NOT assume ownership. + assert handle is not None + self.handle = ffi.new("legion_index_space_t *", handle) + self.owned = owned + self.escaped = False + self._domain = None + + if self.owned: + _my.ctx.track_object(self) + + def __del__(self): + if self.owned and not self.escaped: + self.destroy() + + def __reduce__(self): + return ( + _Ispace_unpickle, + ( + self.handle[0].tid, + self.handle[0].id, + self.handle[0].type_tag, + self.owned and self.escaped, + ), + ) + + def __iter__(self): + return self.domain.__iter__() + + @property + def domain(self): + if self._domain is None: + self._domain = Domain( + None, + _handle=c.legion_index_space_get_domain( + _my.ctx.runtime, self.handle[0] + ), + ) + return self._domain + + @property + def dim(self): + return self.domain.dim + + @property + def volume(self): + return self.domain.volume + + @property + def bounds(self): + return self.domain.bounds + + @staticmethod + def coerce(value): + if not isinstance(value, Ispace): + return Ispace(value) + return value + + def destroy(self): + assert self.owned and not self.escaped + + # This is not something you want to have happen in a + # destructor, since fspaces may outlive the lifetime of the handle. + c.legion_index_space_destroy(_my.ctx.runtime, _my.ctx.context, self.handle[0]) + # Clear out references. Technically unnecessary but avoids abuse. + del self.handle + + def raw_value(self): + return self.handle[0] + + +# Hack: Can't pickle static methods. +def _Fspace_unpickle(fspace_id, field_ids, field_types, owned): + handle = ffi.new("legion_field_space_t *") + handle[0].id = fspace_id + return Fspace( + None, + _handle=handle[0], + _field_ids=field_ids, + _field_types=field_types, + _owned=owned, + ) + + +class Fspace(object): + __slots__ = [ + "handle", + "field_ids", + "field_types", + "owned", + "escaped", + "__weakref__", # allow weak references + ] + + def __init__(self, fields, name=None, **kwargs): + def parse_kwargs( + _handle=None, _field_ids=None, _field_types=None, _owned=False + ): + return _handle, _field_ids, _field_types, _owned + + handle, field_ids, field_types, owned = parse_kwargs(**kwargs) + + if fields is not None: + assert handle is None and field_ids is None and field_types is None + handle = c.legion_field_space_create(_my.ctx.runtime, _my.ctx.context) + if name is not None: + c.legion_field_space_attach_name( + _my.ctx.runtime, handle, name.encode("utf-8"), False + ) + alloc = c.legion_field_allocator_create( + _my.ctx.runtime, _my.ctx.context, handle + ) + field_ids = collections.OrderedDict() + field_types = collections.OrderedDict() + for field_name, field_entry in fields.items(): + try: + field_type, field_id = field_entry + except TypeError: + field_type = field_entry + field_id = ffi.cast("legion_field_id_t", AUTO_GENERATE_ID) + field_id = c.legion_field_allocator_allocate_field( + alloc, field_type.size, field_id + ) + c.legion_field_id_attach_name( + _my.ctx.runtime, handle, field_id, field_name.encode("utf-8"), False + ) + field_ids[field_name] = field_id + field_types[field_name] = field_type + c.legion_field_allocator_destroy(alloc) + owned = True + + # Important: Copy handle. Do NOT assume ownership. + assert handle is not None and field_ids is not None and field_types is not None + self.handle = ffi.new("legion_field_space_t *", handle) + self.field_ids = field_ids + self.field_types = field_types + self.owned = owned + self.escaped = False + + if owned: + _my.ctx.track_object(self) + + def __del__(self): + if self.owned and not self.escaped: + self.destroy() + + def __reduce__(self): + return ( + _Fspace_unpickle, + ( + self.handle[0].id, + self.field_ids, + self.field_types, + self.owned and self.escaped, + ), + ) + + @staticmethod + def coerce(value): + if not isinstance(value, Fspace): + return Fspace(value) + return value + + def destroy(self): + assert self.owned and not self.escaped + + # This is not something you want to have happen in a + # destructor, since fspaces may outlive the lifetime of the handle. + c.legion_field_space_destroy(_my.ctx.runtime, _my.ctx.context, self.handle[0]) + # Clear out references. Technically unnecessary but avoids abuse. + del self.handle + del self.field_ids + del self.field_types + + def raw_value(self): + return self.handle[0] + + def keys(self): + return self.field_ids.keys() + + +# Hack: Can't pickle static methods. +def _Region_unpickle(ispace, fspace, tree_id, owned): + handle = ffi.new("legion_logical_region_t *") + handle[0].tree_id = tree_id + handle[0].index_space = ispace.handle[0] + handle[0].field_space = fspace.handle[0] + + return Region(ispace, fspace, _handle=handle[0], _owned=owned) + + +class Region(object): + __slots__ = [ + "handle", + "ispace", + "fspace", + "parent", + "instances", + "privileges", + "instance_wrappers", + "owned", + "escaped", + "__weakref__", # allow weak references + ] + + # Make this speak the Type interface + numpy_type = None + cffi_type = "legion_logical_region_t" + size = ffi.sizeof(cffi_type) + + def __init__(self, ispace, fspace, name=None, **kwargs): + def parse_kwargs(_handle=None, _parent=None, _owned=False): + return _handle, _parent, _owned + + handle, parent, owned = parse_kwargs(**kwargs) + + if handle is None: + assert parent is None + ispace = Ispace.coerce(ispace) + fspace = Fspace.coerce(fspace) + handle = c.legion_logical_region_create( + _my.ctx.runtime, + _my.ctx.context, + ispace.raw_value(), + fspace.raw_value(), + False, + ) + if name is not None: + c.legion_logical_region_attach_name( + _my.ctx.runtime, handle, name.encode("utf-8"), False + ) + owned = True + + # Important: Copy handle. Do NOT assume ownership. + assert handle is not None + self.handle = ffi.new("legion_logical_region_t *", handle) + self.ispace = ispace + self.fspace = fspace + self.parent = parent + self.owned = owned + self.escaped = False + self.instances = {} + self.privileges = {} + self.instance_wrappers = {} + + if owned: + _my.ctx.track_object(self) + for field_name in fspace.field_ids.keys(): + self._set_privilege(field_name, RW) + + def __del__(self): + if self.owned and not self.escaped: + self.destroy() + + def __reduce__(self): + return ( + _Region_unpickle, + ( + self.ispace, + self.fspace, + self.handle[0].tree_id, + self.owned and self.escaped, + ), + ) + + def destroy(self): + assert self.owned and not self.escaped + + # This is not something you want to have happen in a + # destructor, since regions may outlive the lifetime of the handle. + c.legion_logical_region_destroy( + _my.ctx.runtime, _my.ctx.context, self.handle[0] + ) + # Clear out references. Technically unnecessary but avoids abuse. + del self.parent + del self.instance_wrappers + del self.instances + del self.handle + del self.ispace + del self.fspace + + def raw_value(self): + return self.handle[0] + + def keys(self): + return self.fspace.keys() + + def values(self): + for key in self.keys(): + if key in self.privileges and self.privileges[key] is not None: + yield getattr(self, key) + + def items(self): + for key in self.keys(): + if key in self.privileges and self.privileges[key] is not None: + yield key, getattr(self, key) + + def _set_privilege(self, field_name, privilege): + assert self.parent is None # not supported on subregions + assert field_name not in self.privileges + self.privileges[field_name] = privilege + + def _set_instance(self, field_name, instance, privilege=None): + assert self.parent is None # not supported on subregions + assert field_name not in self.instances + self.instances[field_name] = instance + if privilege is not None: + self._set_privilege(field_name, privilege) + + def _clear_instance(self, field_name): + assert self.parent is None # not supported on subregions + if field_name in self.instances: + # FIXME: need to determine when it is safe to destroy the + # associated instance (may or may not be inline mapped) + del self.instances[field_name] + + def _map_inline(self): + assert self.parent is None # FIXME: support inline mapping subregions + + fields_by_privilege = collections.defaultdict(set) + for field_name, privilege in self.privileges.items(): + fields_by_privilege[privilege].add(field_name) + for privilege, field_names in fields_by_privilege.items(): + launcher = c.legion_inline_launcher_create_logical_region( + self.handle[0], + privilege._legion_privilege(), + 0, # EXCLUSIVE + self.handle[0], + 0, + False, + 0, + 0, + ) + for field_name in field_names: + c.legion_inline_launcher_add_field( + launcher, self.fspace.field_ids[field_name], True + ) + instance = c.legion_inline_launcher_execute( + _my.ctx.runtime, _my.ctx.context, launcher + ) + for field_name in field_names: + self._set_instance(field_name, instance) + + def __getattr__(self, field_name): + if field_name in self.fspace.field_ids: + if field_name not in self.instances: + if self.privileges[field_name] is None: + raise Exception( + 'Invalid attempt to access field "%s" without privileges' + % field_name + ) + self._map_inline() + if field_name not in self.instance_wrappers: + self.instance_wrappers[field_name] = RegionField(self, field_name) + return self.instance_wrappers[field_name] + else: + raise AttributeError() + + +class RegionField(numpy.ndarray): + # NumPy requires us to implement __new__ for subclasses of ndarray: + # https://docs.scipy.org/doc/numpy/user/basics.subclassing.html + def __new__(cls, region, field_name): + accessor = RegionField._get_accessor(region, field_name) + initializer = RegionField._get_array_initializer(region, field_name, accessor) + if initializer is None: + obj = numpy.empty(tuple(0 for i in xrange(region.ispace.dim))).view( + dtype=region.fspace.field_types[field_name].numpy_type, type=cls + ) + else: + obj = numpy.asarray(initializer).view( + dtype=region.fspace.field_types[field_name].numpy_type, type=cls + ) + + obj.accessor = accessor + return obj + + @staticmethod + def _get_accessor(region, field_name): + # Note: the accessor needs to be kept alive, to make sure to + # save the result of this function in an instance variable. + instance = region.instances[field_name] + dim = region.ispace.dim + get_accessor = getattr( + c, "legion_physical_region_get_field_accessor_array_{}d".format(dim) + ) + return get_accessor(instance, region.fspace.field_ids[field_name]) + + @staticmethod + def _get_base_and_stride(region, field_name, accessor): + domain = region.ispace.domain + dim = domain.dim + if domain.volume < 1: + return None, None, None + + rect = getattr(c, "legion_domain_get_rect_{}d".format(dim))(domain.raw_value()) + subrect = ffi.new("legion_rect_{}d_t *".format(dim)) + offsets = ffi.new("legion_byte_offset_t[]", dim) + + base_ptr = getattr(c, "legion_accessor_array_{}d_raw_rect_ptr".format(dim))( + accessor, rect, subrect, offsets + ) + assert base_ptr + for i in xrange(dim): + assert subrect[0].lo.x[i] == rect.lo.x[i] + assert subrect[0].hi.x[i] == rect.hi.x[i] + assert offsets[0].offset == region.fspace.field_types[field_name].size + + shape = tuple(rect.hi.x[i] - rect.lo.x[i] + 1 for i in xrange(dim)) + strides = tuple(offsets[i].offset for i in xrange(dim)) + + return base_ptr, shape, strides + + @staticmethod + def _get_array_initializer(region, field_name, accessor): + base_ptr, shape, strides = RegionField._get_base_and_stride( + region, field_name, accessor + ) + if base_ptr is None: + return None + + field_type = region.fspace.field_types[field_name] + + # Numpy doesn't know about CFFI pointers, so we have to cast + # this to a Python long before we can hand it off to Numpy. + base_ptr = long(ffi.cast("size_t", base_ptr)) + + return _RegionNdarray(shape, field_type, base_ptr, strides, False) + + +# This is a dummy object that is only used as an initializer for the +# RegionField object above. It is thrown away as soon as the +# RegionField is constructed. +class _RegionNdarray(object): + __slots__ = ["__array_interface__"] + + def __init__(self, shape, field_type, base_ptr, strides, read_only): + # See: https://docs.scipy.org/doc/numpy/reference/arrays.interface.html + self.__array_interface__ = { + "version": 3, + "shape": shape, + "typestr": numpy.dtype(field_type.numpy_type).str, + "data": (base_ptr, read_only), + "strides": strides, + } + + +def fill(region, field_names, value): + assert isinstance(region, Region) + if isinstance(field_names, basestring): + field_names = [field_names] + + for field_name in field_names: + field_id = region.fspace.field_ids[field_name] + field_type = region.fspace.field_types[field_name] + raw_value = ffi.new("{} *".format(field_type.cffi_type), value) + c.legion_runtime_fill_field( + _my.ctx.runtime, + _my.ctx.context, + region.raw_value(), + region.parent.raw_value() + if region.parent is not None + else region.raw_value(), + field_id, + raw_value, + field_type.size, + c.legion_predicate_true(), + ) + + +def copy(src_region, src_field_names, dst_region, dst_field_names, redop=None): + assert isinstance(src_region, Region) + assert isinstance(dst_region, Region) + + if isinstance(src_field_names, basestring): + src_field_names = [src_field_names] + if isinstance(dst_field_names, basestring): + dst_field_names = [dst_field_names] + + launcher = c.legion_copy_launcher_create(c.legion_predicate_true(), 0, 0) + + if redop is None: + src_groups = [src_field_names] + dst_groups = [dst_field_names] + add_dst_requirement = ( + c.legion_copy_launcher_add_dst_region_requirement_logical_region + ) + else: + src_groups = zip(src_field_names) + dst_groups = zip(dst_field_names) + add_dst_requirement = ( + c.legion_copy_launcher_add_dst_region_requirement_logical_region_reduction + ) + + for idx, group in enumerate(src_groups): + c.legion_copy_launcher_add_src_region_requirement_logical_region( + launcher, + src_region.raw_value(), + R._legion_privilege(), + 0, # EXCLUSIVE + src_region.parent.raw_value() + if src_region.parent is not None + else src_region.raw_value(), + 0, + False, + ) + for src_field_name in group: + src_field_id = src_region.fspace.field_ids[src_field_name] + c.legion_copy_launcher_add_src_field(launcher, idx, src_field_id, True) + + for idx, group in enumerate(dst_groups): + if redop is None: + dst_privilege = RW._legion_privilege() + else: + dst_field_type = dst_region.fspace.field_types[group[0]] + dst_privilege = Reduce(redop, [group[0]])._legion_redop_id(dst_field_type) + add_dst_requirement( + launcher, + dst_region.raw_value(), + dst_privilege, + 0, # EXCLUSIVE + dst_region.parent.raw_value() + if dst_region.parent is not None + else dst_region.raw_value(), + 0, + False, + ) + for dst_field_name in group: + dst_field_id = dst_region.fspace.field_ids[dst_field_name] + c.legion_copy_launcher_add_dst_field(launcher, idx, dst_field_id, True) + + c.legion_copy_launcher_execute(_my.ctx.runtime, _my.ctx.context, launcher) + + c.legion_copy_launcher_destroy(launcher) + + +@contextlib.contextmanager +def attach_hdf5(region, filename, field_map, mode, restricted=True, mapped=False): + assert isinstance(region, Region) + + assert isinstance(filename, basestring) + filename = filename.encode("utf-8") + + raw_field_map = c.legion_field_map_create() + encoded_values = [] # make sure these don't get deleted before the launcher + for field_name, value in field_map.items(): + encoded_value = value.encode("utf-8") + encoded_values.append(encoded_value) + c.legion_field_map_insert( + raw_field_map, region.fspace.field_ids[field_name], encoded_value + ) + region._clear_instance(field_name) + + assert isinstance(mode, FileMode) + + launcher = c.legion_attach_launcher_create( + region.raw_value(), + region.parent.raw_value() if region.parent is not None else region.raw_value(), + EXTERNAL_HDF5_FILE, + ) + + c.legion_attach_launcher_attach_hdf5(launcher, filename, raw_field_map, mode.value) + c.legion_attach_launcher_set_restricted(launcher, restricted) + c.legion_attach_launcher_set_mapped(launcher, mapped) + + instance = c.legion_attach_launcher_execute( + _my.ctx.runtime, _my.ctx.context, launcher + ) + + c.legion_attach_launcher_destroy(launcher) + c.legion_field_map_destroy(raw_field_map) + + yield + + c.legion_detach_external_resource(_my.ctx.runtime, _my.ctx.context, instance) + + +@contextlib.contextmanager +def acquire(region, field_names): + assert isinstance(region, Region) + + launcher = c.legion_acquire_launcher_create( + region.raw_value(), + region.parent.raw_value() if region.parent is not None else region.raw_value(), + c.legion_predicate_true(), + 0, + 0, + ) + + for field_name in field_names: + c.legion_acquire_launcher_add_field( + launcher, region.fspace.field_ids[field_name] + ) + + c.legion_acquire_launcher_execute(_my.ctx.runtime, _my.ctx.context, launcher) + c.legion_acquire_launcher_destroy(launcher) + + yield + + launcher = c.legion_release_launcher_create( + region.raw_value(), + region.parent.raw_value() if region.parent is not None else region.raw_value(), + c.legion_predicate_true(), + 0, + 0, + ) + + for field_name in field_names: + c.legion_release_launcher_add_field( + launcher, region.fspace.field_ids[field_name] + ) + + c.legion_release_launcher_execute(_my.ctx.runtime, _my.ctx.context, launcher) + c.legion_release_launcher_destroy(launcher) + + +# Hack: Can't pickle static methods. +def _Ipartition_unpickle(tid, id, type_tag, parent, color_space): + handle = ffi.new("legion_index_partition_t *") + handle[0].tid = tid + handle[0].id = id + handle[0].type_tag = type_tag + + return Ipartition(handle[0], parent, color_space) + + +class Ipartition(object): + __slots__ = ["handle", "parent", "color_space"] + + # Make this speak the Type interface + numpy_type = None + cffi_type = "legion_index_partition_t" + size = ffi.sizeof(cffi_type) + + def __init__(self, handle, parent, color_space): + # Important: Copy handle. Do NOT assume ownership. + self.handle = ffi.new("legion_index_partition_t *", handle) + self.parent = parent + self.color_space = color_space + + def __reduce__(self): + return ( + _Ipartition_unpickle, + ( + self.handle[0].tid, + self.handle[0].id, + self.handle[0].type_tag, + self.parent, + self.color_space, + ), + ) + + def __getitem__(self, point): + if isinstance(point, SymbolicExpr): + return SymbolicIndexAccess(self, point) + point = DomainPoint.coerce(point) + subspace = c.legion_index_partition_get_index_subspace_domain_point( + _my.ctx.runtime, self.handle[0], point.raw_value() + ) + return Ispace(None, _handle=subspace) + + def __iter__(self): + for point in self.color_space: + yield self[point] + + @staticmethod + def equal(ispace, color_space, granularity=1, color=AUTO_GENERATE_ID): + assert isinstance(ispace, Ispace) + color_space = Ispace.coerce(color_space) + handle = c.legion_index_partition_create_equal( + _my.ctx.runtime, + _my.ctx.context, + ispace.raw_value(), + color_space.raw_value(), + granularity, + color, + ) + return Ipartition(handle, ispace, color_space) + + @staticmethod + def by_field(region, field, color_space, color=AUTO_GENERATE_ID): + assert isinstance(region, Region) + color_space = Ispace.coerce(color_space) + handle = c.legion_index_partition_create_by_field( + _my.ctx.runtime, + _my.ctx.context, + region.raw_value(), + region.parent.raw_value() + if region.parent is not None + else region.raw_value(), + region.fspace.field_ids[field], + color_space.raw_value(), + color, + 0, + 0, + disjoint.value, + ) + return Ipartition(handle, region.ispace, color_space) + + @staticmethod + def image( + ispace, + projection, + field, + color_space, + part_kind=compute, + color=AUTO_GENERATE_ID, + ): + assert isinstance(ispace, Ispace) + assert isinstance(projection, Partition) + assert isinstance(part_kind, Disjointness) + color_space = Ispace.coerce(color_space) + parent = projection.parent + if is_rect_type(parent.fspace.field_types[field]): + create_by_image = c.legion_index_partition_create_by_image_range + else: + create_by_image = c.legion_index_partition_create_by_image + handle = create_by_image( + _my.ctx.runtime, + _my.ctx.context, + ispace.raw_value(), + projection.raw_value(), + parent.parent.raw_value() + if parent.parent is not None + else parent.raw_value(), + parent.fspace.field_ids[field], + color_space.raw_value(), + part_kind.value, + color, + 0, + 0, + ) + return Ipartition(handle, parent.ispace, color_space) + + @staticmethod + def preimage( + projection, + region, + field, + color_space, + part_kind=compute, + color=AUTO_GENERATE_ID, + ): + assert isinstance(projection, Ipartition) + assert isinstance(region, Region) + assert isinstance(part_kind, Disjointness) + color_space = Ispace.coerce(color_space) + if is_rect_type(region.fspace.field_types[field]): + create_by_preimage = c.legion_index_partition_create_by_preimage_range + else: + create_by_preimage = c.legion_index_partition_create_by_preimage + handle = create_by_preimage( + _my.ctx.runtime, + _my.ctx.context, + projection.raw_value(), + region.raw_value(), + region.parent.raw_value() + if region.parent is not None + else region.raw_value(), + region.fspace.field_ids[field], + color_space.raw_value(), + part_kind.value, + color, + 0, + 0, + ) + return Ipartition(handle, region.ispace, color_space) + + @staticmethod + def restrict( + ispace, + color_space, + transform, + extent, + part_kind=compute, + color=AUTO_GENERATE_ID, + ): + assert isinstance(ispace, Ispace) + assert isinstance(part_kind, Disjointness) + color_space = Ispace.coerce(color_space) + transform = DomainTransform.coerce(transform) + extent = Domain.coerce(extent) + handle = c.legion_index_partition_create_by_restriction( + _my.ctx.runtime, + _my.ctx.context, + ispace.raw_value(), + color_space.raw_value(), + transform.raw_value(), + extent.raw_value(), + part_kind.value, + color, + ) + return Ipartition(handle, ispace, color_space) + + @staticmethod + def pending(ispace, color_space, part_kind=compute, color=AUTO_GENERATE_ID): + assert isinstance(ispace, Ispace) + assert isinstance(part_kind, Disjointness) + color_space = Ispace.coerce(color_space) + handle = c.legion_index_partition_create_pending_partition( + _my.ctx.runtime, + _my.ctx.context, + ispace.raw_value(), + color_space.raw_value(), + part_kind.value, + color, + ) + return Ipartition(handle, ispace, color_space) + + # The following methods are for pending partitions only: + def union(self, color, ispaces): + color = DomainPoint.coerce(color) + + handles = ffi.new( + "legion_index_space_t[]", [ispace.raw_value() for ispace in ispaces] + ) + c.legion_index_partition_create_index_space_union_spaces( + _my.ctx.runtime, + _my.ctx.context, + self.handle[0], + color.raw_value(), + handles, + len(ispaces), + ) + + def destroy(self): + # This is not something you want to have happen in a + # destructor, since partitions may outlive the lifetime of the handle. + c.legion_index_partition_destroy( + _my.ctx.runtime, _my.ctx.context, self.handle[0] + ) + # Clear out references. Technically unnecessary but avoids abuse. + del self.handle + del self.parent + del self.color_space + + def raw_value(self): + return self.handle[0] + + +# Hack: Can't pickle static methods. +def _Partition_unpickle(parent, ipartition): + handle = ffi.new("legion_logical_partition_t *") + handle[0].tree_id = parent.raw_value().tree_id + handle[0].index_partition = ipartition.raw_value() + handle[0].field_space = parent.fspace.raw_value() + + return Partition(parent, ipartition, _handle=handle[0]) + + +class Partition(object): + __slots__ = ["handle", "parent", "ipartition"] + + # Make this speak the Type interface + numpy_type = None + cffi_type = "legion_logical_partition_t" + size = ffi.sizeof(cffi_type) + + def __init__(self, parent, ipartition, **kwargs): + def parse_kwargs(_handle=None): + return _handle + + handle = parse_kwargs(**kwargs) + + if handle is None: + assert isinstance(parent, Region) + assert isinstance(ipartition, Ipartition) + handle = c.legion_logical_partition_create( + _my.ctx.runtime, + _my.ctx.context, + parent.raw_value(), + ipartition.raw_value(), + ) + + # Important: Copy handle. Do NOT assume ownership. + assert handle is not None + self.handle = ffi.new("legion_logical_partition_t *", handle) + self.parent = parent + self.ipartition = ipartition + + def __reduce__(self): + return (_Partition_unpickle, (self.parent, self.ipartition)) + + def __getitem__(self, point): + if isinstance(point, SymbolicExpr): + return SymbolicIndexAccess(self, point) + point = DomainPoint.coerce(point) + subspace = self.ipartition[point] + subregion = c.legion_logical_partition_get_logical_subregion_by_color_domain_point( + _my.ctx.runtime, self.handle[0], point.raw_value() + ) + return Region( + subspace, + self.parent.fspace, + _handle=subregion, + _parent=self.parent.parent + if self.parent.parent is not None + else self.parent, + ) + + def __iter__(self): + for point in self.color_space: + yield self[point] + + @property + def color_space(self): + return self.ipartition.color_space + + @staticmethod + def equal(region, color_space, granularity=1, color=AUTO_GENERATE_ID): + assert isinstance(region, Region) + ipartition = Ipartition.equal(region.ispace, color_space, granularity, color) + return Partition(region, ipartition) + + @staticmethod + def by_field(region, field, color_space, color=AUTO_GENERATE_ID): + assert isinstance(region, Region) + ipartition = Ipartition.by_field(region, field, color_space, color) + return Partition(region, ipartition) + + @staticmethod + def image( + region, + projection, + field, + color_space, + part_kind=compute, + color=AUTO_GENERATE_ID, + ): + assert isinstance(region, Region) + ipartition = Ipartition.image( + region.ispace, projection, field, color_space, part_kind, color + ) + return Partition(region, ipartition) + + @staticmethod + def preimage( + projection, + region, + field, + color_space, + part_kind=compute, + color=AUTO_GENERATE_ID, + ): + assert isinstance(projection, Partition) + ipartition = Ipartition.preimage( + projection.ipartition, region, field, color_space, part_kind, color + ) + return Partition(region, ipartition) + + @staticmethod + def restrict( + region, + color_space, + transform, + extent, + part_kind=compute, + color=AUTO_GENERATE_ID, + ): + assert isinstance(region, Region) + ipartition = Ipartition.restrict( + region.ispace, color_space, transform, extent, part_kind, color + ) + return Partition(region, ipartition) + + @staticmethod + def pending(region, color_space, part_kind=compute, color=AUTO_GENERATE_ID): + assert isinstance(region, Region) + ipartition = Ipartition.pending(region.ispace, color_space, part_kind, color) + return Partition(region, ipartition) + + def union(self, color, regions): + ispaces = [region.ispace for region in regions] + self.ipartition.union(color, ispaces) + + def destroy(self): + # This is not something you want to have happen in a + # destructor, since partitions may outlive the lifetime of the handle. + c.legion_logical_partition_destroy( + _my.ctx.runtime, _my.ctx.context, self.handle[0] + ) + # Clear out references. Technically unnecessary but avoids abuse. + del self.handle + del self.parent + del self.ipartition + + def raw_value(self): + return self.handle[0] + + +def define_regent_argument_struct( + task_id, argument_types, privileges, return_type, arguments +): + if argument_types is None: + raise Exception("Arguments must be typed in extern Regent tasks") + + struct_name = "task_args_%s" % task_id + + n_fields = int(math.ceil(len(argument_types) / 64.0)) + + fields = ["uint64_t %s[%s];" % ("__map", n_fields)] + for i, arg_type in enumerate(argument_types): + arg_name = "__arg_%s" % i + fields.append("%s %s;" % (arg_type.cffi_type, arg_name)) + for i, arg in enumerate(arguments): + if isinstance(arg, Region): + fields.append( + "legion_field_id_t __arg_%s_fields[%s];" + % (i, len(arg.fspace.field_types)) + ) + + struct = "typedef struct %s { %s } %s;" % ( + struct_name, + " ".join(fields), + struct_name, + ) + ffi.cdef(struct) + + return struct_name + + +class ExternTask(object): + __slots__ = [ + "argument_types", + "privileges", + "return_type", + "calling_convention", + "task_id", + "_argument_struct", + ] + + def __init__( + self, + task_id, + argument_types=None, + privileges=None, + return_type=void, + calling_convention=None, + ): + self.argument_types = argument_types + self.privileges = privileges + self.return_type = return_type + self.calling_convention = calling_convention + assert isinstance(task_id, int) + self.task_id = task_id + self._argument_struct = None + + def argument_struct(self, args): + if self.calling_convention == "regent" and self._argument_struct is None: + self._argument_struct = define_regent_argument_struct( + self.task_id, + self.argument_types, + self.privileges, + self.return_type, + args, + ) + return self._argument_struct + + def __call__(self, *args): + return self.spawn_task(*args) + + def spawn_task(self, *args, **kwargs): + if _my.ctx.current_launch: + return _my.ctx.current_launch.spawn_task(self, *args, **kwargs) + return TaskLaunch().spawn_task(self, *args, **kwargs) + + +def extern_task(**kwargs): + return ExternTask(**kwargs) + + +class ExternTaskWrapper(object): + # Note: Can't use __slots__ for this class because __qualname__ + # conflicts with the class variable. + def __init__(self, thunk, name): + self.thunk = thunk + self.__name__ = name + self.__qualname__ = name + + def __call__(self, *args, **kwargs): + f = self.thunk(*args, **kwargs) + if f.value_type != void: + return f.get() + + +_next_wrapper_id = 1000 + + +def extern_task_wrapper(privileges=None, return_type=void, **kwargs): + global _next_wrapper_id + extern = extern_task(privileges=privileges, return_type=return_type, **kwargs) + wrapper_name = str("wrapper_task_%s" % _next_wrapper_id) + _next_wrapper_id += 1 + wrapper = ExternTaskWrapper(extern, wrapper_name) + task_wrapper = task( + wrapper, privileges=privileges, return_type=return_type, inner=True + ) + setattr(sys.modules[__name__], wrapper_name, task_wrapper) + return task_wrapper + + +def get_qualname(fn): + # Python >= 3.3 only + try: + return fn.__qualname__.split(".") + except AttributeError: + pass + + # Python < 3.3 + try: + import qualname + + return qualname.qualname(fn).split(".") + except ImportError: + pass + + # Hack: Issue error if we're wrapping a class method and failed to + # get the qualname + import inspect + + context = [ + x[0].f_code.co_name + for x in inspect.stack() + if "__module__" in x[0].f_code.co_names + and inspect.getmodule(x[0].f_code).__name__ != __name__ + ] + if len(context) > 0: + raise Exception( + "To use a task defined in a class, please upgrade to Python >= 3.3 or install qualname (e.g. pip install qualname)" + ) + + return [fn.__name__] + + +def _postprocess(arg, point): + if hasattr(arg, "_legion_postprocess_task_argument"): + return arg._legion_postprocess_task_argument(point) + return arg + + +def _symbolize(arg): + if hasattr(arg, "_legion_symbolize_task_argument"): + return arg._legion_symbolize_task_argument() + return arg + + +class Task(object): + __slots__ = [ + "body", + "privileges", + "return_type", + "leaf", + "inner", + "idempotent", + "replicable", + "calling_convention", + "argument_struct", + "task_id", + "registered", + ] + + def __init__( + self, + body, + privileges=None, + return_type=None, + leaf=False, + inner=False, + idempotent=False, + replicable=False, + register=True, + task_id=None, + top_level=False, + ): + self.body = body + self.privileges = privileges + self.return_type = return_type + self.leaf = bool(leaf) + self.inner = bool(inner) + self.idempotent = bool(idempotent) + self.replicable = bool(replicable) + self.calling_convention = "python" + self.argument_struct = None + self.task_id = None + if register: + self.register(task_id, top_level) + + def __call__(self, *args, **kwargs): + # Hack: This entrypoint needs to be able to handle both being + # called in user code (to launch a task) and as the task + # wrapper when the task itself executes. Unfortunately isn't a + # good way to disentangle these. Detect if we're in the task + # wrapper case by checking the number and types of arguments. + if ( + len(args) == 3 + and isinstance(args[0], bytearray) + and isinstance(args[1], bytearray) + and isinstance(args[2], long) + ): + return self.execute_task(*args, **kwargs) + else: + return self.spawn_task(*args, **kwargs) + + def spawn_task(self, *args, **kwargs): + if _my.ctx.current_launch: + return _my.ctx.current_launch.spawn_task(self, *args, **kwargs) + return TaskLaunch().spawn_task(self, *args, **kwargs) + + def execute_task(self, raw_args, user_data, proc): + raw_arg_ptr = ffi.new("char[]", bytes(raw_args)) + raw_arg_size = len(raw_args) + + # Execute preamble to obtain Legion API context. + task = ffi.new("legion_task_t *") + raw_regions = ffi.new("legion_physical_region_t **") + num_regions = ffi.new("unsigned *") + context = ffi.new("legion_context_t *") + runtime = ffi.new("legion_runtime_t *") + c.legion_task_preamble( + raw_arg_ptr, + raw_arg_size, + proc, + task, + raw_regions, + num_regions, + context, + runtime, + ) + + # Decode arguments from Pickle format. + arg_ptr = ffi.cast("char *", c.legion_task_get_args(task[0])) + arg_size = c.legion_task_get_arglen(task[0]) + if c.legion_task_get_is_index_space(task[0]) and arg_size == 0: + arg_ptr = ffi.cast("char *", c.legion_task_get_local_args(task[0])) + arg_size = c.legion_task_get_local_arglen(task[0]) + + if arg_size > 0 and c.legion_task_get_depth(task[0]) > 0: + args = pickle.loads(ffi.unpack(arg_ptr, arg_size)) + else: + args = () + + # Unpack regions. + regions = [] + for i in xrange(num_regions[0]): + regions.append(raw_regions[0][i]) + + # Build context. + ctx = Context(context, runtime, task, regions) + + # Ensure that we're not getting tangled up in another + # thread. There should be exactly one thread per task. + try: + _my.ctx + except AttributeError: + pass + else: + raise Exception("thread-local context already set") + + # Store context in thread-local storage. + _my.ctx = ctx + + # Postprocess arguments. + point = DomainPoint(None, _handle=c.legion_task_get_index_point(task[0])) + args = tuple(_postprocess(arg, point) for arg in args) + + # Unpack physical regions. + if self.privileges is not None: + req = 0 + for i, arg in zip(range(len(args)), args): + if isinstance(arg, Region): + assert i < len(self.privileges) + groups = self.privileges[i]._legion_grouped_privileges(arg.fspace) + for priv, _, _, fields in groups: + assert req < num_regions[0] + instance = raw_regions[0][req] + req += 1 + + for field in fields: + arg._set_instance(field, instance, priv) + assert req == num_regions[0] + + # Execute task body. + result = self.body(*args) + + # Mark any remaining objects as escaped. + for ref in ctx.owned_objects: + obj = ref() + if obj is not None: + obj.escaped = True + + # Encode result. + if not self.return_type: + result_str = pickle.dumps(result, protocol=_pickle_version) + result_size = len(result_str) + result_ptr = ffi.new("char[]", result_size) + ffi.buffer(result_ptr, result_size)[:] = result_str + else: + if self.return_type.size > 0: + result_ptr = ffi.new( + ffi.getctype(self.return_type.cffi_type, "*"), result + ) + else: + result_ptr = ffi.NULL + result_size = self.return_type.size + + # Execute postamble. + c.legion_task_postamble(runtime[0], context[0], result_ptr, result_size) + + # Clear thread-local storage. + del _my.ctx + + def register(self, task_id, top_level_task): + assert self.task_id is None + + if not task_id: + global next_legion_task_id + task_id = next_legion_task_id + next_legion_task_id += 1 + # If we ever hit this then we need to allocate more task IDs + assert task_id < max_legion_task_id + + execution_constraints = c.legion_execution_constraint_set_create() + c.legion_execution_constraint_set_add_processor_constraint( + execution_constraints, c.PY_PROC + ) + + layout_constraints = c.legion_task_layout_constraint_set_create() + # FIXME: Add layout constraints + + options = ffi.new("legion_task_config_options_t *") + options[0].leaf = self.leaf + options[0].inner = self.inner + options[0].idempotent = self.idempotent + options[0].replicable = self.replicable + + qualname = get_qualname(self.body) + task_name = "%s.%s" % (self.body.__module__, ".".join(qualname)) + + c_qualname_comps = [ + ffi.new("char []", comp.encode("utf-8")) for comp in qualname + ] + c_qualname = ffi.new("char *[]", c_qualname_comps) + + global global_task_registration_barrier + if global_task_registration_barrier is not None: + c.legion_phase_barrier_arrive( + _my.ctx.runtime, _my.ctx.context, global_task_registration_barrier, 1 + ) + global_task_registration_barrier = c.legion_phase_barrier_advance( + _my.ctx.runtime, _my.ctx.context, global_task_registration_barrier + ) + c.legion_runtime_enable_scheduler_lock() + c.legion_phase_barrier_wait( + _my.ctx.runtime, _my.ctx.context, global_task_registration_barrier + ) + # Need to hold this through the end of registration. + # c.legion_runtime_disable_scheduler_lock() + + c.legion_runtime_register_task_variant_python_source_qualname( + c.legion_runtime_get_runtime(), + task_id, + task_name.encode("utf-8"), + True, # self.replicable or not is_script, # Global + execution_constraints, + layout_constraints, + options[0], + self.body.__module__.encode("utf-8"), + c_qualname, + len(qualname), + ffi.NULL, + 0, + ) + # If we're the top-level task then tell the runtime about our ID + if top_level_task: + c.legion_runtime_set_top_level_task_id(task_id) + if global_task_registration_barrier is not None: + c.legion_phase_barrier_arrive( + _my.ctx.runtime, _my.ctx.context, global_task_registration_barrier, 1 + ) + global_task_registration_barrier = c.legion_phase_barrier_advance( + _my.ctx.runtime, _my.ctx.context, global_task_registration_barrier + ) + # c.legion_runtime_enable_scheduler_lock() + c.legion_phase_barrier_wait( + _my.ctx.runtime, _my.ctx.context, global_task_registration_barrier + ) + c.legion_runtime_disable_scheduler_lock() + + c.legion_execution_constraint_set_destroy(execution_constraints) + c.legion_task_layout_constraint_set_destroy(layout_constraints) + + self.task_id = task_id + return self + + +def task(body=None, **kwargs): + if body is None: + return lambda body: task(body, **kwargs) + return Task(body, **kwargs) + + +_proj_functor_cache = {} + + +class _TaskLauncher(object): + __slots__ = ["task"] + + def __init__(self, task): + self.task = task + + def gather_futures(self, args): + normal = [] + futures = [] + for arg in args: + if isinstance(arg, Future): + arg = Future(arg, argument_number=len(futures)) + futures.append(arg) + normal.append(arg) + return normal, futures + + def encode_args(self, args): + task_args = ffi.new("legion_task_argument_t *") + task_args_buffer = None + if self.task.calling_convention == "python": + arg_str = pickle.dumps(args, protocol=_pickle_version) + task_args_buffer = ffi.new("char[]", arg_str) + task_args[0].args = task_args_buffer + task_args[0].arglen = len(arg_str) + elif self.task.calling_convention == "regent": + arg_struct = self.task.argument_struct(args) + task_args_buffer = ffi.new("%s*" % arg_struct) + # Note: ffi.new returns zeroed memory + for i, arg in enumerate(args): + if isinstance(arg, Future): + getattr(task_args_buffer, "__map")[i // 64] |= 1 << (i % 64) + for i, arg in enumerate(args): + if not isinstance(arg, Future): + arg_name = "__arg_%s" % i + arg_value = arg + if hasattr(arg, "handle") and not isinstance(arg, DomainPoint): + arg_value = arg.handle[0] + setattr(task_args_buffer, arg_name, arg_value) + for i, arg in enumerate(args): + if isinstance(arg, Region): + arg_name = "__arg_%s_fields" % i + arg_slot = getattr(task_args_buffer, arg_name) + for j, field_id in enumerate(arg.fspace.field_ids.values()): + arg_slot[j] = field_id + task_args[0].args = task_args_buffer + task_args[0].arglen = ffi.sizeof(arg_struct) + else: + # FIXME: External tasks need a dedicated calling + # convention to permit the passing of task arguments. + task_args[0].args = ffi.NULL + task_args[0].arglen = 0 + # WARNING: Need to return the interior buffer or else it will be GC'd + return task_args, task_args_buffer + + def attach_region_requirements(self, launcher, args, is_index_launch): + if is_index_launch: + + def add_region_normal(launcher, handle, *args): + return c.legion_index_launcher_add_region_requirement_logical_region( + launcher, handle, 100, *args # projection + ) + + def add_region_reduction(launcher, handle, *args): + return c.legion_index_launcher_add_region_requirement_logical_region_reduction( + launcher, handle, 100, *args # projection + ) + + add_partition_normal = ( + c.legion_index_launcher_add_region_requirement_logical_partition + ) + add_partition_reduction = ( + c.legion_index_launcher_add_region_requirement_logical_partition_reduction + ) + add_field = c.legion_index_launcher_add_field + else: + add_region_normal = ( + c.legion_task_launcher_add_region_requirement_logical_region + ) + add_region_reduction = ( + c.legion_task_launcher_add_region_requirement_logical_region_reduction + ) + add_field = c.legion_task_launcher_add_field + + for i, arg in zip(range(len(args)), args): + if isinstance(arg, Region) or ( + isinstance(arg, SymbolicExpr) and arg.is_region() + ): + if self.task.privileges is None or i >= len(self.task.privileges): + raise Exception("Privileges are required on all Region arguments") + groups = self.task.privileges[i]._legion_grouped_privileges(arg.fspace) + if isinstance(arg, Region): + parent = arg.parent if arg.parent is not None else arg + for _, priv, redop, fields in groups: + if redop is None: + req = add_region_normal( + launcher, + arg.raw_value(), + priv, + 0, # EXCLUSIVE + parent.raw_value(), + 0, + False, + ) + else: + req = add_region_reduction( + launcher, + arg.raw_value(), + redop, + 0, # EXCLUSIVE + parent.raw_value(), + 0, + False, + ) + for field in fields: + add_field(launcher, req, arg.fspace.field_ids[field], True) + elif isinstance(arg, SymbolicExpr): + + def arg_check(): + # P[...] + if not isinstance(arg, SymbolicIndexAccess): + return False, None + # P[i] or P[ID] + if isinstance( + arg.index, (SymbolicLoopIndex, ConcreteLoopIndex) + ): + return True, 0 + # P[f(i)] or P[f(ID)] + if ( + isinstance(arg.index, SymbolicCall) + and isinstance(arg.index.func, ProjectionFunctor) + and len(arg.index.args) == 1 + and isinstance( + arg.index.args[0], + (SymbolicLoopIndex, ConcreteLoopIndex), + ) + ): + return True, arg.index.func.proj_id + # P[i + ...] or P[ID + ...] + if isinstance(arg.index, SymbolicExpr): + curr_arg = _symbolize(arg) + f = ProjectionFunctor.create(curr_arg.index) + _proj_functor_cache[curr_arg.index] = f + return True, f.proj_id + return False, None + + valid, proj_id = arg_check() + assert valid + + parent = arg.parent if arg.parent is not None else arg + parent = parent.parent if parent.parent is not None else parent + for _, priv, redop, fields in groups: + if redop is None: + req = add_partition_normal( + launcher, + arg.raw_value(), + proj_id, + priv, + 0, # EXCLUSIVE + parent.raw_value(), + 0, + False, + ) + else: + req = add_partition_reduction( + launcher, + arg.raw_value(), + proj_id, + redop, + 0, # EXCLUSIVE + parent.raw_value(), + 0, + False, + ) + for field in fields: + add_field(launcher, req, arg.fspace.field_ids[field], True) + elif ( + self.task.privileges is not None + and i < len(self.task.privileges) + and self.task.privileges[i] + ): + raise TypeError( + "Privileges can only be specified for Region arguments, got %s" + % type(arg) + ) + + def spawn_task(self, *args, **kwargs): + # Hack: workaround for Python 2 not having keyword-only arguments + def validate_spawn_task_args(point=None, mapper=0, tag=0): + return point, mapper, tag + + point, mapper, tag = validate_spawn_task_args(**kwargs) + + assert isinstance(_my.ctx, Context) + + args, futures = self.gather_futures(args) + task_args, task_args_root = self.encode_args(args) + + # Construct the task launcher. + launcher = c.legion_task_launcher_create( + self.task.task_id, task_args[0], c.legion_predicate_true(), mapper, tag + ) + if point is not None: + point = DomainPoint.coerce(point) + c.legion_task_launcher_set_point(launcher, point.raw_value()) + self.attach_region_requirements(launcher, args, False) + for i, arg in zip(range(len(args)), args): + if ( + self.task.privileges is not None + and i < len(self.task.privileges) + and self.task.privileges[i] + and not isinstance(arg, Region) + ): + raise TypeError( + "Privileges can only be specified for Region arguments, got %s" + % type(arg) + ) + if isinstance(arg, Region): + pass # Already attached above + elif isinstance(arg, Future): + c.legion_task_launcher_add_future(launcher, arg.handle) + elif self.task.calling_convention is None: + # FIXME: Task arguments aren't being encoded AT ALL; + # at least throw an exception so that the user knows + raise Exception("External tasks do not support non-region arguments") + + # Launch the task. + if _my.ctx.current_launch is not None: + return _my.ctx.current_launch.attach_task_launcher( + launcher, point, root=task_args_root + ) + + result = c.legion_task_launcher_execute( + _my.ctx.runtime, _my.ctx.context, launcher + ) + c.legion_task_launcher_destroy(launcher) + + # Build future of result. + future = Future.from_cdata(result, value_type=self.task.return_type) + c.legion_future_destroy(result) + return future + + +class _IndexLauncher(_TaskLauncher): + __slots__ = [ + "task", + "domain", + "mapper", + "tag", + "global_args", + "local_args", + "region_args", + "future_args", + "reduction_op", + "future_map", + ] + + def __init__(self, task, domain, mapper, tag): + super(_IndexLauncher, self).__init__(task) + self.domain = domain + self.mapper = mapper + self.tag = tag + self.global_args = None + self.local_args = c.legion_argument_map_create() + self.region_args = None + self.future_args = [] + self.reduction_op = None + self.future_map = None + + def __del__(self): + c.legion_argument_map_destroy(self.local_args) + + def spawn_task(self, *args, **kwargs): + raise Exception("IndexLaunch does not support spawn_task") + + def attach_local_args(self, index, *args): + task_args, _ = self.encode_args(args) + c.legion_argument_map_set_point( + self.local_args, index.value.raw_value(), task_args[0], False + ) + + def attach_global_args(self, *args): + assert self.global_args is None + self.global_args = args + + def attach_region_args(self, *args): + self.region_args = args + + def attach_future_args(self, *args): + self.future_args = args + + def set_reduction_op(self, op): + self.reduction_op = op + + def launch(self): + # Encode global args (if any). + if self.global_args is not None: + global_args, global_args_root = self.encode_args(self.global_args) + else: + global_args = ffi.new("legion_task_argument_t *") + global_args[0].args = ffi.NULL + global_args[0].arglen = 0 + global_args_root = None + + # Construct the task launcher. + launcher = c.legion_index_launcher_create( + self.task.task_id, + self.domain.raw_value(), + global_args[0], + self.local_args, + c.legion_predicate_true(), + False, + self.mapper, + self.tag, + ) + + assert (self.global_args is not None) != (self.region_args is not None) + if self.global_args is not None: + self.attach_region_requirements(launcher, self.global_args, True) + if self.region_args is not None: + self.attach_region_requirements(launcher, self.region_args, True) + + for arg in self.future_args: + c.legion_index_launcher_add_future(launcher, arg.handle) + + # Launch the task. + if _my.ctx.current_launch is not None: + return _my.ctx.current_launch.attach_index_launcher( + launcher, root=global_args_root + ) + + launch = c.legion_index_launcher_execute + redop = [] + if self.reduction_op is not None: + assert self.task.return_type is not None + launch = c.legion_index_launcher_execute_reduction + redop = [_redop_ids[self.reduction_op][self.task.return_type]] + + result = launch(_my.ctx.runtime, _my.ctx.context, launcher, *redop) + c.legion_index_launcher_destroy(launcher) + + # Build future (map) of result. + if self.reduction_op is not None: + self.future_map = Future.from_cdata( + result, value_type=self.task.return_type + ) + c.legion_future_destroy(result) + else: + self.future_map = FutureMap(result, value_type=self.task.return_type) + c.legion_future_map_destroy(result) + + +class _MustEpochLauncher(object): + __slots__ = ["domain", "launcher", "roots", "has_sublaunchers"] + + def __init__(self, domain=None): + self.domain = domain + self.launcher = c.legion_must_epoch_launcher_create(0, 0) + if self.domain is not None: + c.legion_must_epoch_launcher_set_launch_domain( + self.launcher, self.domain.raw_value() + ) + self.roots = [] + self.has_sublaunchers = False + + def __del__(self): + c.legion_must_epoch_launcher_destroy(self.launcher) + + def spawn_task(self, *args, **kwargs): + raise Exception("MustEpochLaunch does not support spawn_task") + + def attach_task_launcher(self, task_launcher, point, root=None): + if point is None: + raise Exception("MustEpochLauncher requires a point for each task") + if root is not None: + self.roots.append(root) + c.legion_must_epoch_launcher_add_single_task( + self.launcher, point.raw_value(), task_launcher + ) + self.has_sublaunchers = True + + def attach_index_launcher(self, index_launcher, root=None): + if root is not None: + self.roots.append(root) + c.legion_must_epoch_launcher_add_index_task(self.launcher, index_launcher) + self.has_sublaunchers = True + + def launch(self): + if not self.has_sublaunchers: + raise Exception( + "MustEpochLaunch requires at least one point task to be executed" + ) + result = c.legion_must_epoch_launcher_execute( + _my.ctx.runtime, _my.ctx.context, self.launcher + ) + c.legion_future_map_destroy(result) + + +class TaskLaunch(object): + __slots__ = [] + + def spawn_task(self, task, *args, **kwargs): + launcher = _TaskLauncher(task=task) + return launcher.spawn_task(*args, **kwargs) + + +class _FuturePoint(object): + __slots__ = ["launcher", "point", "future"] + + def __init__(self, launcher, point): + self.launcher = launcher + self.point = point + self.future = None + + def get(self): + if self.future is not None: + return self.future.get() + + if self.launcher.future_map is None: + raise Exception( + "Cannot retrieve a future from an index launch until the launch is complete" + ) + + self.future = self.launcher.future_map[self.point] + + # Clear launcher and point + del self.launcher + del self.point + + return self.future.get() + + +class SymbolicExpr(object): + def __add__(self, other): + return SymbolicBinop(self, other, op="+") + + def __radd__(self, other): + return SymbolicBinop(other, self, op="+") + + def __mul__(self, other): + return SymbolicBinop(self, other, op="*") + + def __rmul__(self, other): + return SymbolicBinop(other, self, op="*") + + def __sub__(self, other): + return SymbolicBinop(self, other, op="-") + + def __rsub__(self, other): + return SymbolicBinop(other, self, op="-") + + def __floordiv__(self, other): + return SymbolicBinop(self, other, op="//") + + def __rdiv__(self, other): + return SymbolicBinop(other, self, op="//") + + def __mod__(self, other): + return SymbolicBinop(self, other, op="%") + + def __rmod__(self, other): + return SymbolicBinop(other, self, op="%") + + def is_region(self): + return False + + +class SymbolicIndexAccess(SymbolicExpr): + __slots__ = ["value", "index"] + + def __init__(self, value, index): + self.value = value + self.index = index + + def __str__(self): + return "%s[%s]" % (self.value, self.index) + + def __repr__(self): + return "%s[%s]" % (self.value, self.index) + + def _legion_postprocess_task_argument(self, point): + result = _postprocess(self.value, point)[_postprocess(self.index, point)] + # FIXME: Clear parent field of regions being used as projection requirements + if isinstance(result, Region): + result.parent = None + return result + + def _legion_symbolize_task_argument(self): + new_value = _symbolize(self.value) + new_index = _symbolize(self.index) + return SymbolicIndexAccess(new_value, new_index) + + def is_region(self): + return isinstance(self.value, Partition) + + @property + def parent(self): + if self.is_region(): + return self.value.parent + assert False + + @property + def fspace(self): + if self.is_region(): + return self.value.parent.fspace + assert False + + def raw_value(self): + if self.is_region(): + return self.value.raw_value() + assert False + + +class SymbolicCall(SymbolicExpr): + __slots__ = ["func", "args"] + + def __init__(self, func, *args): + assert isinstance(func, ProjectionFunctor) + assert len(args) == 1 + assert isinstance(args[0], (SymbolicLoopIndex, ConcreteLoopIndex)) + self.func = func + self.args = args + + def __str__(self): + return "%s(%s)" % (self.func, ", ".join(self.args)) + + def __repr__(self): + return "%s(%s)" % (self.func, ", ".join(self.args)) + + def _legion_postprocess_task_argument(self, point): + return _postprocess(self.func.expr, point) + + def _legion_symbolize_task_argument(self): + symbolized_args = map(_symbolize, self.args) + return SymbolicCall(self.func, symbolized_args) + + +class SymbolicBinop(SymbolicExpr): + import petra as pt + + __slots__ = ["lhs", "rhs", "op"] + + def __init__(self, lhs, rhs, op): + assert op in ["+", "-", "//", "*", "%"] + self.lhs = lhs + self.rhs = rhs + self.op = op + + def __str__(self): + return "%s %s %s" % (self.lhs, self.op, self.rhs) + + def __repr__(self): + return "%s %s %s" % (self.lhs, self.op, self.rhs) + + def get_sides(self): + return (self.lhs, self.op, self.rhs) + + def __eq__(self, other): + if type(other) is type(self): + return self.get_sides() == other.get_sides() + else: + return False + + def __hash__(self): + return hash(self.get_sides()) + + def _legion_symbolize_task_argument(self): + left = _symbolize(self.lhs) + right = _symbolize(self.rhs) + return SymbolicBinop(left, right, self.op) + + def _legion_postprocess_task_argument(self, point): + if self.op == "+": + return _postprocess(self.lhs, point) + _postprocess(self.rhs, point) + elif self.op == "*": + return _postprocess(self.lhs, point) * _postprocess(self.rhs, point) + elif self.op == "-": + return _postprocess(self.lhs, point) - _postprocess(self.rhs, point) + elif self.op == "//": + return _postprocess(self.lhs, point) // _postprocess(self.rhs, point) + elif self.op == "%": + return _postprocess(self.lhs, point) % _postprocess(self.rhs, point) + + def codegen(self): + right = self.rhs.codegen() + left = self.lhs.codegen() + if op == "+": + return pt.Add(left, right) + elif op == "*": + return pt.Mul(left, right) + elif op == "-": + return pt.Sub(left, right) + elif op == "//": + return pt.Div(left, right) + elif op == "%": + return pt.Mod(left, right) + else: + assert False + + +class SymbolicLoopIndex(SymbolicExpr): + import petra as pt + + __slots__ = ["name"] + + def __init__(self, name): + self.name = name + + def __str__(self): + return self.name + + def __repr__(self): + return self.name + + def __eq__(self, other): + if type(other) is type(self): + return self.name == other.name + else: + return False + + def __hash__(self): + return hash(self.name) + + def _legion_symbolize_task_argument(self): + return self + + def _legion_postprocess_task_argument(self, point): + return point + + def codegen(self): + return pt.Var(self.name) + + +ID = SymbolicLoopIndex("ID") + + +class ConcreteLoopIndex(SymbolicExpr): + __slots__ = ["value"] + + def __init__(self, value): + self.value = value + + def __int__(self): + return self.value.__int__() + + def __index__(self): + return self.value.__index__() + + def __str__(self): + return str(self.value) + + def __repr__(self): + return repr(self.value) + + def __eq__(self, other): + if type(other) is type(self): + return self.value == other.value + else: + return False + + def __hash__(self): + return hash(self.value) + + def _legion_symbolize_task_argument(self): + return ID + + def _legion_postprocess_task_argument(self, point): + return self.value + + +_next_proj_functor_id = 100 +engines = [] +programs = [] + +class ProjectionFunctor(object): + __slots__ = ["expr", "proj_id"] + + @staticmethod + def create(expr, force=False): + if not isinstance(expr, SymbolicExpr): + raise Exception( + "ProjectionFunctor requires a symbolic expression as an argument" + ) + if expr in _proj_functor_cache: + return _proj_functor_cache[expr] + assert not force + curr_val = ProjectionFunctor(expr, force) + _proj_functor_cache[expr] = curr_val + return curr_val + + def __init__(self, expr, force=False): + assert not force + if not isinstance(expr, SymbolicExpr): + raise Exception( + "ProjectionFunctor requires a symbolic expression as an argument" + ) + self.expr = expr + self.compile_and_register() + + def __call__(self, *args, **kwargs): + return SymbolicCall(self, *args, **kwargs) + + def compile_and_register(self): + global _next_proj_functor_id + global engines + global programs + self.proj_id = _next_proj_functor_id + _next_proj_functor_id += 1 + + proj_name = "proj_functor_%s" % self.proj_id + + import petra as pt + + rhs = self.expr.rhs + lhs = self.expr.lhs + + if isinstance(lhs, int): + nbr = lhs + elif isinstance(rhs, int): + nbr = rhs + + LEGION_MAX_DIM = _max_dim + MAX_DOMAIN_DIM = 2 * LEGION_MAX_DIM + DIM = 1 + + program = pt.Program("module") + programs.append(program) + + # Define types: + legion_region_tree_id_t = pt.Int32_t # unsigned int + legion_index_partition_id_t = pt.Int32_t # unsigned int + legion_index_tree_id_t = pt.Int32_t # unsigned int + legion_type_tag_t = pt.Int32_t # unsigned int + legion_field_space_id_t = pt.Int32_t # unsigned int + coord_t = pt.Int64_t # long long + legion_index_space_id_t = pt.Int32_t # unsigned int + realm_id_t = pt.Int64_t # unsigned long long + + legion_runtime_t = pt.PointerType(pt.Int8_t) + + legion_index_partition_t = pt.StructType( + { + "id": legion_index_partition_id_t, + "tid": legion_index_tree_id_t, + "type_tag": legion_type_tag_t, + } + ) + + legion_field_space_t = pt.StructType({"id": legion_field_space_id_t}) + + legion_logical_partition_t = pt.StructType( + { + "tree_id": legion_region_tree_id_t, + "index_partition": legion_index_partition_t, + "field_space": legion_field_space_t, + } + ) + + legion_domain_point_t = pt.StructType( + {"dim": pt.Int32_t, "point_data": pt.ArrayType(coord_t, LEGION_MAX_DIM)} + ) + + legion_domain_t = pt.StructType( + { + "is_id": realm_id_t, + "dim": pt.Int32_t, + "rect_data": pt.ArrayType(coord_t, MAX_DOMAIN_DIM), + } + ) + + legion_point_1d_t = pt.Int64_t + + legion_index_space_t = pt.StructType( + { + "id": legion_index_space_id_t, + "tid": legion_index_tree_id_t, + "type_tag": legion_type_tag_t, + } + ) + + legion_logical_region_t = pt.StructType( + { + "tree_id": legion_region_tree_id_t, + "index_space": legion_index_space_t, + "field_space": legion_field_space_t, + } + ) + + # Define functions: + program.add_func_decl( + "legion_domain_point_get_point_1d", + (pt.PointerType(legion_domain_point_t),), + legion_point_1d_t, + attributes=(("byval",),), + ) + + program.add_func_decl( + "legion_domain_point_from_point_1d", + (pt.PointerType(legion_domain_point_t), legion_point_1d_t,), + (), + attributes=(("sret",), None), + ) + program.add_func_decl( + "legion_logical_partition_get_logical_subregion_by_color_domain_point", + ( + pt.PointerType(legion_logical_region_t), + legion_runtime_t, + pt.PointerType(legion_logical_partition_t), + pt.PointerType(legion_domain_point_t), + ), + (), + attributes=(("sret",), None, ("byval",), ("byval",),), + ) + program.add_func_decl( + "malloc", (pt.Int32_t,), pt.PointerType(legion_domain_point_t) + ) + program.add_func_decl("free", (pt.PointerType(legion_domain_point_t),), ()) + + # Define variables: + runtime = pt.Symbol(legion_runtime_t, "runtime") + parent_ptr = pt.Symbol(pt.PointerType(legion_logical_partition_t), "parent_ptr") + point_ptr = pt.Symbol(pt.PointerType(legion_domain_point_t), "point_ptr") + domain_ptr = pt.Symbol(pt.PointerType(legion_domain_t), "domain_ptr") + point1d = pt.Symbol(legion_point_1d_t, "point1d") + point1d_x_plus_1 = pt.Symbol(legion_point_1d_t, "point1d_x_plus_1") + domain_point_x_plus_1_ptr = pt.Symbol( + pt.PointerType(legion_domain_point_t), "point1d_x_plus_1_ptr" + ) + result_ptr = pt.Symbol(pt.PointerType(legion_logical_region_t), "result_ptr") + + target_machine = program.get_target_machine() + + program.add_func( + proj_name, + (result_ptr, runtime, parent_ptr, point_ptr, domain_ptr,), + (), + pt.Block( + [ + pt.DefineVar( + point1d, + pt.Call( + "legion_domain_point_get_point_1d", + [pt.Var(point_ptr),], + attributes=("byval",), + ), + ), + pt.DefineVar( + point1d_x_plus_1, pt.Add(pt.Var(point1d), pt.Int64(nbr)) + ), + pt.DefineVar( + domain_point_x_plus_1_ptr, + pt.Call( + "malloc", + [ + pt.Int32( + legion_domain_point_t.llvm_type().get_abi_size( + target_machine.target_data + ) + ), + ], + ), + ), + pt.Call( + "legion_domain_point_from_point_1d", + [pt.Var(domain_point_x_plus_1_ptr), pt.Var(point1d_x_plus_1),], + attributes=("sret",), + ), + pt.Call( + "legion_logical_partition_get_logical_subregion_by_color_domain_point", + [ + pt.Var(result_ptr), + pt.Var(runtime), + pt.Var(parent_ptr), + pt.Var(domain_point_x_plus_1_ptr), + ], + attributes=("sret", None, "byval", "byval"), + ), + pt.Call("free", [pt.Var(domain_point_x_plus_1_ptr),]), + pt.Return(()), + ] + ), + attributes=(("noalias", "sret"), None, ("byval",), ("byval",), ("byval",)), + ) + + engine = program.compile() + engines.append(engine) + + proj_functor = engine.get_function_address(proj_name) + + c.legion_runtime_register_projection_functor( + _my.ctx.runtime, + self.proj_id, + False, + 0, + ffi.NULL, + ffi.cast("legion_projection_functor_logical_partition_t", proj_functor), + ) + + +def index_launch(domain, task, *args, **kwargs): + def parse_kwargs(reduce=None, mapper=0, tag=0): + return reduce, mapper, tag + + reduce, mapper, tag = parse_kwargs(**kwargs) + + if isinstance(domain, Domain): + domain = domain + elif isinstance(domain, Ispace): + domain = domain.domain + else: + domain = Domain(domain) + launcher = _IndexLauncher(task=task, domain=domain, mapper=mapper, tag=tag) + args, futures = launcher.gather_futures(args) + launcher.attach_global_args(*args) + launcher.attach_future_args(*futures) + launcher.set_reduction_op(reduce) + launcher.launch() + return launcher.future_map + + +class IndexLaunch(object): + __slots__ = [ + "domain", + "mapper", + "tag", + "launcher", + "point", + "saved_task", + "saved_args", + ] + + def __init__(self, domain, **kwargs): + # Hack: workaround for Python 2 not having keyword-only arguments + def validate_spawn_task_args(mapper=0, tag=0): + return mapper, tag + + mapper, tag = validate_spawn_task_args(**kwargs) + + if isinstance(domain, Domain): + self.domain = domain + elif isinstance(domain, Ispace): + self.domain = domain.domain + else: + self.domain = Domain(domain) + self.mapper = mapper + self.tag = tag + self.launcher = None + self.point = None + self.saved_task = None + self.saved_args = None + + def __iter__(self): + _my.ctx.begin_launch(self) + self.point = ConcreteLoopIndex(None) + for i in self.domain: + self.point.value = i + yield self.point + _my.ctx.end_launch(self) + self.launch() + + def ensure_launcher(self, task): + if self.launcher is None: + self.launcher = _IndexLauncher( + task=task, domain=self.domain, mapper=self.mapper, tag=self.tag + ) + + def check_compatibility(self, task, *args): + # The tasks in a launch must conform to the following constraints: + # * Only one task can be launched. + # * The arguments must be compatible: + # * At a given argument position, the value must always + # be a special value, or always not. + # * Special values include: regions and futures. + # * If a region, the value must be symbolic (i.e. able + # to be analyzed as a function of the index expression). + # * If a future, the values must be literally identical + # (i.e. each argument slot in the launch can only + # accept a single future value.) + + if self.saved_task is None: + self.saved_task = task + if task != self.saved_task: + raise Exception("An IndexLaunch may contain only one task launch") + + if self.saved_args is None: + self.saved_args = args + for arg, saved_arg in zip_longest(args, self.saved_args): + # TODO: Add support for region arguments + if isinstance(arg, Region) or isinstance(arg, RegionField): + if arg != saved_arg: + raise Exception( + "Region argument to IndexLaunch does not match previous value at this position" + ) + elif isinstance(arg, Future): + if arg != saved_arg: + raise Exception( + "Future argument to IndexLaunch does not match previous value at this position" + ) + + def spawn_task(self, task, *args): + self.ensure_launcher(task) + self.check_compatibility(task, *args) + args, futures = self.launcher.gather_futures(args) + self.launcher.attach_local_args(self.point, *args) + self.launcher.attach_region_args(*args) + self.launcher.attach_future_args(*futures) + return _FuturePoint(self.launcher, self.point.value) + + def launch(self): + self.launcher.launch() + + +class MustEpochLaunch(object): + __slots__ = ["domain", "launcher"] + + def __init__(self, domain=None): + if isinstance(domain, Domain): + self.domain = domain + elif isinstance(domain, Ispace): + self.domain = ispace.domain + elif domain is not None: + self.domain = Domain(domain) + else: + self.domain = None + self.launcher = None + + def __enter__(self): + self.launcher = _MustEpochLauncher(domain=self.domain) + _my.ctx.begin_launch(self) + + def __exit__(self, exc_type, exc_value, tb): + _my.ctx.end_launch(self) + if exc_value is None: + self.launch() + del self.launcher + + def spawn_task(self, *args, **kwargs): + # TODO: Support index launches + TaskLaunch().spawn_task(*args, **kwargs) + + # TODO: Support return values + + def attach_task_launcher(self, *args, **kwargs): + self.launcher.attach_task_launcher(*args, **kwargs) + + def attach_index_launcher(self, *args, **kwargs): + self.launcher.attach_index_launcher(*args, **kwargs) + + def launch(self): + self.launcher.launch() + + +def execution_fence(block=False, future=False): + f = Future.from_cdata( + c.legion_runtime_issue_execution_fence(_my.ctx.runtime, _my.ctx.context), + value_type=void, + ) + if block or future: + if block: + f.get() + if future: + return f + + +def print_once(*args, **kwargs): + fd = (kwargs["file"] if "file" in kwargs else sys.stdout).fileno() + message = StringIO() + kwargs["file"] = message + print(*args, **kwargs) + c.legion_runtime_print_once_fd( + _my.ctx.runtime, + _my.ctx.context, + fd, + "w".encode("utf-8"), + message.getvalue().encode("utf-8"), + ) + + +class Tunable(object): + # FIXME: Deduplicate this with DefaultMapper::DefaultTunables + NODE_COUNT = 0 + LOCAL_CPUS = 1 + LOCAL_GPUS = 2 + LOCAL_IOS = 3 + LOCAL_OMPS = 4 + LOCAL_PYS = 5 + GLOBAL_CPUS = 6 + GLOBAL_GPUS = 7 + GLOBAL_IOS = 8 + GLOBAL_OMPS = 9 + GLOBAL_PYS = 10 + + @staticmethod + def select(tunable_id): + result = c.legion_runtime_select_tunable_value( + _my.ctx.runtime, _my.ctx.context, tunable_id, 0, 0 + ) + future = Future.from_cdata(result, value_type=uint64) + c.legion_future_destroy(result) + return future + + +class Trace(object): + __slots__ = ["trace_id"] + + def __init__(self): + self.trace_id = _my.ctx.next_trace_id + _my.ctx.next_trace_id += 1 + + def __enter__(self): + c.legion_runtime_begin_trace( + _my.ctx.runtime, _my.ctx.context, self.trace_id, True + ) + + def __exit__(self, exc_type, exc_value, tb): + c.legion_runtime_end_trace(_my.ctx.runtime, _my.ctx.context, self.trace_id) + + +if is_script: + _my.ctx = Context( + legion_top.top_level.context, + legion_top.top_level.runtime, + legion_top.top_level.task, + [], + ) + + def _cleanup(): + del _my.ctx + + legion_top.cleanup_items.append(_cleanup) + + # FIXME: Really this should be the number of control replicated shards at this level + c.legion_runtime_enable_scheduler_lock() + num_procs = Tunable.select(Tunable.GLOBAL_PYS).get() + c.legion_runtime_disable_scheduler_lock() + + global_task_registration_barrier = c.legion_phase_barrier_create( + _my.ctx.runtime, _my.ctx.context, num_procs + ) +elif is_legion_python: + print("WARNING: Executing Python modules via legion_python has been deprecated.") + print("It is now recommended to run the script directly by passing the path") + print("to legion_python.") + print() diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index b7a5f1bce4..dc2f3e3143 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -31,14 +31,11 @@ add_subdirectory(spmd_cgsolver) add_subdirectory(virtual_map) add_subdirectory(attach_2darray_c_fortran_layout) add_subdirectory(attach_array_daxpy) +add_subdirectory(implicit_top_task) if(Legion_MPI_INTEROP) add_subdirectory(mpi_interop) -endif() - -# implicit_top_task doesn't work in master with multiple nodes -if(NOT Legion_NETWORKS) - add_subdirectory(implicit_top_task) + add_subdirectory(mpi_with_ctrl_repl) endif() if(Legion_USE_CUDA) diff --git a/examples/circuit/circuit.cc b/examples/circuit/circuit.cc index 1165c7afaa..d9fc512915 100644 --- a/examples/circuit/circuit.cc +++ b/examples/circuit/circuit.cc @@ -224,6 +224,9 @@ int main(int argc, char **argv) { TaskVariantRegistrar registrar(TOP_LEVEL_TASK_ID, "top_level"); registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); +#ifndef SEQUENTIAL_LOAD_CIRCUIT + registrar.set_replicable(); +#endif Runtime::preregister_task_variant(registrar, "top_level"); } diff --git a/examples/ghost/ghost.cc b/examples/ghost/ghost.cc index 49860080f5..43593292e4 100644 --- a/examples/ghost/ghost.cc +++ b/examples/ghost/ghost.cc @@ -257,6 +257,8 @@ void top_level_task(const Task *task, DomainPoint point(color); must_epoch_launcher.add_single_task(point, spmd_launcher); } + // Specify our launch domain + must_epoch_launcher.launch_domain = Domain(Point<1>(0), Point<1>(num_subregions-1)); FutureMap fm = runtime->execute_must_epoch(ctx, must_epoch_launcher); // wait for completion at least fm.wait_all_results(); @@ -618,6 +620,7 @@ int main(int argc, char **argv) { TaskVariantRegistrar registrar(TOP_LEVEL_TASK_ID, "top_level"); registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); + registrar.set_replicable(); Runtime::preregister_task_variant(registrar, "top_level"); } diff --git a/examples/ghost_pull/ghost.cc b/examples/ghost_pull/ghost.cc index 9c01a63394..218b574419 100644 --- a/examples/ghost_pull/ghost.cc +++ b/examples/ghost_pull/ghost.cc @@ -232,6 +232,8 @@ void top_level_task(const Task *task, DomainPoint point(my_color); must_epoch_launcher.add_single_task(point, spmd_launcher); } + // Specify our launch domain + must_epoch_launcher.launch_domain = Domain(Point<1>(0), Point<1>(num_subregions-1)); FutureMap fm = runtime->execute_must_epoch(ctx, must_epoch_launcher); // wait for completion at least fm.wait_all_results(); @@ -675,6 +677,7 @@ int main(int argc, char **argv) { TaskVariantRegistrar registrar(TOP_LEVEL_TASK_ID, "top_level"); registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); + registrar.set_replicable(); Runtime::preregister_task_variant(registrar, "top_level"); } diff --git a/examples/mpi_with_ctrl_repl/CMakeLists.txt b/examples/mpi_with_ctrl_repl/CMakeLists.txt new file mode 100644 index 0000000000..704d459fa3 --- /dev/null +++ b/examples/mpi_with_ctrl_repl/CMakeLists.txt @@ -0,0 +1,32 @@ +#------------------------------------------------------------------------------# +# Copyright 2019 Stanford University +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#------------------------------------------------------------------------------# + +cmake_minimum_required(VERSION 3.1) +project(LegionExample_mpi_with_ctrl_repl) + +# Only search if were building stand-alone and not as part of Legion +if(NOT Legion_SOURCE_DIR) + find_package(Legion REQUIRED) +endif() + +find_package(MPI REQUIRED) + +add_executable(mpi_with_ctrl_repl mpi_with_ctrl_repl.cc) +target_link_libraries(mpi_with_ctrl_repl Legion::Legion ${MPI_CXX_LIBRARIES}) +target_include_directories(mpi_with_ctrl_repl PRIVATE ${MPI_C_INCLUDE_PATH}) +if(Legion_ENABLE_TESTING) + add_test(NAME mpi_with_ctrl_repl COMMAND ${Legion_TEST_LAUNCHER} $ ${Legion_TEST_ARGS}) +endif() diff --git a/examples/mpi_with_ctrl_repl/Makefile b/examples/mpi_with_ctrl_repl/Makefile new file mode 100644 index 0000000000..ead8bf5dfb --- /dev/null +++ b/examples/mpi_with_ctrl_repl/Makefile @@ -0,0 +1,49 @@ +# Copyright 2019 Stanford University, NVIDIA Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +ifndef LG_RT_DIR +$(error LG_RT_DIR variable is not defined, aborting build) +endif + +# Flags for directing the runtime makefile what to include +DEBUG ?= 1 # Include debugging symbols +OUTPUT_LEVEL ?= LEVEL_DEBUG # Compile time logging level +USE_CUDA ?= 0 # Include CUDA support (requires CUDA) +USE_GASNET ?= 1 # Include GASNet support (requires GASNet) +USE_HDF ?= 0 # Include HDF5 support (requires HDF5) +ALT_MAPPERS ?= 0 # Include alternative mappers (not recommended) + +# Put the binary file name here +OUTFILE ?= mpi_with_ctrl_repl +# List all the application source files here +GEN_SRC ?= mpi_with_ctrl_repl.cc # .cc files +GEN_GPU_SRC ?= # .cu files + +# You can modify these variables, some will be appended to by the runtime makefile +INC_FLAGS ?= +CC_FLAGS ?= +NVCC_FLAGS ?= +GASNET_FLAGS ?= +LD_FLAGS ?= + +########################################################################### +# +# Don't change anything below here +# +########################################################################### + +include $(LG_RT_DIR)/runtime.mk + diff --git a/examples/mpi_with_ctrl_repl/mpi_with_ctrl_repl.cc b/examples/mpi_with_ctrl_repl/mpi_with_ctrl_repl.cc new file mode 100644 index 0000000000..c61764c279 --- /dev/null +++ b/examples/mpi_with_ctrl_repl/mpi_with_ctrl_repl.cc @@ -0,0 +1,220 @@ +/* Copyright 2020 Stanford University, NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//////////////////////////////////////////////////////////// +// +// This example must be built with a Realm network layer +// that is compatible with MPI (e.g. a GASNet conduit that +// supports (and is built with) --enable-mpi-compat, or the +// MPI network layer). +// +// Any network layer that uses MPI for any communication +// during the application (rather than just for bootstrapping) +// additionally requires that the MPI implementation support +// MPI_THREAD_MULTIPLE. +// +//////////////////////////////////////////////////////////// + +#include +// Need MPI header file +#include + +#include "legion.h" + +using namespace Legion; + +enum TaskID +{ + TOP_LEVEL_TASK_ID, + WORKER_TASK_ID, +}; + +// Here is our global MPI-Legion handshake +// You can have as many of these as you +// want but the common case is just to +// have one per Legion-MPI rank pair +MPILegionHandshake handshake; + +// Have a global static number of iterations for +// this example, but you can easily configure it +// from command line arguments which get passed +// to both MPI and Legion +const int total_iterations = 10; + +void worker_task(const Task *task, + const std::vector ®ions, + Context ctx, Runtime *runtime) +{ + printf("Legion Doing Work in Rank %lld\n", + task->index_point[0]); +} + +void top_level_task(const Task *task, + const std::vector ®ions, + Context ctx, Runtime *runtime) +{ + // The default mapper currently has the policy that it will + // launch one top-level task per process if the top-level task + // is control replicable + + // Now we can use our local processor to figure out our corresponding MPI rank + const Processor local_proc = runtime->get_executing_processor(ctx); + const AddressSpace as = local_proc.address_space(); + // Get the mapping from address spaces to MPI ranks + const std::map &reverse_mapping = + runtime->find_reverse_MPI_mapping(); + std::map::const_iterator finder = + reverse_mapping.find(as); + assert(finder != reverse_mapping.end()); + const int rank = finder->second;; + printf("Hello from Legion Top-Level Task in Address Space %d " + "with MPI rank %d\n", as, rank); + + // Iterate and perform the handshakes, we use an index space task + // launch to perform the work since we want one worker task per rank + const Rect<1> launch_bounds(0,reverse_mapping.size()- 1); + const ArgumentMap args_map; + for (int i = 0; i < total_iterations; i++) + { + // Legion can interop with MPI in blocking and non-blocking + // ways. You can use the calls to 'legion_wait_on_mpi' and + // 'legion_handoff_to_mpi' in the same way as the MPI thread + // does. Alternatively, you can get a phase barrier associated + // with a LegionMPIHandshake object which will allow you to + // continue launching more sub-tasks without blocking. + // For deferred execution we prefer the later style, but + // both will work correctly. + if (i < (total_iterations/2)) + { + // This is the blocking way of using handshakes, it + // is not the ideal way, but it works correctly + // Wait for MPI to give us control to run our worker + // This is a blocking call + handshake.legion_wait_on_mpi(); + // Launch our worker task + IndexLauncher worker_launcher(WORKER_TASK_ID, launch_bounds, + TaskArgument(NULL, 0), args_map); + FutureMap fm = runtime->execute_index_space(ctx, worker_launcher); + // Have to wait for the result before signaling MPI + fm.wait_all_results(); + // Perform a non-blocking call to signal + // MPI that we are giving it control back + handshake.legion_handoff_to_mpi(); + } + else + { + // This is the preferred way of using handshakes in Legion + IndexLauncher worker_launcher(WORKER_TASK_ID, launch_bounds, + TaskArgument(NULL, 0), args_map); + // We can user our handshake as a phase barrier + // Record that we will wait on this handshake + worker_launcher.add_wait_handshake(handshake); + // Advance the handshake to the next version + handshake.advance_legion_handshake(); + // Then record that we will arrive on this versions + worker_launcher.add_arrival_handshake(handshake); + // Launch our worker task + // No need to wait for anything + runtime->execute_index_space(ctx, worker_launcher); + } + } +} + +int main(int argc, char **argv) +{ +#if defined(GASNET_CONDUIT_MPI) || defined(REALM_USE_MPI) + // The GASNet MPI conduit and/or the Realm MPI network layer + // require that MPI be initialized for multiple threads + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + // If you fail this assertion, then your version of MPI + // does not support calls from multiple threads and you + // cannot use the GASNet MPI conduit + if (provided < MPI_THREAD_MULTIPLE) + printf("ERROR: Your implementation of MPI does not support " + "MPI_THREAD_MULTIPLE which is required for use of the " + "GASNet MPI conduit or the Realm MPI network layer " + "with the Legion-MPI Interop!\n"); + assert(provided == MPI_THREAD_MULTIPLE); +#else + // Perform MPI start-up like normal for most GASNet conduits + MPI_Init(&argc, &argv); +#endif + + int rank = -1, size = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + printf("Hello from MPI process %d of %d\n", rank, size); + + // Configure the Legion runtime with the rank of this process + Runtime::configure_MPI_interoperability(rank); + // Register our task variants + { + TaskVariantRegistrar top_level_registrar(TOP_LEVEL_TASK_ID); + top_level_registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); + // Mark that the top-level task is control replicable + top_level_registrar.set_replicable(); + Runtime::preregister_task_variant(top_level_registrar, + "Top Level Task"); + Runtime::set_top_level_task_id(TOP_LEVEL_TASK_ID); + } + { + TaskVariantRegistrar worker_task_registrar(WORKER_TASK_ID); + worker_task_registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); + Runtime::preregister_task_variant(worker_task_registrar, + "Worker Task"); + } + // Create a handshake for passing control between Legion and MPI + // Indicate that MPI has initial control and that there is one + // participant on each side + handshake = Runtime::create_handshake(true/*MPI initial control*/, + 1/*MPI participants*/, + 1/*Legion participants*/); + // Start the Legion runtime in background mode + // This call will return immediately + Runtime::start(argc, argv, true/*background*/); + // Run your MPI program like normal + // If you want strict bulk-synchronous execution include + // the barriers protected by this variable, otherwise + // you can elide them, they are not required for correctness + const bool strict_bulk_synchronous_execution = true; + for (int i = 0; i < total_iterations; i++) + { + printf("MPI Doing Work on rank %d\n", rank); + if (strict_bulk_synchronous_execution) + MPI_Barrier(MPI_COMM_WORLD); + // Perform a handoff to Legion, this call is + // asynchronous and will return immediately + handshake.mpi_handoff_to_legion(); + // You can put additional work in here if you like + // but it may interfere with Legion work + + // Wait for Legion to hand control back, + // This call will block until a Legion task + // running in this same process hands control back + handshake.mpi_wait_on_legion(); + if (strict_bulk_synchronous_execution) + MPI_Barrier(MPI_COMM_WORLD); + } + // When you're done wait for the Legion runtime to shutdown + Runtime::wait_for_shutdown(); +#ifndef GASNET_CONDUIT_MPI + // Then finalize MPI like normal + // Exception for the MPI conduit which does its own finalization + MPI_Finalize(); +#endif + + return 0; +} diff --git a/examples/spmd_cgsolver/cgsolver.cc b/examples/spmd_cgsolver/cgsolver.cc index bc5a6c5973..d7619acad1 100644 --- a/examples/spmd_cgsolver/cgsolver.cc +++ b/examples/spmd_cgsolver/cgsolver.cc @@ -1334,6 +1334,7 @@ int main(int argc, char **argv) { TaskVariantRegistrar tvr(TOP_LEVEL_TASK_ID, "top_level_task"); tvr.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); + tvr.set_replicable(); Runtime::preregister_task_variant(tvr, "top_level_task"); Runtime::set_top_level_task_id(TOP_LEVEL_TASK_ID); } diff --git a/examples/virtual_map/virtual_map.cc b/examples/virtual_map/virtual_map.cc index 2051747181..b1c841de2e 100644 --- a/examples/virtual_map/virtual_map.cc +++ b/examples/virtual_map/virtual_map.cc @@ -367,6 +367,7 @@ int main(int argc, char **argv) { TaskVariantRegistrar top_level_registrar(TOP_LEVEL_TASK_ID); top_level_registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); + top_level_registrar.set_replicable(); Runtime::preregister_task_variant(top_level_registrar, "Top Level Task"); Runtime::set_top_level_task_id(TOP_LEVEL_TASK_ID); diff --git a/language/tests/cuda/run_pass/scalar_reduce_multiple.rg b/language/tests/cuda/run_pass/scalar_reduce_multiple.rg index 8063ffdef7..7a119f8508 100644 --- a/language/tests/cuda/run_pass/scalar_reduce_multiple.rg +++ b/language/tests/cuda/run_pass/scalar_reduce_multiple.rg @@ -34,12 +34,14 @@ where reads(r) do var sum1 : double = 0.0 - var sum2 : int = 0 + var sum2 : double = 0.0 + var sum3 : double = 0.0 for e in r do sum1 += @e - sum2 += [int](@e) + sum2 += -(@e) + sum3 += @e end - return [int](sum1) + sum2 + return sum1 + sum2 + sum3 end task main() @@ -48,7 +50,7 @@ task main() var r = region(ispace(int2d, {size, size}), double) init(r, v) var res = red(r) - regentlib.assert(res == [int](2 * r.volume * v), "test failed") + regentlib.assert(res == r.volume * v, "test failed") end regentlib.start(main) diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index fdcaa286e8..859f267ba2 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -451,6 +451,7 @@ list(APPEND LEGION_SRC legion/legion_ops.h legion/legion_ops.cc legion/legion_profiling.h legion/legion_profiling.cc legion/legion_profiling_serializer.h legion/legion_profiling_serializer.cc + legion/legion_replication.h legion/legion_replication.cc legion/legion_spy.h legion/legion_spy.cc legion/legion_tasks.h legion/legion_tasks.cc legion/legion_trace.h legion/legion_trace.cc diff --git a/runtime/legion.h b/runtime/legion.h index 2a54af39fc..428975c00a 100644 --- a/runtime/legion.h +++ b/runtime/legion.h @@ -64,9 +64,6 @@ runtime->print_once(ctx, file, message); \ } -// A guard macro that will exist until control replication is available -#define NO_LEGION_CONTROL_REPLICATION - /** * \namespace Legion * Namespace for all Legion runtime objects @@ -1364,6 +1361,13 @@ namespace Legion { */ void wait_all_results(bool silence_warnings = false, const char *warning_string = NULL) const; + public: + /** + * This method will return the domain of points that can be + * used to index into this future map. + * @return domain of all points in the future map + */ + const Domain& get_future_map_domain(void) const; }; @@ -1552,7 +1556,7 @@ namespace Legion { IndexSpace launch_space; // Will only be used in control replication context. If left // unset the runtime will use launch_domain/launch_space - IndexSpace sharding_space; + IndexSpace sharding_space; std::vector index_requirements; std::vector region_requirements; std::vector futures; @@ -1785,7 +1789,7 @@ namespace Legion { IndexSpace launch_space; // Will only be used in control replication context. If left // unset the runtime will use launch_domain/launch_space - IndexSpace sharding_space; + IndexSpace sharding_space; Predicate predicate; MapperID map_id; MappingTagID tag; @@ -1928,7 +1932,7 @@ namespace Legion { IndexSpace launch_space; // Will only be used in control replication context. If left // unset the runtime will use launch_domain/launch_space - IndexSpace sharding_space; + IndexSpace sharding_space; LogicalRegion region; LogicalPartition partition; LogicalRegion parent; @@ -1969,10 +1973,12 @@ namespace Legion { public: inline void attach_file(const char *file_name, const std::vector &fields, - LegionFileMode mode); + LegionFileMode mode, + bool local_file = false); inline void attach_hdf5(const char *file_name, const std::map &field_map, - LegionFileMode mode); + LegionFileMode mode, + bool local_files = false); // Helper methods for AOS and SOA arrays, but it is totally // acceptable to fill in the layout constraint set manually inline void attach_array_aos(void *base, bool column_major, @@ -1995,6 +2001,7 @@ namespace Legion { LegionFileMode mode; std::vector file_fields; // normal files std::map field_files; // hdf5 files + bool local_files; public: // Data for external instances LayoutConstraintSet constraints; @@ -3325,6 +3332,7 @@ namespace Legion { bool is_index_space; Domain index_domain; DomainPoint index_point; + IndexSpace sharding_space; public: // Parent task for the copy operation const Task* parent_task; @@ -3507,6 +3515,7 @@ namespace Legion { bool is_index_space; Domain index_domain; DomainPoint index_point; + IndexSpace sharding_space; public: // Parent task for the fill operation const Task* parent_task; @@ -3571,7 +3580,7 @@ namespace Legion { FRIEND_ALL_RUNTIME_CLASSES MustEpoch(void); public: - virtual MappableType get_mappable_type(void) const + virtual MappableType get_mappable_type(void) const { return MUST_EPOCH_MAPPABLE; } virtual const Task* as_task(void) const { return NULL; } virtual const Copy* as_copy(void) const { return NULL; } @@ -3838,8 +3847,8 @@ namespace Legion { */ class ShardingFunctor { public: - ShardingFunctor(void) { } - virtual ~ShardingFunctor(void) { } + ShardingFunctor(void); + virtual ~ShardingFunctor(void); public: virtual ShardID shard(const DomainPoint &point, const Domain &full_space, @@ -7019,6 +7028,11 @@ namespace Legion { //------------------------------------------------------------------------ // MPI Interoperability //------------------------------------------------------------------------ + /** + * @return true if the MPI interop has been established + */ + bool is_MPI_interop_configured(void); + /** * Return a reference to the mapping from MPI ranks to address spaces. * This method is only valid if the static initialization method @@ -7041,11 +7055,6 @@ namespace Legion { * Return the local MPI rank ID for the current Legion runtime */ int find_local_MPI_rank(void); - - /** - * @return true if the MPI interop has been established - */ - bool is_MPI_interop_configured(void); public: //------------------------------------------------------------------------ // Semantic Information @@ -7559,6 +7568,14 @@ namespace Legion { */ static void preregister_sharding_functor(ShardingID sid, ShardingFunctor *functor); + + /** + * Return a pointer to a given sharding functor object. + * The runtime retains ownership of this object. + * @param sid ID of the sharding functor to find + * @return a pointer o the sharding functor if it exists + */ + static ShardingFunctor* get_sharding_functor(ShardingID sid); public: /** * Dynamically generate a unique reduction ID for use across the machine @@ -7796,6 +7813,11 @@ namespace Legion { * checks on mapper calls regardless of the * optimization level. (Default: true in debug mode, * false in release mode.) + * -lg:safe_ctrlrepl Perform dynamic checks to verify the correctness + * of control replication. This will compute a hash + * all the arguments to each call into the runtime + * and perform a collective to compare it across + * the shards to see if they all align. * -lg:local Specify the maximum number of local fields * permitted in any field space within a context. * --------------------- @@ -8569,24 +8591,26 @@ namespace Legion { PartitionKind part_kind, Color color); IndexSpace create_index_space_union_internal(Context ctx, IndexPartition parent, - const void *realm_color, TypeTag type_tag, + const void *realm_color,size_t color_size, + TypeTag type_tag, const std::vector &handles); IndexSpace create_index_space_union_internal(Context ctx, IndexPartition parent, - const void *realm_color, TypeTag type_tag, - IndexPartition handle); + const void *realm_color,size_t color_size, + TypeTag type_tag, IndexPartition handle); IndexSpace create_index_space_intersection_internal(Context ctx, IndexPartition parent, - const void *realm_color, TypeTag type_tag, + const void *realm_color,size_t color_size, + TypeTag type_tag, const std::vector &handles); IndexSpace create_index_space_intersection_internal(Context ctx, IndexPartition parent, - const void *realm_color, TypeTag type_tag, - IndexPartition handle); + const void *realm_color,size_t color_size, + TypeTag type_tag, IndexPartition handle); IndexSpace create_index_space_difference_internal(Context ctx, IndexPartition paretn, - const void *realm_color, TypeTag type_tag, - IndexSpace initial, + const void *realm_color, size_t color_size, + TypeTag type_tag, IndexSpace initial, const std::vector &handles); IndexSpace get_index_subspace_internal(IndexPartition handle, const void *realm_color,TypeTag type_tag); @@ -8628,6 +8652,26 @@ namespace Legion { // We'll also allow users to get the total number of shards in the context // if they also ar willing to attest they know what they are doing size_t get_num_shards(Context ctx, bool I_know_what_I_am_doing = false); + // This is another hidden method for control replication because it's + // still somewhat experimental. In some cases there are unavoidable + // sources of randomness that can mess with the needed invariants for + // control replication (e.g. garbage collectors). This method will + // allow the application to pass in an array of elements from each shard + // and the runtime will fill in an output buffer with an ordered array of + // elements that were passed in by every shard. Each shard will get the + // same elements that were present in all the other shards in the same + // order in the output array. The number of elements in the output buffer + // is returned as a future (of type size_t) as the runtime will return + // immediately and the application can continue running ahead. The + // application must keep the input and output buffers allocated until + // the future resolves. By definition the output buffer need be no bigger + // than the input buffer since only elements that are in the input buffer + // on any shard can appear in the output buffer. Note you can use this + // method safely in contexts that are not control replicated as well: + // the input will just be mem-copied to the output and num_elements + // returned as the future result. + Future consensus_match(Context ctx, const void *input, void *output, + size_t num_elements, size_t element_size); private: friend class Mapper; Internal::Runtime *runtime; diff --git a/runtime/legion/garbage_collection.cc b/runtime/legion/garbage_collection.cc index 1b592e03d1..23b0251643 100644 --- a/runtime/legion/garbage_collection.cc +++ b/runtime/legion/garbage_collection.cc @@ -1402,16 +1402,21 @@ namespace Legion { //-------------------------------------------------------------------------- void DistributedCollectable::update_remote_instances( - AddressSpaceID remote_inst) + AddressSpaceID remote_inst, bool need_lock) //-------------------------------------------------------------------------- { - AutoLock gc(gc_lock); - remote_instances.add(remote_inst); + if (need_lock) + { + AutoLock gc(gc_lock); + remote_instances.add(remote_inst); + } + else + remote_instances.add(remote_inst); } //-------------------------------------------------------------------------- void DistributedCollectable::register_with_runtime( - ReferenceMutator *mutator) + ReferenceMutator *mutator, bool notify_remote) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -1419,7 +1424,7 @@ namespace Legion { #endif registered_with_runtime = true; runtime->register_distributed_collectable(did, this); - if (!is_owner() && (mutator != NULL)) + if (notify_remote && !is_owner() && (mutator != NULL)) send_remote_registration(mutator); } diff --git a/runtime/legion/garbage_collection.h b/runtime/legion/garbage_collection.h index 1a3b326c96..da8773cf96 100644 --- a/runtime/legion/garbage_collection.h +++ b/runtime/legion/garbage_collection.h @@ -43,7 +43,7 @@ namespace Legion { REDUCTION_VIEW_DC = 0x5, FILL_VIEW_DC = 0x6, PHI_VIEW_DC = 0x7, - VERSION_STATE_DC = 0x8, + SHARDED_VIEW_DC = 0x8, FUTURE_DC = 0x9, FUTURE_MAP_DC = 0xA, INDEX_TREE_NODE_DC = 0xB, @@ -84,7 +84,8 @@ namespace Legion { TRACE_REF = 26, AGGREGATORE_REF = 27, FIELD_STATE_REF = 28, - LAST_SOURCE_REF = 29, + REPLICATION_REF = 29, + LAST_SOURCE_REF = 30, }; enum ReferenceKind { @@ -120,10 +121,11 @@ namespace Legion { "Region Tree Reference", \ "Layout Description Reference", \ "Runtime Reference", \ - "Index Space Expression Reference", \ "Physical Trace Reference", \ + "Index Space Expression Reference", \ "Aggregator Reference", \ "Field State Reference", \ + "Replication Reference", \ } extern Realm::Logger log_garbage; @@ -370,14 +372,17 @@ namespace Legion { inline bool is_owner(void) const { return (owner_space == local_space); } inline bool is_registered(void) const { return registered_with_runtime; } bool has_remote_instance(AddressSpaceID remote_space) const; - void update_remote_instances(AddressSpaceID remote_space); + void update_remote_instances(AddressSpaceID remote_space, + bool need_lock = true); public: inline bool has_remote_instances(void) const; + inline size_t count_remote_instances(void) const; template inline void map_over_remote_instances(FUNCTOR &functor); public: // This is for the owner node only - void register_with_runtime(ReferenceMutator *mutator); + void register_with_runtime(ReferenceMutator *mutator, + bool notify_remote = true); protected: void unregister_with_runtime(void) const; RtEvent send_unregister_messages(VirtualChannelKind vc) const; @@ -537,6 +542,14 @@ namespace Legion { return !remote_instances.empty(); } + //-------------------------------------------------------------------------- + inline size_t DistributedCollectable::count_remote_instances(void) const + //-------------------------------------------------------------------------- + { + AutoLock gc(gc_lock,1,false/*exclusive*/); + return remote_instances.size(); + } + //-------------------------------------------------------------------------- template void DistributedCollectable::map_over_remote_instances(FUNCTOR &functor) diff --git a/runtime/legion/legion.cc b/runtime/legion/legion.cc index 207928b385..59cd34ca20 100644 --- a/runtime/legion/legion.cc +++ b/runtime/legion/legion.cc @@ -165,6 +165,17 @@ namespace Legion { { } + ///////////////////////////////////////////////////////////// + // MustEpoch + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + MustEpoch::MustEpoch(void) + : Mappable(), parent_task(NULL) + //-------------------------------------------------------------------------- + { + } + ///////////////////////////////////////////////////////////// // IndexSpace ///////////////////////////////////////////////////////////// @@ -1379,7 +1390,8 @@ namespace Legion { //-------------------------------------------------------------------------- StaticDependence::StaticDependence(unsigned prev, unsigned prev_req, - unsigned current_req, DependenceType dtype, bool val, bool shard) + unsigned current_req, DependenceType dtype, + bool val, bool shard) : previous_offset(prev), previous_req_index(prev_req), current_req_index(current_req), dependence_type(dtype), validates(val), shard_only(shard) @@ -1394,7 +1406,8 @@ namespace Legion { //-------------------------------------------------------------------------- TaskLauncher::TaskLauncher(void) : task_id(0), argument(TaskArgument()), predicate(Predicate::TRUE_PRED), - map_id(0), tag(0), point(DomainPoint()), static_dependences(NULL), + map_id(0), tag(0), point(DomainPoint(0)), + sharding_space(IndexSpace::NO_SPACE), static_dependences(NULL), enable_inlining(false), local_function_task(false), independent_requirements(false), silence_warnings(false) //-------------------------------------------------------------------------- @@ -1406,9 +1419,10 @@ namespace Legion { Predicate pred /*= Predicate::TRUE_PRED*/, MapperID mid /*=0*/, MappingTagID t /*=0*/) : task_id(tid), argument(arg), predicate(pred), map_id(mid), tag(t), - point(DomainPoint()), static_dependences(NULL), - enable_inlining(false), local_function_task(false), - independent_requirements(false), silence_warnings(false) + point(DomainPoint(0)), sharding_space(IndexSpace::NO_SPACE), + static_dependences(NULL), enable_inlining(false), + local_function_task(false), independent_requirements(false), + silence_warnings(false) //-------------------------------------------------------------------------- { } @@ -1420,7 +1434,8 @@ namespace Legion { //-------------------------------------------------------------------------- IndexTaskLauncher::IndexTaskLauncher(void) : task_id(0), launch_domain(Domain::NO_DOMAIN), - launch_space(IndexSpace::NO_SPACE), global_arg(TaskArgument()), + launch_space(IndexSpace::NO_SPACE), + sharding_space(IndexSpace::NO_SPACE), global_arg(TaskArgument()), argument_map(ArgumentMap()), predicate(Predicate::TRUE_PRED), must_parallelism(false), map_id(0), tag(0), static_dependences(NULL), enable_inlining(false), independent_requirements(false), @@ -1437,10 +1452,10 @@ namespace Legion { bool must /*=false*/, MapperID mid /*=0*/, MappingTagID t /*=0*/) : task_id(tid), launch_domain(dom), launch_space(IndexSpace::NO_SPACE), - global_arg(global), argument_map(map), predicate(pred), - must_parallelism(must), map_id(mid), tag(t), static_dependences(NULL), - enable_inlining(false), independent_requirements(false), - silence_warnings(false) + sharding_space(IndexSpace::NO_SPACE), global_arg(global), + argument_map(map), predicate(pred), must_parallelism(must), map_id(mid), + tag(t), static_dependences(NULL), enable_inlining(false), + independent_requirements(false), silence_warnings(false) //-------------------------------------------------------------------------- { } @@ -1454,10 +1469,10 @@ namespace Legion { bool must /*=false*/, MapperID mid /*=0*/, MappingTagID t /*=0*/) : task_id(tid), launch_domain(Domain::NO_DOMAIN), launch_space(space), - global_arg(global), argument_map(map), predicate(pred), - must_parallelism(must), map_id(mid), tag(t), static_dependences(NULL), - enable_inlining(false), independent_requirements(false), - silence_warnings(false) + sharding_space(IndexSpace::NO_SPACE), global_arg(global), + argument_map(map), predicate(pred), must_parallelism(must), map_id(mid), + tag(t), static_dependences(NULL), enable_inlining(false), + independent_requirements(false), silence_warnings(false) //-------------------------------------------------------------------------- { } @@ -1490,7 +1505,8 @@ namespace Legion { //-------------------------------------------------------------------------- CopyLauncher::CopyLauncher(Predicate pred /*= Predicate::TRUE_PRED*/, MapperID mid /*=0*/, MappingTagID t /*=0*/) - : predicate(pred), map_id(mid), tag(t), static_dependences(NULL), + : predicate(pred), map_id(mid), tag(t), point(DomainPoint(0)), + sharding_space(IndexSpace::NO_SPACE), static_dependences(NULL), possible_src_indirect_out_of_range(true), possible_dst_indirect_out_of_range(true), possible_dst_indirect_aliasing(true), silence_warnings(false) @@ -1505,8 +1521,9 @@ namespace Legion { //-------------------------------------------------------------------------- IndexCopyLauncher::IndexCopyLauncher(void) : launch_domain(Domain::NO_DOMAIN), launch_space(IndexSpace::NO_SPACE), - predicate(Predicate::TRUE_PRED), map_id(0), tag(0), - static_dependences(NULL), possible_src_indirect_out_of_range(true), + sharding_space(IndexSpace::NO_SPACE), predicate(Predicate::TRUE_PRED), + map_id(0), tag(0), static_dependences(NULL), + possible_src_indirect_out_of_range(true), possible_dst_indirect_out_of_range(true), possible_dst_indirect_aliasing(true), collective_src_indirect_points(true), @@ -1519,8 +1536,9 @@ namespace Legion { IndexCopyLauncher::IndexCopyLauncher(Domain dom, Predicate pred /*= Predicate::TRUE_PRED*/, MapperID mid /*=0*/, MappingTagID t /*=0*/) - : launch_domain(dom), launch_space(IndexSpace::NO_SPACE), predicate(pred), - map_id(mid),tag(t), static_dependences(NULL), + : launch_domain(dom), launch_space(IndexSpace::NO_SPACE), + sharding_space(IndexSpace::NO_SPACE), predicate(pred), map_id(mid), + tag(t), static_dependences(NULL), possible_src_indirect_out_of_range(true), possible_dst_indirect_out_of_range(true), possible_dst_indirect_aliasing(true), @@ -1534,8 +1552,9 @@ namespace Legion { IndexCopyLauncher::IndexCopyLauncher(IndexSpace space, Predicate pred /*= Predicate::TRUE_PRED*/, MapperID mid /*=0*/, MappingTagID t /*=0*/) - : launch_domain(Domain::NO_DOMAIN), launch_space(space), predicate(pred), - map_id(mid), tag(t), static_dependences(NULL), + : launch_domain(Domain::NO_DOMAIN), launch_space(space), + sharding_space(IndexSpace::NO_SPACE), predicate(pred), map_id(mid), + tag(t), static_dependences(NULL), possible_src_indirect_out_of_range(true), possible_dst_indirect_out_of_range(true), possible_dst_indirect_aliasing(true), @@ -1584,7 +1603,8 @@ namespace Legion { //-------------------------------------------------------------------------- FillLauncher::FillLauncher(void) : handle(LogicalRegion::NO_REGION), parent(LogicalRegion::NO_REGION), - map_id(0), tag(0), static_dependences(NULL), silence_warnings(false) + map_id(0), tag(0), point(DomainPoint(0)), static_dependences(NULL), + silence_warnings(false) //-------------------------------------------------------------------------- { } @@ -1595,7 +1615,8 @@ namespace Legion { Predicate pred /*= Predicate::TRUE_PRED*/, MapperID id /*=0*/, MappingTagID t /*=0*/) : handle(h), parent(p), argument(arg), predicate(pred), map_id(id), - tag(t), static_dependences(NULL), silence_warnings(false) + tag(t), point(DomainPoint(0)), static_dependences(NULL), + silence_warnings(false) //-------------------------------------------------------------------------- { } @@ -1605,7 +1626,7 @@ namespace Legion { Predicate pred /*= Predicate::TRUE_PRED*/, MapperID id /*=0*/, MappingTagID t /*=0*/) : handle(h), parent(p), future(f), predicate(pred), map_id(id), tag(t), - static_dependences(NULL), silence_warnings(false) + point(DomainPoint(0)), static_dependences(NULL), silence_warnings(false) //-------------------------------------------------------------------------- { } @@ -1617,9 +1638,9 @@ namespace Legion { //-------------------------------------------------------------------------- IndexFillLauncher::IndexFillLauncher(void) : launch_domain(Domain::NO_DOMAIN), launch_space(IndexSpace::NO_SPACE), - region(LogicalRegion::NO_REGION), partition(LogicalPartition::NO_PART), - projection(0), map_id(0), tag(0), static_dependences(NULL), - silence_warnings(false) + sharding_space(IndexSpace::NO_SPACE), region(LogicalRegion::NO_REGION), + partition(LogicalPartition::NO_PART), projection(0), map_id(0), tag(0), + static_dependences(NULL), silence_warnings(false) //-------------------------------------------------------------------------- { } @@ -1629,7 +1650,8 @@ namespace Legion { LogicalRegion p, TaskArgument arg, ProjectionID proj, Predicate pred, MapperID id /*=0*/, MappingTagID t /*=0*/) - : launch_domain(dom), launch_space(IndexSpace::NO_SPACE), region(h), + : launch_domain(dom), launch_space(IndexSpace::NO_SPACE), + sharding_space(IndexSpace::NO_SPACE), region(h), partition(LogicalPartition::NO_PART), parent(p), projection(proj), argument(arg), predicate(pred), map_id(id), tag(t), static_dependences(NULL), silence_warnings(false) @@ -1642,7 +1664,8 @@ namespace Legion { LogicalRegion p, Future f, ProjectionID proj, Predicate pred, MapperID id /*=0*/, MappingTagID t /*=0*/) - : launch_domain(dom), launch_space(IndexSpace::NO_SPACE), region(h), + : launch_domain(dom), launch_space(IndexSpace::NO_SPACE), + sharding_space(IndexSpace::NO_SPACE), region(h), partition(LogicalPartition::NO_PART), parent(p), projection(proj), future(f), predicate(pred), map_id(id), tag(t), static_dependences(NULL), silence_warnings(false) @@ -1655,7 +1678,8 @@ namespace Legion { LogicalRegion p, TaskArgument arg, ProjectionID proj, Predicate pred, MapperID id /*=0*/, MappingTagID t /*=0*/) - : launch_domain(Domain::NO_DOMAIN), launch_space(space), region(h), + : launch_domain(Domain::NO_DOMAIN), launch_space(space), + sharding_space(IndexSpace::NO_SPACE), region(h), partition(LogicalPartition::NO_PART), parent(p), projection(proj), argument(arg), predicate(pred), map_id(id), tag(t), static_dependences(NULL), silence_warnings(false) @@ -1668,7 +1692,8 @@ namespace Legion { LogicalRegion p, Future f, ProjectionID proj, Predicate pred, MapperID id /*=0*/, MappingTagID t /*=0*/) - : launch_domain(Domain::NO_DOMAIN), launch_space(space), region(h), + : launch_domain(Domain::NO_DOMAIN), launch_space(space), + sharding_space(IndexSpace::NO_SPACE), region(h), partition(LogicalPartition::NO_PART), parent(p), projection(proj), future(f), predicate(pred), map_id(id), tag(t), static_dependences(NULL), silence_warnings(false) @@ -1683,8 +1708,8 @@ namespace Legion { MapperID id /*=0*/, MappingTagID t /*=0*/) : launch_domain(dom), launch_space(IndexSpace::NO_SPACE), - region(LogicalRegion::NO_REGION), partition(h), - parent(p), projection(proj), argument(arg), predicate(pred), + sharding_space(IndexSpace::NO_SPACE), region(LogicalRegion::NO_REGION), + partition(h), parent(p), projection(proj),argument(arg),predicate(pred), map_id(id), tag(t), static_dependences(NULL), silence_warnings(false) //-------------------------------------------------------------------------- { @@ -1697,8 +1722,8 @@ namespace Legion { MapperID id /*=0*/, MappingTagID t /*=0*/) : launch_domain(dom), launch_space(IndexSpace::NO_SPACE), - region(LogicalRegion::NO_REGION), partition(h), - parent(p), projection(proj), future(f), predicate(pred), + sharding_space(IndexSpace::NO_SPACE), region(LogicalRegion::NO_REGION), + partition(h), parent(p), projection(proj), future(f), predicate(pred), map_id(id), tag(t), static_dependences(NULL), silence_warnings(false) //-------------------------------------------------------------------------- { @@ -1711,8 +1736,8 @@ namespace Legion { MapperID id /*=0*/, MappingTagID t /*=0*/) : launch_domain(Domain::NO_DOMAIN), launch_space(space), - region(LogicalRegion::NO_REGION), partition(h), - parent(p), projection(proj), argument(arg), predicate(pred), + sharding_space(IndexSpace::NO_SPACE), region(LogicalRegion::NO_REGION), + partition(h), parent(p), projection(proj),argument(arg),predicate(pred), map_id(id), tag(t), static_dependences(NULL), silence_warnings(false) //-------------------------------------------------------------------------- { @@ -1725,8 +1750,8 @@ namespace Legion { MapperID id /*=0*/, MappingTagID t /*=0*/) : launch_domain(Domain::NO_DOMAIN), launch_space(space), - region(LogicalRegion::NO_REGION), partition(h), - parent(p), projection(proj), future(f), predicate(pred), + sharding_space(IndexSpace::NO_SPACE), region(LogicalRegion::NO_REGION), + partition(h), parent(p), projection(proj), future(f), predicate(pred), map_id(id), tag(t), static_dependences(NULL), silence_warnings(false) //-------------------------------------------------------------------------- { @@ -1778,7 +1803,9 @@ namespace Legion { //-------------------------------------------------------------------------- MustEpochLauncher::MustEpochLauncher(MapperID id /*= 0*/, MappingTagID tag/*= 0*/) - : map_id(id), mapping_tag(tag), silence_warnings(false) + : map_id(id), mapping_tag(tag), launch_domain(Domain::NO_DOMAIN), + launch_space(IndexSpace::NO_SPACE), + sharding_space(IndexSpace::NO_SPACE), silence_warnings(false) //-------------------------------------------------------------------------- { } @@ -1810,7 +1837,8 @@ namespace Legion { TaskVariantRegistrar::TaskVariantRegistrar(void) : task_id(0), global_registration(true), task_variant_name(NULL), leaf_variant(false), - inner_variant(false), idempotent_variant(false) + inner_variant(false), idempotent_variant(false), + replicable_variant(false) //-------------------------------------------------------------------------- { } @@ -1820,7 +1848,8 @@ namespace Legion { const char *variant_name) : task_id(task_id), global_registration(global), task_variant_name(variant_name), leaf_variant(false), - inner_variant(false), idempotent_variant(false) + inner_variant(false), idempotent_variant(false), + replicable_variant(false) //-------------------------------------------------------------------------- { } @@ -1831,7 +1860,8 @@ namespace Legion { bool global/*=true*/) : task_id(task_id), global_registration(global), task_variant_name(variant_name), leaf_variant(false), - inner_variant(false), idempotent_variant(false) + inner_variant(false), idempotent_variant(false), + replicable_variant(false) //-------------------------------------------------------------------------- { } @@ -2186,7 +2216,7 @@ namespace Legion { #ifdef DEBUG_LEGION assert(impl != NULL); #endif - return impl->get_future(point); + return impl->get_future(point, false/*internal*/); } //-------------------------------------------------------------------------- @@ -2208,6 +2238,16 @@ namespace Legion { impl->wait_all_results(silence_warnings, warning_string); } + //-------------------------------------------------------------------------- + const Domain& FutureMap::get_future_map_domain(void) const + //-------------------------------------------------------------------------- + { + if (impl == NULL) + return Domain::NO_DOMAIN; + else + return impl->future_map_domain; + } + ///////////////////////////////////////////////////////////// // Physical Region ///////////////////////////////////////////////////////////// @@ -3010,8 +3050,7 @@ namespace Legion { //-------------------------------------------------------------------------- LogicalRegion ProjectionFunctor::project(LogicalRegion upper_bound, - const DomainPoint &point, - const Domain &launch_domain) + const DomainPoint &point, const Domain &launch_domain) //-------------------------------------------------------------------------- { REPORT_LEGION_ERROR(ERROR_DEPRECATED_PROJECTION, @@ -3022,8 +3061,7 @@ namespace Legion { //-------------------------------------------------------------------------- LogicalRegion ProjectionFunctor::project(LogicalPartition upper_bound, - const DomainPoint &point, - const Domain &launch_domain) + const DomainPoint &point, const Domain &launch_domain) //-------------------------------------------------------------------------- { REPORT_LEGION_ERROR(ERROR_DEPRECATED_PROJECTION, @@ -3071,6 +3109,22 @@ namespace Legion { // Must be override by derived classes assert(false); } + + ///////////////////////////////////////////////////////////// + // ShardingFunctor + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardingFunctor::ShardingFunctor(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardingFunctor::~ShardingFunctor(void) + //-------------------------------------------------------------------------- + { + } ///////////////////////////////////////////////////////////// // Coloring Serializer @@ -3946,14 +4000,8 @@ namespace Legion { PartitionKind part_kind, Color color) //-------------------------------------------------------------------------- { - ArgumentMap argmap; - for (std::map::const_iterator it = - domains.begin(); it != domains.end(); it++) - argmap.set_point(it->first, - TaskArgument(&it->second, sizeof(it->second))); - FutureMap future_map(argmap.impl->freeze(ctx)); - return ctx->create_partition_by_domain(parent, future_map, color_space, - perform_intersections, part_kind, color); + return ctx->create_partition_by_domain(parent, domains, color_space, + perform_intersections, part_kind, color); } //-------------------------------------------------------------------------- @@ -4047,8 +4095,8 @@ namespace Legion { case DIM: \ { \ Point point = color; \ - return ctx->create_index_space_union(parent, &point, \ - TYPE_TAG_##DIM##D, handles); \ + return ctx->create_index_space_union(parent, &point, sizeof(point),\ + TYPE_TAG_##DIM##D, handles); \ } LEGION_FOREACH_N(DIMFUNC) #undef DIMFUNC @@ -4060,11 +4108,12 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace Runtime::create_index_space_union_internal(Context ctx, - IndexPartition parent, const void *color, TypeTag type_tag, - const std::vector &handles) + IndexPartition parent, const void *color, size_t color_size, + TypeTag type_tag, const std::vector &handles) //-------------------------------------------------------------------------- { - return ctx->create_index_space_union(parent, color, type_tag, handles); + return ctx->create_index_space_union(parent, color, color_size, + type_tag, handles); } //-------------------------------------------------------------------------- @@ -4079,8 +4128,8 @@ namespace Legion { case DIM: \ { \ Point point = color; \ - return ctx->create_index_space_union(parent, &point, \ - TYPE_TAG_##DIM##D, handle); \ + return ctx->create_index_space_union(parent, &point, sizeof(point),\ + TYPE_TAG_##DIM##D, handle); \ } LEGION_FOREACH_N(DIMFUNC) #undef DIMFUNC @@ -4093,10 +4142,11 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace Runtime::create_index_space_union_internal(Context ctx, IndexPartition parent, const void *realm_color, - TypeTag type_tag, IndexPartition handle) + size_t size, TypeTag type_tag, IndexPartition handle) //-------------------------------------------------------------------------- { - return ctx->create_index_space_union(parent, realm_color,type_tag,handle); + return ctx->create_index_space_union(parent, realm_color, size, + type_tag, handle); } //-------------------------------------------------------------------------- @@ -4112,7 +4162,7 @@ namespace Legion { { \ Point point = color; \ return ctx->create_index_space_intersection(parent, &point, \ - TYPE_TAG_##DIM##D, handles); \ + sizeof(point), TYPE_TAG_##DIM##D, handles); \ } LEGION_FOREACH_N(DIMFUNC) #undef DIMFUNC @@ -4124,11 +4174,11 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace Runtime::create_index_space_intersection_internal(Context ctx, - IndexPartition parent, const void *color, TypeTag type_tag, - const std::vector &handles) + IndexPartition parent, const void *color, size_t color_size, + TypeTag type_tag, const std::vector &handles) //-------------------------------------------------------------------------- { - return ctx->create_index_space_intersection(parent, color, + return ctx->create_index_space_intersection(parent, color, color_size, type_tag, handles); } @@ -4145,7 +4195,7 @@ namespace Legion { { \ Point point = color; \ return ctx->create_index_space_intersection(parent, &point, \ - TYPE_TAG_##DIM##D, handle); \ + sizeof(point), TYPE_TAG_##DIM##D, handle); \ } LEGION_FOREACH_N(DIMFUNC) #undef DIMFUNC @@ -4157,12 +4207,12 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace Runtime::create_index_space_intersection_internal(Context ctx, - IndexPartition parent, const void *realm_color, - TypeTag type_tag, IndexPartition handle) + IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) //-------------------------------------------------------------------------- { - return ctx->create_index_space_intersection(parent, realm_color, - type_tag, handle); + return ctx->create_index_space_intersection(parent, realm_color, + color_size, type_tag, handle); } //-------------------------------------------------------------------------- @@ -4178,7 +4228,7 @@ namespace Legion { { \ Point point = color; \ return ctx->create_index_space_difference(parent, &point, \ - TYPE_TAG_##DIM##D, initial, handles); \ + sizeof(point), TYPE_TAG_##DIM##D, initial, handles); \ } LEGION_FOREACH_N(DIMFUNC) #undef DIMFUNC @@ -4190,12 +4240,13 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace Runtime::create_index_space_difference_internal(Context ctx, - IndexPartition parent, const void *realm_color, TypeTag type_tag, - IndexSpace initial, const std::vector &handles) + IndexPartition parent, const void *realm_color, size_t color_size, + TypeTag type_tag, IndexSpace initial, + const std::vector &handles) //-------------------------------------------------------------------------- { - return ctx->create_index_space_difference(parent, realm_color, type_tag, - initial, handles); + return ctx->create_index_space_difference(parent, realm_color, color_size, + type_tag, initial, handles); } //-------------------------------------------------------------------------- @@ -5256,7 +5307,8 @@ namespace Legion { FieldAllocator Runtime::create_field_allocator(Context ctx,FieldSpace space) //-------------------------------------------------------------------------- { - return FieldAllocator(ctx->create_field_allocator(space)); + return FieldAllocator( + ctx->create_field_allocator(space, false/*unordered*/)); } //-------------------------------------------------------------------------- @@ -5943,6 +5995,13 @@ namespace Legion { runtime->yield(ctx); } + //-------------------------------------------------------------------------- + bool Runtime::is_MPI_interop_configured(void) + //-------------------------------------------------------------------------- + { + return runtime->is_MPI_interop_configured(); + } + //-------------------------------------------------------------------------- const std::map& Runtime::find_forward_MPI_mapping(void) @@ -5966,13 +6025,6 @@ namespace Legion { return runtime->find_local_MPI_rank(); } - //-------------------------------------------------------------------------- - bool Runtime::is_MPI_interop_configured(void) - //-------------------------------------------------------------------------- - { - return runtime->is_MPI_interop_configured(); - } - //-------------------------------------------------------------------------- Mapping::MapperRuntime* Runtime::get_mapper_runtime(void) //-------------------------------------------------------------------------- @@ -6069,25 +6121,22 @@ namespace Legion { ShardingID Runtime::generate_dynamic_sharding_id(void) //-------------------------------------------------------------------------- { - // Not implemented until control replication - return 0; + return runtime->generate_dynamic_sharding_id(); } //-------------------------------------------------------------------------- - ShardingID Runtime::generate_library_sharding_ids( - const char *name, size_t count) + ShardingID Runtime::generate_library_sharding_ids(const char *name, + size_t count) //-------------------------------------------------------------------------- { - // Not implemented until control replication - return 0; + return runtime->generate_library_sharding_ids(name, count); } //-------------------------------------------------------------------------- - ShardingID Runtime::generate_static_sharding_id(void) + /*static*/ ShardingID Runtime::generate_static_sharding_id(void) //-------------------------------------------------------------------------- { - // Not implemented until control replication - return 0; + return Internal::Runtime::generate_static_sharding_id(); } //-------------------------------------------------------------------------- @@ -6097,15 +6146,23 @@ namespace Legion { const char *warning_string) //-------------------------------------------------------------------------- { - // Not implemented until control replication + runtime->register_sharding_functor(sid, functor, true/*need zero check*/, + silence_warnings, warning_string); } //-------------------------------------------------------------------------- /*static*/ void Runtime::preregister_sharding_functor(ShardingID sid, - ShardingFunctor *functor) + ShardingFunctor *func) //-------------------------------------------------------------------------- { - // Not implemented until control replication + Internal::Runtime::preregister_sharding_functor(sid, func); + } + + //-------------------------------------------------------------------------- + /*static*/ ShardingFunctor* Runtime::get_sharding_functor(ShardingID sid) + //-------------------------------------------------------------------------- + { + return Internal::Runtime::get_sharding_functor(sid); } //-------------------------------------------------------------------------- @@ -6406,14 +6463,14 @@ namespace Legion { void Runtime::print_once(Context ctx, FILE *f, const char *message) //-------------------------------------------------------------------------- { - fprintf(f, "%s", message); + runtime->print_once(ctx, f, message); } //-------------------------------------------------------------------------- void Runtime::log_once(Context ctx, Realm::LoggerMessage &message) //-------------------------------------------------------------------------- { - // Do nothing, just don't deactivate it + runtime->log_once(ctx, message); } //-------------------------------------------------------------------------- @@ -6808,7 +6865,7 @@ namespace Legion { REPORT_LEGION_ERROR(ERROR_CONFUSED_USER, "User does not know what " "they are doing asking for the shard ID in task %s (UID %lld)", ctx->get_task_name(), ctx->get_unique_id()) - return 0; + return ctx->get_shard_id(); } //-------------------------------------------------------------------------- @@ -6819,7 +6876,15 @@ namespace Legion { REPORT_LEGION_ERROR(ERROR_CONFUSED_USER, "User does not know what they" " are doing asking for the number of shards in task %s (UID %lld)", ctx->get_task_name(), ctx->get_unique_id()) - return 1; + return ctx->get_num_shards(); + } + + //-------------------------------------------------------------------------- + Future Runtime::consensus_match(Context ctx, const void *input,void *output, + size_t num_elements, size_t element_size) + //-------------------------------------------------------------------------- + { + return ctx->consensus_match(input, output, num_elements, element_size); } //-------------------------------------------------------------------------- diff --git a/runtime/legion/legion.inl b/runtime/legion/legion.inl index 8afcf45dbd..559970a6bf 100644 --- a/runtime/legion/legion.inl +++ b/runtime/legion/legion.inl @@ -18338,7 +18338,7 @@ namespace Legion { //-------------------------------------------------------------------------- inline void AttachLauncher::attach_file(const char *name, const std::vector &fields, - LegionFileMode m) + LegionFileMode m, bool local_file) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -18347,12 +18347,13 @@ namespace Legion { file_name = name; mode = m; file_fields = fields; + local_files = local_file; } //-------------------------------------------------------------------------- inline void AttachLauncher::attach_hdf5(const char *name, const std::map &field_map, - LegionFileMode m) + LegionFileMode m, bool local) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -18361,6 +18362,7 @@ namespace Legion { file_name = name; mode = m; field_files = field_map; + local_files = local; } //-------------------------------------------------------------------------- @@ -19795,7 +19797,7 @@ namespace Legion { for (unsigned idx = 0; idx < handles.size(); idx++) untyped_handles[idx] = handles[idx]; return IndexSpaceT(create_index_space_union_internal(ctx, - IndexPartition(parent), &color, + IndexPartition(parent), &color, sizeof(color), Internal::NT_TemplateHelper::encode_tag(), untyped_handles)); } @@ -19809,7 +19811,7 @@ namespace Legion { //-------------------------------------------------------------------------- { return IndexSpaceT(create_index_space_union_internal(ctx, - IndexPartition(parent), &color, + IndexPartition(parent), &color, sizeof(color), Internal::NT_TemplateHelper::encode_tag(), IndexPartition(handle))); } @@ -19827,7 +19829,7 @@ namespace Legion { for (unsigned idx = 0; idx < handles.size(); idx++) untyped_handles[idx] = handles[idx]; return IndexSpaceT(create_index_space_intersection_internal(ctx, - IndexPartition(parent), &color, + IndexPartition(parent), &color, sizeof(color), Internal::NT_TemplateHelper::encode_tag(), untyped_handles)); } @@ -19841,7 +19843,7 @@ namespace Legion { //-------------------------------------------------------------------------- { return IndexSpaceT(create_index_space_intersection_internal(ctx, - IndexPartition(parent), &color, + IndexPartition(parent), &color, sizeof(color), Internal::NT_TemplateHelper::encode_tag(), IndexPartition(handle))); } @@ -19860,7 +19862,7 @@ namespace Legion { for (unsigned idx = 0; idx < handles.size(); idx++) untyped_handles[idx] = handles[idx]; return IndexSpaceT(create_index_space_difference_internal(ctx, - IndexPartition(parent), &color, + IndexPartition(parent), &color, sizeof(color), Internal::NT_TemplateHelper::encode_tag(), IndexSpace(initial), untyped_handles)); } diff --git a/runtime/legion/legion_analysis.cc b/runtime/legion/legion_analysis.cc index f587207b43..aa08c4647a 100644 --- a/runtime/legion/legion_analysis.cc +++ b/runtime/legion/legion_analysis.cc @@ -25,6 +25,7 @@ #include "legion/legion_views.h" #include "legion/legion_analysis.h" #include "legion/legion_context.h" +#include "legion/legion_replication.h" namespace Legion { namespace Internal { @@ -381,6 +382,32 @@ namespace Legion { remote_tpl->record_get_term_event(memo); } + //-------------------------------------------------------------------------- + void RemoteTraceRecorder::request_term_event(ApUserEvent &term_event) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!term_event.exists() || term_event.has_triggered()); +#endif + if (local_space != origin_space) + { + RtUserEvent ready = Runtime::create_rt_user_event(); + Serializer rez; + { + RezCheck z(rez); + rez.serialize(remote_tpl); + rez.serialize(REMOTE_TRACE_REQUEST_TERM_EVENT); + rez.serialize(&term_event); + rez.serialize(ready); + } + runtime->send_remote_trace_update(origin_space, rez); + // Wait for the result to be set + ready.wait(); + } + else + remote_tpl->request_term_event(term_event); + } + //-------------------------------------------------------------------------- void RemoteTraceRecorder::record_create_ap_user_event( ApUserEvent lhs, Memoizable *memo) @@ -799,7 +826,8 @@ namespace Legion { InstanceView *view, const RegionUsage &usage, const FieldMask &user_mask, - bool update_validity) + bool update_validity, + std::set &effects) //-------------------------------------------------------------------------- { if (local_space != origin_space) @@ -824,7 +852,7 @@ namespace Legion { } else remote_tpl->record_op_view(memo, idx, view, usage, - user_mask, update_validity); + user_mask, update_validity, effects); } //-------------------------------------------------------------------------- @@ -1008,6 +1036,28 @@ namespace Legion { tpl->record_remote_memoizable(memo); break; } + case REMOTE_TRACE_REQUEST_TERM_EVENT: + { + ApUserEvent *target; + derez.deserialize(target); + RtUserEvent ready; + derez.deserialize(ready); + ApUserEvent result; + tpl->request_term_event(result); +#ifdef DEBUG_LEGION + assert(result.exists()); +#endif + Serializer rez; + { + RezCheck z2(rez); + rez.serialize(REMOTE_TRACE_REQUEST_TERM_EVENT); + rez.serialize(target); + rez.serialize(result); + rez.serialize(ready); + } + runtime->send_remote_trace_response(source, rez); + break; + } case REMOTE_TRACE_CREATE_USER_EVENT: { RtUserEvent applied; @@ -1436,9 +1486,13 @@ namespace Legion { derez.deserialize(update_validity); if (ready.exists() && !ready.has_triggered()) ready.wait(); + std::set effects; tpl->record_op_view(memo, index, view, usage, - user_mask, update_validity); - Runtime::trigger_event(applied); + user_mask, update_validity, effects); + if (!effects.empty()) + Runtime::trigger_event(applied, Runtime::merge_events(effects)); + else + Runtime::trigger_event(applied); if (memo->get_origin_space() != runtime->address_space) delete memo; break; @@ -1588,6 +1642,7 @@ namespace Legion { derez.deserialize(kind); switch (kind) { + case REMOTE_TRACE_REQUEST_TERM_EVENT: case REMOTE_TRACE_MERGE_EVENTS: case REMOTE_TRACE_ISSUE_COPY: case REMOTE_TRACE_ISSUE_FILL: @@ -1919,15 +1974,22 @@ namespace Legion { } ///////////////////////////////////////////////////////////// - // ProjectionInfo + // ProjectionInfo ///////////////////////////////////////////////////////////// //-------------------------------------------------------------------------- ProjectionInfo::ProjectionInfo(Runtime *runtime, - const RegionRequirement &req, IndexSpaceNode *launch_space) + const RegionRequirement &req, + IndexSpaceNode *launch_space, + ShardingFunction *f/*=NULL*/, + IndexSpace shard_space/*=NO_SPACE*/) : projection((req.handle_type != LEGION_SINGULAR_PROJECTION) ? runtime->find_projection_function(req.projection) : NULL), - projection_type(req.handle_type), projection_space(launch_space) + projection_type(req.handle_type), projection_space( + (req.handle_type != LEGION_SINGULAR_PROJECTION) ? launch_space : NULL), + sharding_function(f), sharding_space(shard_space.exists() ? + runtime->forest->get_node(shard_space) : + (f == NULL) ? NULL : projection_space) //-------------------------------------------------------------------------- { } @@ -2295,22 +2357,19 @@ namespace Legion { } ///////////////////////////////////////////////////////////// - // Projection Epoch + // ProjectionTree ///////////////////////////////////////////////////////////// - // C++ is really dumb - const ProjectionEpochID ProjectionEpoch::first_epoch; - //-------------------------------------------------------------------------- - ProjectionEpoch::ProjectionEpoch(ProjectionEpochID id, const FieldMask &m) - : epoch_id(id), valid_fields(m) + ProjectionTree::ProjectionTree(IndexTreeNode *n, ShardID owner) + : node(n), owner_shard(owner) //-------------------------------------------------------------------------- { } //-------------------------------------------------------------------------- - ProjectionEpoch::ProjectionEpoch(const ProjectionEpoch &rhs) - : epoch_id(rhs.epoch_id), valid_fields(rhs.valid_fields) + ProjectionTree::ProjectionTree(const ProjectionTree &rhs) + : node(rhs.node), owner_shard(rhs.owner_shard) //-------------------------------------------------------------------------- { // should never be called @@ -2318,13 +2377,16 @@ namespace Legion { } //-------------------------------------------------------------------------- - ProjectionEpoch::~ProjectionEpoch(void) + ProjectionTree::~ProjectionTree(void) //-------------------------------------------------------------------------- { + for (std::map::const_iterator it = + children.begin(); it != children.end(); it++) + delete it->second; } //-------------------------------------------------------------------------- - ProjectionEpoch& ProjectionEpoch::operator=(const ProjectionEpoch &rhs) + ProjectionTree& ProjectionTree::operator=(const ProjectionTree &rhs) //-------------------------------------------------------------------------- { // should never be called @@ -2333,14 +2395,133 @@ namespace Legion { } //-------------------------------------------------------------------------- - void ProjectionEpoch::insert(ProjectionFunction *function, - IndexSpaceNode* node) + void ProjectionTree::add_child(ProjectionTree *child) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert(!!valid_fields); + assert(child != this); #endif - write_projections[function].insert(node); + children[child->node] = child; + } + + //-------------------------------------------------------------------------- + bool ProjectionTree::dominates(const ProjectionTree *other) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(node == other->node); +#endif + // If we have no children we definitely dominate + // Assuming of course that we are on the same shard + if (children.empty()) + { + if (other->children.empty()) + return (owner_shard == other->owner_shard); + return other->all_same_shard(owner_shard); + } + // If we have children and the other one doesn't then we don't + if (other->children.empty()) + return false; + // Check to see if we have a child that dominates each of the + // other trees children + for (std::map::const_iterator it = + other->children.begin(); it != other->children.end(); it++) + { + std::map::const_iterator finder = + children.find(it->first); + if (finder == children.end()) + return false; + if (!finder->second->dominates(it->second)) + return false; + } + return true; + } + + //-------------------------------------------------------------------------- + bool ProjectionTree::disjoint(const ProjectionTree *other) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(node == other->node); +#endif + if (children.empty() || other->children.empty()) + return false; + if (!node->is_index_space_node() && + node->as_index_part_node()->is_disjoint()) + { + // All children are disjoint, if there are any common ones + // see if they are disjoint with respect to each other + for (std::map::const_iterator it = + children.begin(); it != children.end(); it++) + { + std::map::const_iterator finder = + other->children.find(it->first); + if ((finder != other->children.end()) && + !it->second->disjoint(finder->second)) + return false; + } + } + else + { + // Children are not disjoint, so any that don't match + // cause the test to fail, otherwise we can still recurse + if (node->is_index_space_node()) + { + IndexSpaceNode *space = node->as_index_space_node(); + for (std::map::const_iterator it1 = + children.begin(); it1 != children.end(); it1++) + { + for (std::map::const_iterator it2 = + other->children.begin(); it2 != other->children.end(); it2++) + { + if (it1->first == it2->first) + { + if (!it1->second->disjoint(it2->second)) + return false; + } + else if (!space->are_disjoint(it1->first->color, + it2->first->color)) + return false; + } + } + + } + else + { + IndexPartNode *part = node->as_index_part_node(); + for (std::map::const_iterator it1 = + children.begin(); it1 != children.end(); it1++) + { + for (std::map::const_iterator it2 = + other->children.begin(); it2 != other->children.end(); it2++) + { + if (it1->first == it2->first) + { + if (!it1->second->disjoint(it2->second)) + return false; + } + else if (!part->are_disjoint(it1->first->color,it2->first->color)) + return false; + } + } + } + } + return true; + } + + //-------------------------------------------------------------------------- + bool ProjectionTree::all_same_shard(ShardID other_shard) const + //-------------------------------------------------------------------------- + { + if (children.empty()) + return (owner_shard == other_shard); + for (std::map::const_iterator it = + children.begin(); it != children.end(); it++) + { + if (!it->second->all_same_shard(other_shard)) + return false; + } + return true; } ///////////////////////////////////////////////////////////// @@ -2386,7 +2567,6 @@ namespace Legion { assert(field_states.empty()); assert(curr_epoch_users.empty()); assert(prev_epoch_users.empty()); - assert(projection_epochs.empty()); assert(!reduction_fields); #endif } @@ -2425,10 +2605,6 @@ namespace Legion { clear_logical_users(); reduction_fields.clear(); outstanding_reductions.clear(); - for (std::list::const_iterator it = - projection_epochs.begin(); it != projection_epochs.end(); it++) - delete *it; - projection_epochs.clear(); } //-------------------------------------------------------------------------- @@ -2463,73 +2639,111 @@ namespace Legion { } } + ///////////////////////////////////////////////////////////// + // Projection Summary + ///////////////////////////////////////////////////////////// + //-------------------------------------------------------------------------- - void LogicalState::advance_projection_epochs(const FieldMask &advance_mask) + ProjectionSummary::ProjectionSummary(void) + : domain(NULL), projection(NULL), sharding(NULL), sharding_domain(NULL) //-------------------------------------------------------------------------- { - // See if we can get some coalescing going on here - std::map to_add; - for (std::list::iterator it = - projection_epochs.begin(); it != - projection_epochs.end(); /*nothing*/) - { - FieldMask overlap = (*it)->valid_fields & advance_mask; - if (!overlap) - { - it++; - continue; - } - const ProjectionEpochID next_epoch_id = (*it)->epoch_id + 1; - std::map::iterator finder = - to_add.find(next_epoch_id); - if (finder == to_add.end()) - { - ProjectionEpoch *next_epoch = - new ProjectionEpoch((*it)->epoch_id+1, overlap); - to_add[next_epoch_id] = next_epoch; - } - else - finder->second->valid_fields |= overlap; - // Filter the fields from our old one - (*it)->valid_fields -= overlap; - if (!((*it)->valid_fields)) - { - delete (*it); - it = projection_epochs.erase(it); - } - else - it++; - } - if (!to_add.empty()) - { - for (std::map::const_iterator it = - to_add.begin(); it != to_add.end(); it++) - projection_epochs.push_back(it->second); - } - } + } //-------------------------------------------------------------------------- - void LogicalState::update_projection_epochs(FieldMask capture_mask, - const ProjectionInfo &info) + ProjectionSummary::ProjectionSummary(IndexSpaceNode *is, + ProjectionFunction *p, + ShardingFunction *s, + IndexSpaceNode *sd) + : domain(is), projection(p), sharding(s), sharding_domain(sd) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(!!capture_mask); -#endif - for (std::list::const_iterator it = - projection_epochs.begin(); it != projection_epochs.end(); it++) - { - FieldMask overlap = (*it)->valid_fields & capture_mask; - if (!overlap) - continue; - capture_mask -= overlap; - if (!capture_mask) - return; - } - // If it didn't already exist, start a new projection epoch - ProjectionEpoch *new_epoch = - new ProjectionEpoch(ProjectionEpoch::first_epoch, capture_mask); - projection_epochs.push_back(new_epoch); + } + + //-------------------------------------------------------------------------- + ProjectionSummary::ProjectionSummary(const ProjectionInfo &info) + : domain(info.projection_space), projection(info.projection), + sharding(info.sharding_function), sharding_domain(info.sharding_space) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + bool ProjectionSummary::operator<(const ProjectionSummary &rhs) const + //-------------------------------------------------------------------------- + { + if (domain->handle < rhs.domain->handle) + return true; + else if (domain->handle > rhs.domain->handle) + return false; + else if (projection->projection_id < rhs.projection->projection_id) + return true; + else if (projection->projection_id > rhs.projection->projection_id) + return false; + else if ((sharding == NULL) && (rhs.sharding == NULL)) + return false; + else if ((sharding == NULL) && (rhs.sharding != NULL)) + return true; + else if (rhs.sharding == NULL) + return false; + else if (sharding->sharding_id < rhs.sharding->sharding_id) + return true; + else if (sharding->sharding_id > rhs.sharding->sharding_id) + return false; + else + return sharding_domain->handle < rhs.sharding_domain->handle; + } + + //-------------------------------------------------------------------------- + bool ProjectionSummary::operator==(const ProjectionSummary &rhs) const + //-------------------------------------------------------------------------- + { + if (domain->handle != rhs.domain->handle) + return false; + else if (projection->projection_id != rhs.projection->projection_id) + return false; + else if ((sharding == NULL) && (rhs.sharding == NULL)) + return true; + else if ((sharding == NULL) && (rhs.sharding != NULL)) + return false; + else if (rhs.sharding == NULL) + return false; + if (sharding->sharding_id != rhs.sharding->sharding_id) + return false; + if (sharding_domain->handle != rhs.sharding_domain->handle) + return false; + return true; + } + + //-------------------------------------------------------------------------- + bool ProjectionSummary::operator!=(const ProjectionSummary &rhs) const + //-------------------------------------------------------------------------- + { + return !(*this == rhs); + } + + //-------------------------------------------------------------------------- + void ProjectionSummary::pack_summary(Serializer &rez) const + //-------------------------------------------------------------------------- + { + rez.serialize(domain->handle); + rez.serialize(projection->projection_id); + // We don't handle packing the sharding information + } + + //-------------------------------------------------------------------------- + /*static*/ ProjectionSummary ProjectionSummary::unpack_summary( + Deserializer &derez, RegionTreeForest *context) + //-------------------------------------------------------------------------- + { + ProjectionSummary result; + IndexSpace handle; + derez.deserialize(handle); + result.domain = context->get_node(handle); + ProjectionID pid; + derez.deserialize(pid); + result.projection = context->runtime->find_projection_function(pid); + return result; } ///////////////////////////////////////////////////////////// @@ -2538,8 +2752,8 @@ namespace Legion { //-------------------------------------------------------------------------- FieldState::FieldState(void) - : open_state(NOT_OPEN), redop(0), projection(NULL), - projection_space(NULL), rebuild_timeout(1) + : open_state(NOT_OPEN), redop(0), rebuild_timeout(1), + disjoint_shallow(false) //-------------------------------------------------------------------------- { } @@ -2547,7 +2761,7 @@ namespace Legion { //-------------------------------------------------------------------------- FieldState::FieldState(const GenericUser &user, const FieldMask &m, RegionTreeNode *child, std::set &applied) - : redop(0), projection(NULL), projection_space(NULL), rebuild_timeout(1) + : redop(0), rebuild_timeout(1), disjoint_shallow(false) //-------------------------------------------------------------------------- { if (IS_READ_ONLY(user.usage)) @@ -2569,12 +2783,13 @@ namespace Legion { //-------------------------------------------------------------------------- FieldState::FieldState(const RegionUsage &usage, const FieldMask &m, ProjectionFunction *proj, IndexSpaceNode *proj_space, - bool disjoint, bool dirty_reduction) - : redop(0),projection(proj),projection_space(proj_space),rebuild_timeout(1) + ShardingFunction *fn, IndexSpaceNode *shard_space, + RegionTreeNode *node, bool dirty_reduction) + : redop(0), rebuild_timeout(1), disjoint_shallow(false) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert(projection != NULL); + assert(proj != NULL); #endif open_children.relax_valid_mask(m); if (IS_READ_ONLY(usage)) @@ -2587,21 +2802,27 @@ namespace Legion { open_state = OPEN_REDUCE_PROJ; redop = usage.redop; } - else if (disjoint && (projection->depth == 0)) - open_state = OPEN_READ_WRITE_PROJ_DISJOINT_SHALLOW; else + { open_state = OPEN_READ_WRITE_PROJ; + projections.insert(ProjectionSummary(proj_space, proj, fn,shard_space)); + // Check for disjoint shallow completeness + if ((fn == NULL) && (proj->depth == 0) && !node->is_region() && + node->are_all_children_disjoint()) + disjoint_shallow = true; + } } //-------------------------------------------------------------------------- FieldState::FieldState(const FieldState &rhs) : open_state(rhs.open_state), redop(rhs.redop), - projection(rhs.projection), projection_space(rhs.projection_space), - rebuild_timeout(rhs.rebuild_timeout) + rebuild_timeout(rhs.rebuild_timeout), + disjoint_shallow(rhs.disjoint_shallow) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION assert(rhs.open_children.empty()); + assert(rhs.projections.empty()); #endif } @@ -2622,12 +2843,13 @@ namespace Legion { #ifdef DEBUG_LEGION assert(open_children.empty()); assert(rhs.open_children.empty()); + assert(projections.empty()); + assert(rhs.projections.empty()); #endif open_state = rhs.open_state; redop = rhs.redop; - projection = rhs.projection; - projection_space = rhs.projection_space; rebuild_timeout = rhs.rebuild_timeout; + disjoint_shallow = rhs.disjoint_shallow; return *this; } @@ -2637,11 +2859,19 @@ namespace Legion { { if (redop != rhs.redop) return false; - if (projection != rhs.projection) - return false; - // Only do this test if they are both projections - if ((projection != NULL) && (projection_space != rhs.projection_space)) + if (is_projection_state()) + { + if (!rhs.is_projection_state()) + return false; + // Both projection spaces, check to see if they have the same + // set of projections + if (!projections_match(rhs)) + return false; + // if we make it past here they are all the same + } + else if (rhs.is_projection_state()) return false; + // Now check the privilege states if (redop == 0) return (open_state == rhs.open_state); else @@ -2662,6 +2892,24 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + bool FieldState::projections_match(const FieldState &rhs) const + //-------------------------------------------------------------------------- + { + if (projections.size() != rhs.projections.size()) + return false; + std::set::const_iterator it1 = projections.begin(); + std::set::const_iterator it2 = rhs.projections.begin(); + // zip the projections so we can compare them + while (it1 != projections.end()) + { + if ((*it1) != (*it2)) + return false; + it1++; it2++; + } + return true; + } + //-------------------------------------------------------------------------- void FieldState::merge(FieldState &rhs, RegionTreeNode *node) //-------------------------------------------------------------------------- @@ -2679,7 +2927,7 @@ namespace Legion { open_children.relax_valid_mask(rhs.open_children.get_valid_mask()); #ifdef DEBUG_LEGION assert(redop == rhs.redop); - assert(projection == rhs.projection); + assert(projections_match(rhs)); #endif if (redop > 0) { @@ -2702,6 +2950,7 @@ namespace Legion { open_state = OPEN_MULTI_REDUCE; } } + // no need to merge projections, we know they are the same } //-------------------------------------------------------------------------- @@ -2711,7 +2960,6 @@ namespace Legion { if (is_projection_state()) { #ifdef DEBUG_LEGION - assert(projection != NULL); assert(open_children.empty()); #endif open_children.filter_valid_mask(mask); @@ -2778,20 +3026,188 @@ namespace Legion { } //-------------------------------------------------------------------------- - bool FieldState::projection_domain_dominates( - IndexSpaceNode *next_space) const + bool FieldState::can_elide_close_operation(Operation *op, unsigned index, + const ProjectionInfo &info, RegionTreeNode *node, bool reduction) const //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert(projection_space != NULL); + assert(!projections.empty()); // should be in a projection mode #endif - if (projection_space == next_space) - return true; - // If the domains do not have the same type, the answer must be no - if (projection_space->handle.get_type_tag() != - next_space->handle.get_type_tag()) + // This function is super important! It decides whether this new + // projection info can be added to the current projection epoch, if + // it can't then we will need a close operation to be inserted + bool elide = true; + // We have two different paths corresponding to whether we are in + // a control replication context or not + if (info.sharding_function == NULL) + { + // If we don't have a sharding function, then we aren't in a + // control replication context + // See if we are in a disjoint shallow complete mode + // If we're disjoint shallow then we can definitely always + // elide the close operation as we can do more writes or + // reads on this epoch without any issues, for reductions + // though we can only go in the same epoch if we're reducing + // to a subset of the writes that have already been done + if (!disjoint_shallow || reduction) + { + // If we're not disjoint shallow complete we have more work to do + // Run through the list and see if all the index spaces + // are the same or dominate our index space and all the + // projection functions are the same as ours, if this is + // true then we know this is a totally data parallel + // computation and there is no need for a close + for (std::set::const_iterator it = + projections.begin(); it != projections.end(); it++) + { + if (it->projection != info.projection) + { + elide = false; + break; + } + if ((it->domain != info.projection_space) && + !it->domain->dominates(info.projection_space)) + { + elide = false; + break; + } + } + if (!elide) + { + // Next we're going to need to compute the actual interference + // sets so check to see if we've memoized the result or not + if (!info.projection->find_elide_close_result(info, projections, + node, elide)) + { + elide = expensive_elide_test(op, index, info, node, reduction); + // Now memoize the results for later + info.projection->record_elide_close_result(info, projections, + node, elide); + } + } + } + } + else + { + // We have a sharding function so we are in a + // control replication context + + // See if all the index spaces dominate, and the projection + // functions are all the same, and the sharding functions are + // all the same, in which case we know this is totally data + // parallel and each shard has the same subregion set + bool check_expensive = true; + for (std::set::const_iterator it = + projections.begin(); it != projections.end(); it++) + { + if (it->projection != info.projection) + { + elide = false; + break; + } + if (it->sharding != info.sharding_function) + { + elide = false; + // No need to check expensive here since we know + // that we need the close operation no matter what + check_expensive = false; + break; + } + if ((it->domain != info.projection_space) && + !it->domain->dominates(info.projection_space)) + { + elide = false; + break; + } + } + + if (!elide && check_expensive) + { + // Next we're going to need to compute the actual interference + // set so check to see if we've memoized the result or not + if (!info.projection->find_elide_close_result(info, projections, + node, elide)) + { + elide = expensive_elide_test(op, index, info, node, reduction); + // Now memoize the results for later + info.projection->record_elide_close_result(info, projections, + node, elide); + } + } + } + return elide; + } + + //-------------------------------------------------------------------------- + void FieldState::record_projection_summary(const ProjectionInfo &info, + RegionTreeNode *node) + //-------------------------------------------------------------------------- + { + if (disjoint_shallow || projections.empty()) + { + if ((info.sharding_function == NULL) && + (info.projection->depth == 0) && !node->is_region() && + node->are_all_children_disjoint()) + disjoint_shallow = true; + else + disjoint_shallow = false; + } + projections.insert(ProjectionSummary(info)); + } + + //-------------------------------------------------------------------------- + bool FieldState::expensive_elide_test(Operation *op, unsigned index, + const ProjectionInfo &info, RegionTreeNode *node, bool reduction) const + //-------------------------------------------------------------------------- + { + // We can't do this test if the projection function is not functional + if (!info.projection->is_functional) + { + REPORT_LEGION_WARNING(LEGION_WARNING_SLOW_NON_FUNCTIONAL_PROJECTION, + "We strongly encourage all projection functors to be functional, " + "however, projection function %d is not and therefore an " + "expensive analysis cannot be memoized. Please consider making " + "it functional to avoid performance degredation.", + info.projection->projection_id) return false; - return projection_space->dominates(next_space); + } + // Then check whether one of two conditions are true: + // 1. We can build an injective mapping for each region in + // the next projection where it is the same as or a subregion + // of every region it interferes with + // 2. If the new projection is not a reduction then if + // every access is independent of the prior writes then + // we know that we can safely add this to the epoch without + // needing to do a close operation + IndexTreeNode *root_source = node->get_row_source(); + ProjectionTree *prev = new ProjectionTree(root_source); + // Construct the previous projection tree from all prior projections + { + std::map node_map; + node_map[root_source] = prev; + for (std::set::const_iterator it = + projections.begin(); it != projections.end(); it++) + it->projection->construct_projection_tree(op, index, node, it->domain, + it->sharding, it->sharding_domain, node_map); + } + // Then construct the new projection tree + ProjectionTree *next = + info.projection->construct_projection_tree(op, index, node, + info.projection_space, info.sharding_function, info.sharding_space); + // First check to see if the previous dominates + bool has_mapping = false; + if (prev->dominates(next)) + has_mapping = true; + bool all_disjoint = false; + if (!has_mapping && !reduction) // no disjoint reductions + all_disjoint = prev->disjoint(next); + // Clean up our data structures + delete prev; + delete next; +#ifdef DEBUG_LEGION + assert(!has_mapping || !all_disjoint); // can't both be true +#endif + return (has_mapping || all_disjoint); } //-------------------------------------------------------------------------- @@ -2834,32 +3250,20 @@ namespace Legion { } case OPEN_READ_ONLY_PROJ: { - logger->log("Field State: OPEN READ-ONLY PROJECTION %d", - projection->projection_id); + logger->log("Field State: OPEN READ-ONLY PROJECTION %zd", + projections.size()); break; } case OPEN_READ_WRITE_PROJ: { - logger->log("Field State: OPEN READ WRITE PROJECTION %d", - projection->projection_id); - break; - } - case OPEN_READ_WRITE_PROJ_DISJOINT_SHALLOW: - { - logger->log("Field State: OPEN READ WRITE PROJECTION (Disjoint Shallow) %d", - projection->projection_id); + logger->log("Field State: OPEN READ WRITE PROJECTION %zd", + projections.size()); break; } case OPEN_REDUCE_PROJ: { - logger->log("Field State: OPEN REDUCE PROJECTION %d Mode %d", - projection->projection_id, redop); - break; - } - case OPEN_REDUCE_PROJ_DIRTY: - { - logger->log("Field State: OPEN REDUCE PROJECTION (Dirty) %d Mode %d", - projection->projection_id, redop); + logger->log("Field State: OPEN REDUCE PROJECTION %zd Mode %d", + projections.size(), redop); break; } default: @@ -2919,32 +3323,26 @@ namespace Legion { } case OPEN_READ_ONLY_PROJ: { - logger->log("Field State: OPEN READ-ONLY PROJECTION %d", - projection->projection_id); + logger->log("Field State: OPEN READ-ONLY PROJECTION %zd", + projections.size()); break; } case OPEN_READ_WRITE_PROJ: { - logger->log("Field State: OPEN READ WRITE PROJECTION %d", - projection->projection_id); - break; - } - case OPEN_READ_WRITE_PROJ_DISJOINT_SHALLOW: - { - logger->log("Field State: OPEN READ WRITE PROJECTION (Disjoint Shallow) %d", - projection->projection_id); + logger->log("Field State: OPEN READ WRITE PROJECTION %zd", + projections.size()); break; } case OPEN_REDUCE_PROJ: { - logger->log("Field State: OPEN REDUCE PROJECTION %d Mode %d", - projection->projection_id, redop); + logger->log("Field State: OPEN REDUCE PROJECTION %zd Mode %d", + projections.size()); break; } case OPEN_REDUCE_PROJ_DIRTY: { - logger->log("Field State: OPEN REDUCE PROJECTION (Dirty) %d Mode %d", - projection->projection_id, redop); + logger->log("Field State: OPEN REDUCE PROJECTION (Dirty) %zd " + "Mode %d", projections.size(), redop); break; } default: @@ -3135,7 +3533,12 @@ namespace Legion { else req = RegionRequirement(root_node->as_partition_node()->handle, 0, LEGION_READ_WRITE, LEGION_EXCLUSIVE, trace_info.req.parent); - close_op = creator->runtime->get_available_merge_close_op(); + TaskContext *ctx = creator->get_context(); +#ifdef DEBUG_LEGION_COLLECTIVES + close_op = ctx->get_merge_close_op(user, root_node); +#else + close_op = ctx->get_merge_close_op(); +#endif merge_close_gen = close_op->get_generation(); req.privilege_fields.clear(); root_node->column_source->get_field_set(close_mask, @@ -6306,13 +6709,14 @@ namespace Legion { std::vector &target_vws, const PhysicalTraceInfo &t_info, const ApEvent pre, const ApEvent term, - const bool track, const bool check, const bool record) + const bool track, const bool check, + const bool record, const bool skip) : PhysicalAnalysis(rt, o, idx, info, true/*on heap*/), usage(req), node(rn), target_instances(target_insts), target_views(target_vws), trace_info(t_info), precondition(pre), term_event(term), - track_effects(track), check_initialized(check && - !IS_DISCARD(usage) && !IS_SIMULT(usage)), - record_valid(record), output_aggregator(NULL) + track_effects(track), check_initialized(check && !IS_DISCARD(usage) && + !IS_SIMULT(usage)), record_valid(record), skip_output(skip), + output_aggregator(NULL) //-------------------------------------------------------------------------- { } @@ -6325,13 +6729,13 @@ namespace Legion { std::vector &target_vws, const PhysicalTraceInfo &info, const RtEvent user_reg, const ApEvent pre, - const ApEvent term, const bool track, const bool check, - const bool record) + const ApEvent term, const bool track, + const bool check, const bool record, const bool skip) : PhysicalAnalysis(rt, src, prev, o, idx, man, true/*on heap*/), usage(use), node(rn), target_instances(target_insts), target_views(target_vws), trace_info(info), precondition(pre), term_event(term), track_effects(track), check_initialized(check), - record_valid(record), output_aggregator(NULL), + record_valid(record), skip_output(skip), output_aggregator(NULL), remote_user_registered(user_reg) //-------------------------------------------------------------------------- { @@ -6343,7 +6747,8 @@ namespace Legion { target_instances(rhs.target_instances), target_views(rhs.target_views), trace_info(rhs.trace_info), precondition(rhs.precondition), term_event(rhs.term_event), track_effects(rhs.track_effects), - check_initialized(rhs.check_initialized), record_valid(rhs.record_valid) + check_initialized(rhs.check_initialized),record_valid(rhs.record_valid), + skip_output(rhs.skip_output) //-------------------------------------------------------------------------- { // should never be called @@ -6472,6 +6877,7 @@ namespace Legion { rez.serialize(version_manager); rez.serialize(check_initialized); rez.serialize(record_valid); + rez.serialize(skip_output); rez.serialize(rit->first.second); } runtime->send_equivalence_set_remote_updates(target, rez); @@ -6550,12 +6956,23 @@ namespace Legion { DeferPerformOutputArgs args(this, trace_info); runtime->issue_runtime_meta_task(args, LG_THROUGHPUT_DEFERRED_PRIORITY, perform_precondition); - applied_events.insert(args.applied_event); - return args.effects_event; + // If we're skipping the output we still need to launch this + // meta-task to prevent the analysis from being deleted until + // everything else is done, we just don't record any output + if (!skip_output) + { + applied_events.insert(args.applied_event); + return args.effects_event; + } + else + return ApEvent::NO_AP_EVENT; } ApEvent result; if (output_aggregator != NULL) { +#ifdef DEBUG_LEGION + assert(!skip_output); +#endif output_aggregator->issue_updates(trace_info, term_event); // We need to wait for the aggregator updates to be applied // here before we can summarize the output @@ -6649,6 +7066,8 @@ namespace Legion { derez.deserialize(check_initialized); bool record_valid; derez.deserialize(record_valid); + bool skip_output; + derez.deserialize(skip_output); bool cached_sets; derez.deserialize(cached_sets); @@ -6657,7 +7076,7 @@ namespace Legion { UpdateAnalysis *analysis = new UpdateAnalysis(runtime, original_source, previous, op, index, version_manager, usage, node, targets, target_views, trace_info, remote_user_registered, precondition, - term_event, track_effects, check_initialized, record_valid); + term_event, track_effects,check_initialized,record_valid,skip_output); analysis->add_reference(); std::set deferral_events, applied_events; // Make sure that all our pointers are ready @@ -12540,6 +12959,8 @@ namespace Legion { #ifdef DEBUG_LEGION assert(!(restricted_mask - first->second)); #endif + if (!first->first->is_instance_view()) + assert(false); // TODO: handle sharded view case InstanceView *dst_view = first->first->as_instance_view(); FieldMaskSet srcs; for (unsigned idx = 0; idx < src_views.size(); idx++) @@ -12576,6 +12997,8 @@ namespace Legion { const FieldMask overlap = it->second & restricted_mask; if (!overlap) continue; + if (!it->first->is_instance_view()) + assert(false); // TODO: handle sharded view case InstanceView *dst_view = it->first->as_instance_view(); if (aggregator == NULL) aggregator = new CopyFillAggregator(runtime->forest, op, index, @@ -12592,6 +13015,8 @@ namespace Legion { const FieldMask dst_overlap = it->second & restricted_mask; if (!dst_overlap) continue; + if (!it->first->is_instance_view()) + assert(false); // TODO: handle sharded view case InstanceView *dst_view = it->first->as_instance_view(); FieldMaskSet srcs; for (unsigned idx = 0; idx < src_views.size(); idx++) @@ -12634,6 +13059,8 @@ namespace Legion { const FieldMask overlap = it->second & restricted_mask; if (!overlap) continue; + if (!it->first->is_instance_view()) + assert(false); // TODO: handle sharded view case InstanceView *dst_view = it->first->as_instance_view(); if (aggregator == NULL) aggregator = new CopyFillAggregator(runtime->forest, op, index, diff --git a/runtime/legion/legion_analysis.h b/runtime/legion/legion_analysis.h index 7dba2b94e4..139d39712e 100644 --- a/runtime/legion/legion_analysis.h +++ b/runtime/legion/legion_analysis.h @@ -135,6 +135,7 @@ namespace Legion { virtual PhysicalTraceRecorder* clone(Memoizable *memo) { return this; } public: virtual void record_get_term_event(Memoizable *memo) = 0; + virtual void request_term_event(ApUserEvent &term_event) = 0; virtual void record_create_ap_user_event(ApUserEvent lhs, Memoizable *memo) = 0; virtual void record_trigger_event(ApUserEvent lhs, ApEvent rhs, @@ -202,7 +203,8 @@ namespace Legion { InstanceView *view, const RegionUsage &usage, const FieldMask &user_mask, - bool update_validity) = 0; + bool update_validity, + std::set &applied) = 0; virtual void record_set_op_sync_event(ApEvent &lhs, Memoizable *memo) = 0; virtual void record_mapper_output(Memoizable *memo, const Mapper::MapTaskOutput &output, @@ -224,6 +226,7 @@ namespace Legion { public: enum RemoteTraceKind { REMOTE_TRACE_RECORD_GET_TERM, + REMOTE_TRACE_REQUEST_TERM_EVENT, REMOTE_TRACE_CREATE_USER_EVENT, REMOTE_TRACE_TRIGGER_EVENT, REMOTE_TRACE_MERGE_EVENTS, @@ -259,6 +262,7 @@ namespace Legion { virtual PhysicalTraceRecorder* clone(Memoizable *memo); public: virtual void record_get_term_event(Memoizable *memo); + virtual void request_term_event(ApUserEvent &term_event); virtual void record_create_ap_user_event(ApUserEvent lhs, Memoizable *memo); virtual void record_trigger_event(ApUserEvent lhs, ApEvent rhs, @@ -325,7 +329,8 @@ namespace Legion { InstanceView *view, const RegionUsage &usage, const FieldMask &user_mask, - bool update_validity); + bool update_validity, + std::set &applied); virtual void record_set_op_sync_event(ApEvent &lhs, Memoizable *memo); virtual void record_mapper_output(Memoizable *memo, const Mapper::MapTaskOutput &output, @@ -380,6 +385,11 @@ namespace Legion { base_sanity_check(); rec->record_get_term_event(memo); } + inline void request_term_event(ApUserEvent &term_event) + { + base_sanity_check(); + rec->request_term_event(term_event); + } inline void record_create_ap_user_event(ApUserEvent result) const { base_sanity_check(); @@ -568,10 +578,12 @@ namespace Legion { } inline void record_op_view(const RegionUsage &usage, const FieldMask &user_mask, - InstanceView *view) const + InstanceView *view, + std::set &applied) const { sanity_check(); - rec->record_op_view(memo,index,view,usage,user_mask,update_validity); + rec->record_op_view(memo, index, view, usage, user_mask, + update_validity, applied); } public: template @@ -606,13 +618,16 @@ namespace Legion { : projection(NULL), projection_type(LEGION_SINGULAR_PROJECTION), projection_space(NULL) { } ProjectionInfo(Runtime *runtime, const RegionRequirement &req, - IndexSpaceNode *launch_space); + IndexSpaceNode *launch_space,ShardingFunction *func = NULL, + IndexSpace shard_space = IndexSpace::NO_SPACE); public: inline bool is_projecting(void) const { return (projection != NULL); } public: ProjectionFunction *projection; ProjectionType projection_type; IndexSpaceNode *projection_space; + ShardingFunction *sharding_function; + IndexSpaceNode *sharding_space; }; /** @@ -653,6 +668,35 @@ namespace Legion { const bool covers; // whether the expr covers the ExprView its in }; + /** + * \struct ProjectionSummary + * A small helper class that tracks the triple that + * uniquely defines a set of region requirements + * for a projection operation + */ + struct ProjectionSummary { + public: + ProjectionSummary(void); + ProjectionSummary(IndexSpaceNode *is, + ProjectionFunction *p, + ShardingFunction *s, + IndexSpaceNode *sd); + ProjectionSummary(const ProjectionInfo &info); + public: + bool operator<(const ProjectionSummary &rhs) const; + bool operator==(const ProjectionSummary &rhs) const; + bool operator!=(const ProjectionSummary &rhs) const; + public: + void pack_summary(Serializer &rez) const; + static ProjectionSummary unpack_summary(Deserializer &derez, + RegionTreeForest *context); + public: + IndexSpaceNode *domain; + ProjectionFunction *projection; + ShardingFunction *sharding; + IndexSpaceNode *sharding_domain; + }; + /** * \struct FieldState * Track the field state more accurately @@ -666,7 +710,9 @@ namespace Legion { RegionTreeNode *child, std::set &applied); FieldState(const RegionUsage &u, const FieldMask &m, ProjectionFunction *proj, IndexSpaceNode *proj_space, - bool dis, bool dirty_reduction = false); + ShardingFunction *sharding_function, + IndexSpaceNode *sharding_space, + RegionTreeNode *node, bool dirty_reduction = false); FieldState(const FieldState &rhs); FieldState& operator=(const FieldState &rhs); ~FieldState(void); @@ -680,19 +726,28 @@ namespace Legion { open_children.swap(lhs.open_children); lhs.open_state = open_state; lhs.redop = redop; - lhs.projection = projection; - lhs.projection_space = projection_space; + projections.swap(lhs.projections); lhs.rebuild_timeout = rebuild_timeout; + lhs.disjoint_shallow = disjoint_shallow; } public: bool overlaps(const FieldState &rhs) const; + bool projections_match(const FieldState &rhs) const; void merge(FieldState &rhs, RegionTreeNode *node); bool filter(const FieldMask &mask); void add_child(RegionTreeNode *child, const FieldMask &mask, std::set &applied); void remove_child(RegionTreeNode *child); public: - bool projection_domain_dominates(IndexSpaceNode *next_space) const; + bool can_elide_close_operation(Operation *op, unsigned index, + const ProjectionInfo &info, + RegionTreeNode *node,bool reduction) const; + void record_projection_summary(const ProjectionInfo &info, + RegionTreeNode *node); + protected: + bool expensive_elide_test(Operation *op, unsigned index, + const ProjectionInfo &info, + RegionTreeNode *node, bool reduction) const; public: void print_state(TreeStateLogger *logger, const FieldMask &capture_mask, @@ -704,10 +759,10 @@ namespace Legion { FieldMaskSet open_children; OpenState open_state; ReductionOpID redop; - ProjectionFunction *projection; - IndexSpaceNode *projection_space; + std::set projections; unsigned rebuild_timeout; - }; + bool disjoint_shallow; + }; // A helper class for containing field states class FieldStateDeque : public LegionDeque::aligned { @@ -720,31 +775,28 @@ namespace Legion { }; /** - * \class ProjectionEpoch - * This class captures the set of projection functions - * and domains that have performed in current open - * projection epoch + * \class ProjectionTree + * This is a tree that stores the summary of a region + * tree that is accessed by an index launch and which + * node owns the leaves for in the case of control replication */ - class ProjectionEpoch : public LegionHeapify { - public: - static const ProjectionEpochID first_epoch = 1; + class ProjectionTree { public: - ProjectionEpoch(ProjectionEpochID epoch_id, - const FieldMask &mask); - ProjectionEpoch(const ProjectionEpoch &rhs); - ~ProjectionEpoch(void); + ProjectionTree(IndexTreeNode *source, + ShardID owner_shard = 0); + ProjectionTree(const ProjectionTree &rhs); + ~ProjectionTree(void); public: - ProjectionEpoch& operator=(const ProjectionEpoch &rhs); + ProjectionTree& operator=(const ProjectionTree &rhs); public: - void insert(ProjectionFunction *function, IndexSpaceNode *space); + void add_child(ProjectionTree *child); + bool dominates(const ProjectionTree *other) const; + bool disjoint(const ProjectionTree *other) const; + bool all_same_shard(ShardID other_shard) const; public: - const ProjectionEpochID epoch_id; - FieldMask valid_fields; - public: - // For now we only record the write projections since we use them - // for constructing composite view write sets - std::map > write_projections; + IndexTreeNode *const node; + const ShardID owner_shard; + std::map children; }; /** @@ -768,10 +820,6 @@ namespace Legion { void clear_logical_users(void); void reset(void); void clear_deleted_state(const FieldMask &deleted_mask); - public: - void advance_projection_epochs(const FieldMask &advance_mask); - void update_projection_epochs(FieldMask capture_mask, - const ProjectionInfo &info); public: RegionTreeNode *const owner; public: @@ -785,22 +833,19 @@ namespace Legion { // Keep track of which fields we've done a reduction to here FieldMask reduction_fields; LegionMap::aligned outstanding_reductions; - public: - // Keep track of the current projection epoch for each field - std::list projection_epochs; }; typedef DynamicTableAllocator LogicalStateAllocator; /** - * \struct LogicalCloser + * \class LogicalCloser * This structure helps keep track of the state * necessary for performing a close operation * on the logical region tree. */ class LogicalCloser { public: - LogicalCloser(ContextID ctx, const LogicalUser &u, + LogicalCloser(ContextID ctx, const LogicalUser &u, RegionTreeNode *root, bool validates); LogicalCloser(const LogicalCloser &rhs); ~LogicalCloser(void); @@ -858,7 +903,7 @@ namespace Legion { protected: FieldMask close_mask; protected: - // At most we will ever generate three close operations at a node + // At most we will ever generate one close operation at a node MergeCloseOp *close_op; protected: // Cache the generation IDs so we can kick off ops before adding users @@ -966,6 +1011,8 @@ namespace Legion { inline InstanceManager* get_manager(void) const { return manager; } inline const FieldMask& get_valid_fields(void) const { return valid_fields; } + inline void update_fields(const FieldMask &update) + { valid_fields |= update; } public: inline bool is_local(void) const { return local; } MappingInstance get_mapping_instance(void) const; @@ -1645,7 +1692,7 @@ namespace Legion { const PhysicalTraceInfo &trace_info, const ApEvent precondition, const ApEvent term_event, const bool track_effects, const bool check_initialized, - const bool record_valid); + const bool record_valid, const bool skip_output); UpdateAnalysis(Runtime *rt, AddressSpaceID src, AddressSpaceID prev, Operation *op, unsigned index, VersionManager *man, const RegionUsage &usage, RegionNode *node, @@ -1655,7 +1702,7 @@ namespace Legion { const RtEvent user_registered, const ApEvent precondition, const ApEvent term_event, const bool track_effects, const bool check_initialized, - const bool record_valid); + const bool record_valid, const bool skip_output); UpdateAnalysis(const UpdateAnalysis &rhs); virtual ~UpdateAnalysis(void); public: @@ -1685,7 +1732,10 @@ namespace Legion { static void handle_remote_updates(Deserializer &derez, Runtime *rt, AddressSpaceID previous); public: - const RegionUsage usage; + // TODO: make this const again after we update the runtime to + // support collective views so that we don't need to modify + // this field in ReplMapOp::trigger_mapping + /*const*/ RegionUsage usage; RegionNode *const node; const InstanceSet target_instances; const std::vector target_views; @@ -1695,6 +1745,7 @@ namespace Legion { const bool track_effects; const bool check_initialized; const bool record_valid; + const bool skip_output; public: // Have to lock the analysis to access these safely std::map input_aggregators; @@ -2803,7 +2854,7 @@ namespace Legion { protected: const ContextID ctx; const FieldMask &deletion_mask; - }; + }; /** * \class VersioningInvalidator diff --git a/runtime/legion/legion_c.cc b/runtime/legion/legion_c.cc index deeeef4eb6..17b76b53e8 100644 --- a/runtime/legion/legion_c.cc +++ b/runtime/legion/legion_c.cc @@ -3031,6 +3031,14 @@ legion_future_map_get_future(legion_future_map_t fm_, return CObjectWrapper::wrap(new Future(fm->get_future(dp))); } +legion_domain_t +legion_future_map_get_domain(legion_future_map_t fm_) +{ + FutureMap *fm = CObjectWrapper::unwrap(fm_); + const Domain &domain = fm->get_future_map_domain(); + return CObjectWrapper::wrap(domain); +} + legion_future_t legion_future_map_reduce(legion_runtime_t runtime_, legion_context_t ctx_, @@ -6365,6 +6373,64 @@ legion_runtime_replace_default_mapper( runtime->replace_default_mapper(mapper, proc); } +int64_t legion_projection_functor_logical_partition_print_arguments( + legion_runtime_t runtime_, + legion_logical_partition_t upper_bound_, + legion_domain_point_t point_, + legion_domain_t launch_domain_) +{ + // legion_runtime_t runtime_ = CObjectWrapper::unwrap(runtime_); + // legion_logical_partition_t upper_bound_ = CObjectWrapper::unwrap(upper_bound); + // legion_domain_point_t point_ = CObjectWrapper::unwrap(point); + // legion_domain_t launch_domain_ = CObjectWrapper::unwrap(launch_domain); + printf("legion_projection_functor_logical_partition_print_arguments\n"); + return 0; +} + +int64_t legion_projection_functor_logical_partition_print_arguments_1( + legion_runtime_t runtime_) +{ + // legion_runtime_t runtime_ = CObjectWrapper::unwrap(runtime_); + // legion_logical_partition_t upper_bound_ = CObjectWrapper::unwrap(upper_bound); + // legion_domain_point_t point_ = CObjectWrapper::unwrap(point); + // legion_domain_t launch_domain_ = CObjectWrapper::unwrap(launch_domain); + printf("legion_projection_functor_logical_partition_print_arguments_1\n"); + return 0; +} + +int64_t legion_projection_functor_logical_partition_print_arguments_2( + legion_logical_partition_t upper_bound_) +{ + // legion_runtime_t runtime_ = CObjectWrapper::unwrap(runtime_); + // legion_logical_partition_t upper_bound_ = CObjectWrapper::unwrap(upper_bound); + // legion_domain_point_t point_ = CObjectWrapper::unwrap(point); + // legion_domain_t launch_domain_ = CObjectWrapper::unwrap(launch_domain); + printf("legion_projection_functor_logical_partition_print_arguments_2\n"); + return 0; +} + +int64_t legion_projection_functor_logical_partition_print_arguments_3( + legion_domain_point_t point_) +{ + // legion_runtime_t runtime_ = CObjectWrapper::unwrap(runtime_); + // legion_logical_partition_t upper_bound_ = CObjectWrapper::unwrap(upper_bound); + // legion_domain_point_t point_ = CObjectWrapper::unwrap(point); + // legion_domain_t launch_domain_ = CObjectWrapper::unwrap(launch_domain); + printf("legion_projection_functor_logical_partition_print_arguments_3\n"); + return 0; +} + +int64_t legion_projection_functor_logical_partition_print_arguments_4( + legion_domain_t domain_) +{ + // legion_runtime_t runtime_ = CObjectWrapper::unwrap(runtime_); + // legion_logical_partition_t upper_bound_ = CObjectWrapper::unwrap(upper_bound); + // legion_domain_point_t point_ = CObjectWrapper::unwrap(point); + // legion_domain_t launch_domain_ = CObjectWrapper::unwrap(launch_domain); + printf("legion_projection_functor_logical_partition_print_arguments_4\n"); + return 0; +} + class FunctorWrapper : public ProjectionFunctor { public: FunctorWrapper(bool exc, bool func, unsigned dep, @@ -6873,7 +6939,10 @@ legion_task_preamble( regions, ctx, runtime); - + printf("num logical regions %lu\n", regions->size()); + for (size_t i = 0; i < regions->size(); i++) { + printf("region %lu is %ld %ld %ld\n", i, (*regions)[i].get_logical_region().get_tree_id(), (*regions)[i].get_logical_region().get_index_space().get_tree_id(), (*regions)[i].get_logical_region().get_index_space().get_id()); + } CContext *cctx = new CContext(ctx, *regions); *taskptr = CObjectWrapper::wrap_const(task); *regionptr = cctx->regions(); @@ -7701,6 +7770,20 @@ legion_context_get_num_shards(legion_runtime_t runtime_, return runtime->get_num_shards(ctx, I_know_what_I_am_doing); } +legion_future_t +legion_context_consensus_match(legion_runtime_t runtime_, + legion_context_t context_, + const void *input, void *output, + size_t num_elements, size_t element_size) +{ + Runtime *runtime = CObjectWrapper::unwrap(runtime_); + Context ctx = CObjectWrapper::unwrap(context_)->context(); + + Future f = runtime->consensus_match(ctx, input, output, + num_elements, element_size); + return CObjectWrapper::wrap(new Future(f)); +} + legion_physical_region_t legion_get_physical_region_by_id( legion_physical_region_t *regionptr, diff --git a/runtime/legion/legion_c.h b/runtime/legion/legion_c.h index 0f4d894231..2ba74b511b 100644 --- a/runtime/legion/legion_c.h +++ b/runtime/legion/legion_c.h @@ -408,6 +408,20 @@ extern "C" { legion_domain_point_t /* point */, legion_domain_t /* launch domain */); + int64_t legion_projection_functor_logical_partition_print_arguments( + legion_runtime_t /* runtime */, + legion_logical_partition_t /* upper_bound */, + legion_domain_point_t /* point */, + legion_domain_t /* launch domain */); + int64_t legion_projection_functor_logical_partition_print_arguments_1( + legion_runtime_t /* point */); + int64_t legion_projection_functor_logical_partition_print_arguments_2( + legion_logical_partition_t /* point */); + int64_t legion_projection_functor_logical_partition_print_arguments_3( + legion_domain_point_t /* point */); + int64_t legion_projection_functor_logical_partition_print_arguments_4( + legion_domain_t /* point */); + /** * Interface for a Legion C projection functor (Logical Region * upper bound). @@ -2557,6 +2571,12 @@ extern "C" { legion_future_map_get_future(legion_future_map_t handle, legion_domain_point_t point); + /** + * @see Legion::FutureMap::get_future_map_domain + */ + legion_domain_t + legion_future_map_get_domain(legion_future_map_t handle); + /** * @return Caller takes ownership of return value * @@ -2580,7 +2600,7 @@ extern "C" { legion_domain_t domain, legion_domain_point_t *points, legion_future_t *futures, - size_t num_futures); + size_t num_futures); // ----------------------------------------------------------------------- // Deferred Buffer Operations @@ -5629,7 +5649,18 @@ extern "C" { legion_context_get_num_shards(legion_runtime_t /*runtime*/, legion_context_t /*context*/, bool /*I know what I am doing*/); - + // Another hidden method for control replication that most + // people should not be using but for which there are legitamite + // user, especially in garbage collected languages + // Note the caller takes ownership of the future + legion_future_t + legion_context_consensus_match(legion_runtime_t /*runtime*/, + legion_context_t /*context*/, + const void* /*input*/, + void* /*output*/, + size_t /*num elements*/, + size_t /*element size*/); + /** * used by fortran API */ diff --git a/runtime/legion/legion_config.h b/runtime/legion/legion_config.h index cca1c16cce..c8f9305139 100644 --- a/runtime/legion/legion_config.h +++ b/runtime/legion/legion_config.h @@ -114,13 +114,13 @@ // Try to be nice in case someone else defined this #ifndef LEGION_DISABLE_DEPRECATED_ENUMS #ifndef GC_FIRST_PRIORITY -#define GC_FIRST_PRIORITY LEGION_GC_MAX_PRIORITY +#define GC_FIRST_PRIORITY LEGION_GC_FIRST_PRIORITY #endif #ifndef GC_DEFAULT_PRIORITY -#define GC_DEFAULT_PRIORITY 0 +#define GC_DEFAULT_PRIORITY LEGION_GC_DEFAULT_PRIORITY #endif #ifndef GC_LAST_PRIORITY -#define GC_LAST_PRIORITY (LEGION_GC_MIN_PRIORITY+1) +#define GC_LAST_PRIORITY LEGION_GC_LAST_PRIORITY #endif #endif @@ -200,6 +200,10 @@ #define LEGION_MAX_APPLICATION_MAPPER_ID (MAX_APPLICATION_MAPPER_ID) #endif #endif +// Maximum ID for an application reduction ID +#ifndef MAX_APPLICATION_REDUCTION_ID +#define MAX_APPLICATION_REDUCTION_ID (1<<20) +#endif // Maximum ID for an application trace ID #ifndef LEGION_MAX_APPLICATION_TRACE_ID #define LEGION_MAX_APPLICATION_TRACE_ID (1<<20) @@ -358,6 +362,15 @@ #define LEGION_DEFAULT_GC_EPOCH_SIZE (DEFAULT_GC_EPOCH_SIZE) #endif #endif +// Number of control replications to be supported +#ifndef LEGION_DEFAULT_MAX_CONTROL_REPLICATION_CONTEXTS +#define LEGION_DEFAULT_MAX_CONTROL_REPLICATION_CONTEXTS 1 +#endif +// Number of phase barriers for communication of +// close operation composite view meta data +#ifndef LEGION_CONTROL_REPLICATION_COMMUNICATION_BARRIERS +#define LEGION_CONTROL_REPLICATION_COMMUNICATION_BARRIERS 32 +#endif // Used for debugging memory leaks // How often tracing information is dumped @@ -1100,7 +1113,6 @@ typedef enum legion_error_t { ERROR_ILLEGAL_RUNTIME_REMAPPING = 377, ERROR_UNABLE_FIND_TASK_LOCAL = 378, ERROR_INDEXPARTITION_NOT_SAME_INDEX_TREE = 379, - ERROR_TASK_ATTEMPTED_ALLOCATE_FILED = 386, ERROR_EXCEEDED_MAXIMUM_NUMBER_LOCAL_FIELDS = 387, ERROR_UNABLE_ALLOCATE_LOCAL_FIELD = 388, ERROR_TASK_ATTEMPTED_ALLOCATE_FIELD = 389, @@ -1197,7 +1209,7 @@ typedef enum legion_error_t { ERROR_UNKNOWN_PROFILER_OPTION = 539, ERROR_MISSING_PROFILER_OPTION = 540, ERROR_INVALID_PROFILER_SERIALIZER = 541, - ERROR_INVALID_PROFILER_FILE = 542, + ERROR_INVALID_PROFILER_FILE = 542, ERROR_ILLEGAL_LAYOUT_CONSTRAINT = 543, ERROR_UNSUPPORTED_LAYOUT_CONSTRAINT = 544, ERROR_ACCESSOR_FIELD_SIZE_CHECK = 545, @@ -1225,11 +1237,19 @@ typedef enum legion_error_t { ERROR_INVALID_PARTITION_BY_WEIGHT_VALUE = 567, ERROR_LEGION_CONFIGURATION = 568, ERROR_CREATION_FUTURE_TYPE_MISMATCH = 569, - ERROR_ILLEGAL_LOCAL_FUNCTION_TASK_LAUNCH = 570, - ERROR_ILLEGAL_SHARED_OWNERSHIP = 571, - ERROR_NON_PIECE_RECTANGLE = 572, - ERROR_ILLEGAL_PERFORM_REGISTRATION_CALLBACK = 573, - ERROR_CONFUSED_USER = 574, + ERROR_ARGUMENT_MAP_DIMENSIONALITY = 570, + ERROR_INVALID_FUTURE_MAP_POINT = 571, + ERROR_ILLEGAL_LOCAL_FUNCTION_TASK_LAUNCH = 572, + ERROR_ILLEGAL_SHARED_OWNERSHIP = 573, + ERROR_ILLEGAL_PERFORM_REGISTRATION_CALLBACK = 574, + ERROR_NON_PIECE_RECTANGLE = 575, + ERROR_RESERVED_SHARDING_ID = 601, + ERROR_DUPLICATE_SHARDING_ID = 602, + ERROR_INVALID_SHARDING_ID = 603, + ERROR_REPLICATE_TASK_VIOLATION = 604, + ERROR_ILLEGAL_SHARDING_FUNCTOR_OUTPUT = 605, + ERROR_CONFUSED_USER = 606, + ERROR_CONTROL_REPLICATION_VIOLATION = 607, LEGION_WARNING_FUTURE_NONLEAF = 1000, @@ -1290,6 +1310,10 @@ typedef enum legion_error_t { LEGION_WARNING_NEW_TEMPLATE_COUNT_EXCEEDED = 1102, LEGION_WARNING_NON_CALLBACK_REGISTRATION = 1103, LEGION_WARNING_COLLECTIVE_INSTANCE_VIOLATION = 1104, + LEGION_WARNING_DYNAMIC_SHARDING_REG = 1105, + LEGION_WARNING_SLOW_NON_FUNCTIONAL_PROJECTION = 1106, + LEGION_WARNING_MISMATCHED_REPLICATED_FUTURES = 1107, + LEGION_WARNING_INLINING_NOT_SUPPORTED = 1108, LEGION_FATAL_MUST_EPOCH_NOADDRESS = 2000, @@ -1298,12 +1322,14 @@ typedef enum legion_error_t { LEGION_FATAL_SHIM_MAPPER_SUPPORT = 2006, LEGION_FATAL_UNKNOWN_FIELD_ID = 2007, LEGION_FATAL_RESTRICTED_SIMULTANEOUS = 2008, - LEGION_FATAL_INCONSISTENT_PHI_VIEW = 2009, - LEGION_FATAL_EXCEEDED_LIBRARY_ID_OFFSET = 2010, - LEGION_FATAL_SEPARATE_RUNTIME_INSTANCES = 2011, - LEGION_FATAL_UNIMPLEMENTED_FEATURE = 2012, - LEGION_FATAL_CALLBACK_NOT_PORTABLE = 2013, - LEGION_FATAL_REDUCTION_ABA_PROBLEM = 2014, + LEGION_FATAL_CTRL_REPL_RETURN_PRIV = 2009, + LEGION_FATAL_UNIMPLEMENTED_FEATURE = 2010, + LEGION_FATAL_INCONSISTENT_PHI_VIEW = 2011, + LEGION_FATAL_EXCEEDED_LIBRARY_ID_OFFSET = 2012, + LEGION_FATAL_SEPARATE_RUNTIME_INSTANCES = 2013, + LEGION_FATAL_UNSUPPORTED_CONSENSUS_SIZE = 2014, + LEGION_FATAL_CALLBACK_NOT_PORTABLE = 2015, + LEGION_FATAL_REDUCTION_ABA_PROBLEM = 2016, } legion_error_t; diff --git a/runtime/legion/legion_context.cc b/runtime/legion/legion_context.cc index 903b2ea6de..ba8fa71dde 100644 --- a/runtime/legion/legion_context.cc +++ b/runtime/legion/legion_context.cc @@ -19,6 +19,7 @@ #include "legion/legion_context.h" #include "legion/legion_instances.h" #include "legion/legion_views.h" +#include "legion/legion_replication.h" #define SWAP_PART_KINDS(k1, k2) \ { \ @@ -40,9 +41,8 @@ namespace Legion { TaskContext::TaskContext(Runtime *rt, TaskOp *owner, int d, const std::vector &reqs) : runtime(rt), owner_task(owner), regions(reqs), depth(d), - next_created_index(reqs.size()), - executing_processor(Processor::NO_PROC), total_tunable_count(0), - overhead_tracker(NULL), task_executed(false), + next_created_index(reqs.size()),executing_processor(Processor::NO_PROC), + total_tunable_count(0), overhead_tracker(NULL), task_executed(false), has_inline_accessor(false), mutable_priority(false), children_complete_invoked(false), children_commit_invoked(false) //-------------------------------------------------------------------------- @@ -145,6 +145,46 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + void TaskContext::print_once(FILE *f, const char *message) const + //-------------------------------------------------------------------------- + { + fprintf(f, "%s", message); + } + + //-------------------------------------------------------------------------- + void TaskContext::log_once(Realm::LoggerMessage &message) const + //-------------------------------------------------------------------------- + { + // Do nothing, just don't deactivate it + } + + //-------------------------------------------------------------------------- + ShardID TaskContext::get_shard_id(void) const + //-------------------------------------------------------------------------- + { + return 0; + } + + //-------------------------------------------------------------------------- + size_t TaskContext::get_num_shards(void) const + //-------------------------------------------------------------------------- + { + return 1; + } + + //-------------------------------------------------------------------------- + Future TaskContext::consensus_match(const void *input, void *output, + size_t num_elements,size_t element_size) + //-------------------------------------------------------------------------- + { + // No need to do a match here, there is just one shard + memcpy(output, input, num_elements * element_size); + Future result = runtime->help_create_future(ApEvent::NO_AP_EVENT); + result.impl->set_result(&num_elements, sizeof(num_elements),false/*own*/); + return result; + } + //-------------------------------------------------------------------------- VariantID TaskContext::register_variant( const TaskVariantRegistrar ®istrar, const void *user_data, @@ -177,6 +217,13 @@ namespace Legion { return runtime->generate_dynamic_projection_id(false/*check context*/); } + //-------------------------------------------------------------------------- + ShardingID TaskContext::generate_dynamic_sharding_id(void) + //-------------------------------------------------------------------------- + { + return runtime->generate_dynamic_sharding_id(false/*check context*/); + } + //-------------------------------------------------------------------------- TaskID TaskContext::generate_dynamic_task_id(void) //-------------------------------------------------------------------------- @@ -262,9 +309,9 @@ namespace Legion { const std::vector &spaces) //-------------------------------------------------------------------------- { + AutoRuntimeCall call(this); if (spaces.empty()) return IndexSpace::NO_SPACE; - AutoRuntimeCall call(this); bool none_exists = true; for (std::vector::const_iterator it = spaces.begin(); it != spaces.end(); it++) @@ -294,9 +341,9 @@ namespace Legion { const std::vector &spaces) //-------------------------------------------------------------------------- { + AutoRuntimeCall call(this); if (spaces.empty()) return IndexSpace::NO_SPACE; - AutoRuntimeCall call(this); bool none_exists = true; for (std::vector::const_iterator it = spaces.begin(); it != spaces.end(); it++) @@ -446,7 +493,8 @@ namespace Legion { } //-------------------------------------------------------------------------- - FieldAllocatorImpl* TaskContext::create_field_allocator(FieldSpace handle) + FieldAllocatorImpl* TaskContext::create_field_allocator(FieldSpace handle, + bool unordered) //-------------------------------------------------------------------------- { AutoRuntimeCall call(this); @@ -468,7 +516,7 @@ namespace Legion { const RtEvent ready = runtime->forest->create_field_space_allocator(handle); // Don't have one so make a new one - FieldAllocatorImpl *result = new FieldAllocatorImpl(handle,this,ready); + FieldAllocatorImpl *result = new FieldAllocatorImpl(handle, this, ready); // Save it for later field_allocators[handle] = result; return result; @@ -500,25 +548,43 @@ namespace Legion { fid = runtime->get_unique_field_id(); #ifdef DEBUG_LEGION else if (fid >= LEGION_MAX_APPLICATION_FIELD_ID) - REPORT_LEGION_ERROR(ERROR_TASK_ATTEMPTED_ALLOCATE_FILED, - "Task %s (ID %lld) attempted to allocate a field with " - "ID %d which exceeds the LEGION_MAX_APPLICATION_FIELD_ID " - "bound set in legion_config.h", get_task_name(), get_unique_id(), fid) + REPORT_LEGION_ERROR(ERROR_TASK_ATTEMPTED_ALLOCATE_FIELD, + "Task %s (ID %lld) attempted to allocate a field with ID %d which " + "exceeds the LEGION_MAX_APPLICATION_FIELD_ID bound set in " + "legion_config.h", get_task_name(), get_unique_id(), fid) #endif if (runtime->legion_spy_enabled) LegionSpy::log_field_creation(space.id, fid, field_size); std::set done_events; - if (local) + if (!local) + { + const RtEvent precondition = + runtime->forest->allocate_field(space, field_size, fid, serdez_id); + if (precondition.exists()) + done_events.insert(precondition); + } + else allocate_local_field(space, field_size, fid, serdez_id, done_events); - else - runtime->forest->allocate_field(space, field_size, fid, serdez_id); register_field_creation(space, fid, local); if (!done_events.empty()) { - RtEvent wait_on = Runtime::merge_events(done_events); - wait_on.wait(); + const RtEvent precondition = Runtime::merge_events(done_events); + if (precondition.exists() && !precondition.has_triggered()) + { + if (is_inner_context()) + { + InnerContext *ctx = static_cast(this); + // Need a fence to make sure that no one tries to use these + // fields where they haven't been made visible yet + CreationOp *creator = runtime->get_available_creation_op(); + creator->initialize_fence(ctx, precondition); + add_to_dependence_queue(creator); + } + else + precondition.wait(); + } } return fid; } @@ -550,17 +616,34 @@ namespace Legion { resulting_fields[idx], sizes[idx]); } std::set done_events; - if (local) + if (!local) + { + const RtEvent precondition = runtime->forest->allocate_fields(space, + sizes, resulting_fields, serdez_id); + if (precondition.exists()) + done_events.insert(precondition); + } + else allocate_local_fields(space, sizes, resulting_fields, serdez_id, done_events); - else - runtime->forest->allocate_fields(space, sizes, - resulting_fields, serdez_id); register_all_field_creations(space, local, resulting_fields); if (!done_events.empty()) { - RtEvent wait_on = Runtime::merge_events(done_events); - wait_on.wait(); + const RtEvent precondition = Runtime::merge_events(done_events); + if (precondition.exists() && !precondition.has_triggered()) + { + if (is_inner_context()) + { + // Need a fence to make sure that no one tries to use these + // fields where they haven't been made visible yet + InnerContext *ctx = static_cast(this); + CreationOp *creator = runtime->get_available_creation_op(); + creator->initialize_fence(ctx, precondition); + add_to_dependence_queue(creator); + } + else + precondition.wait(); + } } } @@ -806,7 +889,9 @@ namespace Legion { { AutoLock priv_lock(privilege_lock); #ifdef DEBUG_LEGION - assert(created_index_spaces.find(space) == created_index_spaces.end()); + // This assertion is not valid anymore because of aliased sharded + // index spaces in control replication contexts + //assert(created_index_spaces.find(space) == created_index_spaces.end()); #endif created_index_spaces[space] = 1; } @@ -2415,8 +2500,12 @@ namespace Legion { const IndexTaskLauncher &launcher) //-------------------------------------------------------------------------- { - FutureMapImpl *result = new FutureMapImpl(this, runtime, - runtime->get_available_distributed_id(), + Domain launch_domain = launcher.launch_domain; + if (!launch_domain.exists()) + runtime->forest->find_launch_space_domain(launcher.launch_space, + launch_domain); + FutureMapImpl *result = new FutureMapImpl(this, runtime, + launch_domain, runtime->get_available_distributed_id(), runtime->address_space, RtEvent::NO_RT_EVENT); if (launcher.predicate_false_future.impl != NULL) { @@ -2431,7 +2520,7 @@ namespace Legion { for (Domain::DomainPointIterator itr(launcher.launch_domain); itr; itr++) { - Future f = result->get_future(itr.p); + Future f = result->get_future(itr.p, true/*internal*/); f.impl->set_result(f_result, f_result_size, false/*own*/); } } @@ -2471,7 +2560,7 @@ namespace Legion { for (Domain::DomainPointIterator itr(launcher.launch_domain); itr; itr++) { - Future f = result->get_future(itr.p); + Future f = result->get_future(itr.p, true/*internal*/); f.impl->set_result(NULL, 0, false/*own*/); } } @@ -2482,7 +2571,7 @@ namespace Legion { for (Domain::DomainPointIterator itr(launcher.launch_domain); itr; itr++) { - Future f = result->get_future(itr.p); + Future f = result->get_future(itr.p, true/*internal*/); f.impl->set_result(ptr, ptr_size, false/*own*/); } } @@ -2989,7 +3078,8 @@ namespace Legion { delete_now.begin(); it != delete_now.end(); it++) { DeletionOp *op = runtime->get_available_deletion_op(); - FieldAllocatorImpl *allocator = create_field_allocator(it->first); + FieldAllocatorImpl *allocator = + create_field_allocator(it->first, true/*unordered*/); op->initialize_field_deletions(this, it->first, it->second, true/*unordered*/, allocator); op->set_execution_precondition(precondition); @@ -3567,6 +3657,35 @@ namespace Legion { return ready; } + //-------------------------------------------------------------------------- + EquivalenceSet* InnerContext::find_or_create_top_equivalence_set( + RegionTreeID tree_id) + //-------------------------------------------------------------------------- + { + RegionNode *root_node = runtime->forest->get_tree(tree_id); + IndexSpaceExpression *root_expr = + root_node->get_index_space_expression(); + AutoLock tree_lock(tree_set_lock); + // See if we lost the race + std::map::const_iterator finder = + tree_equivalence_sets.find(tree_id); + if (finder == tree_equivalence_sets.end()) + { + // Didn't loose the race so we have to make the top-level + // equivalence set for this region tree + const AddressSpaceID local_space = runtime->address_space; + EquivalenceSet *root = new EquivalenceSet(runtime, + runtime->get_available_distributed_id(), + local_space, local_space, root_expr, root_node->row_source, + true/*register now*/); + tree_equivalence_sets[tree_id] = root; + root->add_base_resource_ref(CONTEXT_REF); + return root; + } + else + return finder->second; + } + //-------------------------------------------------------------------------- InnerContext* InnerContext::find_parent_logical_context(unsigned index) //-------------------------------------------------------------------------- @@ -3652,7 +3771,7 @@ namespace Legion { //-------------------------------------------------------------------------- void InnerContext::pack_remote_context(Serializer &rez, - AddressSpaceID target) + AddressSpaceID target,bool replicate) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, PACK_REMOTE_CONTEXT_CALL); @@ -3691,6 +3810,7 @@ namespace Legion { for (unsigned idx = 0; idx < it->second.size(); idx++) rez.serialize(it->second[idx]); } + rez.serialize(replicate); } //-------------------------------------------------------------------------- @@ -3698,7 +3818,7 @@ namespace Legion { std::set &preconditions) //-------------------------------------------------------------------------- { - assert(false); // should only be called for RemoteTask + assert(false); // should only be called for RemoteContext } //-------------------------------------------------------------------------- @@ -3742,13 +3862,13 @@ namespace Legion { // Get a new creation operation CreationOp *creator_op = runtime->get_available_creation_op(); const ApEvent ready = creator_op->get_completion_event(); - IndexSpaceNode *node = - runtime->forest->create_index_space(handle, NULL, did, ready); + IndexSpaceNode *node = runtime->forest->create_index_space(handle, + NULL/*domain*/, did, true/*notify remote*/, 0/*expr id*/, ready); creator_op->initialize_index_space(this, node, future); register_index_space_creation(handle); add_to_dependence_queue(creator_op); return handle; - } + } //-------------------------------------------------------------------------- void InnerContext::destroy_index_space(IndexSpace handle, @@ -4424,6 +4544,26 @@ namespace Legion { return pid; } + //-------------------------------------------------------------------------- + IndexPartition InnerContext::create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color) + //-------------------------------------------------------------------------- + { + ArgumentMap argmap; + for (std::map::const_iterator it = + domains.begin(); it != domains.end(); it++) + argmap.set_point(it->first, + TaskArgument(&it->second, sizeof(it->second))); + FutureMap future_map(argmap.impl->freeze(this)); + return create_partition_by_domain(parent, future_map, color_space, + perform_intersections, part_kind,color); + } + //-------------------------------------------------------------------------- IndexPartition InnerContext::create_partition_by_domain( IndexSpace parent, @@ -4839,10 +4979,11 @@ namespace Legion { LegionColor part_color = INVALID_COLOR; if (color != LEGION_AUTO_GENERATE_ID) part_color = color; - const ApUserEvent partition_ready = Runtime::create_ap_user_event(NULL); - RtEvent safe = runtime->forest->create_pending_partition(this, pid, - parent, color_space, part_color, part_kind, - did, partition_ready, partition_ready); + size_t color_space_size = runtime->forest->get_domain_volume(color_space); + const ApBarrier partition_ready( + Realm::Barrier::create_barrier(color_space_size)); + RtEvent safe = runtime->forest->create_pending_partition(this, pid,parent, + color_space, part_color, part_kind,did,partition_ready,partition_ready); // Wait for any notifications to occur before returning if (safe.exists()) safe.wait(); @@ -4861,6 +5002,7 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace InnerContext::create_index_space_union(IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles) //-------------------------------------------------------------------------- @@ -4870,13 +5012,11 @@ namespace Legion { log_index.debug("Creating index space union in task %s (ID %lld)", get_task_name(), get_unique_id()); #endif - ApUserEvent domain_ready; - IndexSpace result = runtime->forest->find_pending_space(parent, - realm_color, type_tag, domain_ready); PendingPartitionOp *part_op = runtime->get_available_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); part_op->initialize_index_space_union(this, result, handles); - Runtime::trigger_event(NULL,domain_ready,part_op->get_completion_event()); // Now we can add the operation to the queue add_to_dependence_queue(part_op); return result; @@ -4885,6 +5025,7 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace InnerContext::create_index_space_union(IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) //-------------------------------------------------------------------------- @@ -4894,13 +5035,11 @@ namespace Legion { log_index.debug("Creating index space union in task %s (ID %lld)", get_task_name(), get_unique_id()); #endif - ApUserEvent domain_ready; - IndexSpace result = runtime->forest->find_pending_space(parent, - realm_color, type_tag, domain_ready); PendingPartitionOp *part_op = runtime->get_available_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); part_op->initialize_index_space_union(this, result, handle); - Runtime::trigger_event(NULL,domain_ready,part_op->get_completion_event()); // Now we can add the operation to the queue add_to_dependence_queue(part_op); return result; @@ -4910,6 +5049,7 @@ namespace Legion { IndexSpace InnerContext::create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles) //-------------------------------------------------------------------------- @@ -4919,13 +5059,11 @@ namespace Legion { log_index.debug("Creating index space intersection in task %s (ID %lld)", get_task_name(), get_unique_id()); #endif - ApUserEvent domain_ready; - IndexSpace result = runtime->forest->find_pending_space(parent, - realm_color, type_tag, domain_ready); PendingPartitionOp *part_op = runtime->get_available_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); part_op->initialize_index_space_intersection(this, result, handles); - Runtime::trigger_event(NULL,domain_ready,part_op->get_completion_event()); // Now we can add the operation to the queue add_to_dependence_queue(part_op); return result; @@ -4935,6 +5073,7 @@ namespace Legion { IndexSpace InnerContext::create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) //-------------------------------------------------------------------------- @@ -4944,13 +5083,11 @@ namespace Legion { log_index.debug("Creating index space intersection in task %s (ID %lld)", get_task_name(), get_unique_id()); #endif - ApUserEvent domain_ready; - IndexSpace result = runtime->forest->find_pending_space(parent, - realm_color, type_tag, domain_ready); PendingPartitionOp *part_op = runtime->get_available_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); part_op->initialize_index_space_intersection(this, result, handle); - Runtime::trigger_event(NULL,domain_ready,part_op->get_completion_event()); // Now we can add the operation to the queue add_to_dependence_queue(part_op); return result; @@ -4960,6 +5097,7 @@ namespace Legion { IndexSpace InnerContext::create_index_space_difference( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexSpace initial, const std::vector &handles) @@ -4970,13 +5108,11 @@ namespace Legion { log_index.debug("Creating index space difference in task %s (ID %lld)", get_task_name(), get_unique_id()); #endif - ApUserEvent domain_ready; - IndexSpace result = runtime->forest->find_pending_space(parent, - realm_color, type_tag, domain_ready); PendingPartitionOp *part_op = runtime->get_available_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); part_op->initialize_index_space_difference(this, result, initial,handles); - Runtime::trigger_event(NULL,domain_ready,part_op->get_completion_event()); // Now we can add the operation to the queue add_to_dependence_queue(part_op); return result; @@ -5221,6 +5357,26 @@ namespace Legion { vargs->proxy_this->verify_partition(vargs->pid, vargs->kind, vargs->func); } + //-------------------------------------------------------------------------- + FieldSpace InnerContext::create_field_space(void) + //-------------------------------------------------------------------------- + { + return TaskContext::create_field_space(); + } + + //-------------------------------------------------------------------------- + FieldSpace InnerContext::create_field_space( + const std::vector &sizes, + std::vector &resulting_fields, + CustomSerdezID serdez_id) + //-------------------------------------------------------------------------- + { + FieldSpace result = TaskContext::create_field_space(); + TaskContext::allocate_fields(result, sizes, resulting_fields, + false/*local*/, serdez_id); + return result; + } + //-------------------------------------------------------------------------- FieldSpace InnerContext::create_field_space( const std::vector &sizes, @@ -5320,7 +5476,7 @@ namespace Legion { fid = runtime->get_unique_field_id(); #ifdef DEBUG_LEGION else if (fid >= LEGION_MAX_APPLICATION_FIELD_ID) - REPORT_LEGION_ERROR(ERROR_TASK_ATTEMPTED_ALLOCATE_FILED, + REPORT_LEGION_ERROR(ERROR_TASK_ATTEMPTED_ALLOCATE_FIELD, "Task %s (ID %lld) attempted to allocate a field with " "ID %d which exceeds the LEGION_MAX_APPLICATION_FIELD_ID " "bound set in legion_config.h", get_task_name(), get_unique_id(), fid) @@ -5334,9 +5490,10 @@ namespace Legion { const ApEvent ready = creator_op->get_completion_event(); // Tell the node that we're allocating a field of size zero // which will indicate that we'll fill in the size later - FieldSpaceNode *node = - runtime->forest->allocate_field(space, ready, fid, serdez_id); - creator_op->initialize_field(this, node, fid, field_size); + RtEvent precondition; + FieldSpaceNode *node = runtime->forest->allocate_field(space, ready, fid, + serdez_id, precondition); + creator_op->initialize_field(this, node, fid, field_size, precondition); register_field_creation(space, fid, local); add_to_dependence_queue(creator_op); return fid; @@ -5440,9 +5597,11 @@ namespace Legion { const ApEvent ready = creator_op->get_completion_event(); // Tell the node that we're allocating a field of size zero // which will indicate that we'll fill in the size later + RtEvent precondition; FieldSpaceNode *node = runtime->forest->allocate_fields(space, ready, - resulting_fields, serdez_id); - creator_op->initialize_fields(this, node, resulting_fields, sizes); + resulting_fields, serdez_id, precondition); + creator_op->initialize_fields(this, node, resulting_fields, + sizes, precondition); register_all_field_creations(space, local, resulting_fields); add_to_dependence_queue(creator_op); } @@ -5598,6 +5757,8 @@ namespace Legion { //-------------------------------------------------------------------------- { AutoRuntimeCall call(this); + if (!handle.exists()) + return; #ifdef DEBUG_LEGION log_region.debug("Deleting logical region (%x,%x) in task %s (ID %lld)", handle.index_space.id, handle.field_space.id, @@ -5750,9 +5911,6 @@ namespace Legion { return result; } AutoRuntimeCall call(this); - // Quick out for predicate false - if (launcher.predicate == Predicate::FALSE_PRED) - return predicate_index_task_false(launcher); if (launcher.launch_domain.exists() && (launcher.launch_domain.get_volume() == 0)) { @@ -5761,6 +5919,9 @@ namespace Legion { get_task_name(), get_unique_id()); return FutureMap(); } + // Quick out for predicate false + if (launcher.predicate == Predicate::FALSE_PRED) + return predicate_index_task_false(launcher); IndexSpace launch_space = launcher.launch_space; if (!launch_space.exists()) launch_space = find_index_launch_space(launcher.launch_domain); @@ -5792,9 +5953,6 @@ namespace Legion { return reduce_future_map(result, redop, deterministic); } AutoRuntimeCall call(this); - // Quick out for predicate false - if (launcher.predicate == Predicate::FALSE_PRED) - return predicate_index_task_reduce_false(launcher); if (launcher.launch_domain.exists() && (launcher.launch_domain.get_volume() == 0)) { @@ -5803,6 +5961,9 @@ namespace Legion { get_task_name(), get_unique_id()); return Future(); } + // Quick out for predicate false + if (launcher.predicate == Predicate::FALSE_PRED) + return predicate_index_task_reduce_false(launcher); IndexSpace launch_space = launcher.launch_space; if (!launch_space.exists()) launch_space = find_index_launch_space(launcher.launch_domain); @@ -5838,7 +5999,8 @@ namespace Legion { //-------------------------------------------------------------------------- FutureMap InnerContext::construct_future_map(const Domain &domain, - const std::map &futures, bool internal) + const std::map &futures, + RtUserEvent domain_deletion, bool internal) //-------------------------------------------------------------------------- { if (!internal) @@ -5850,13 +6012,15 @@ namespace Legion { "does not match the volume of the domain (%zd) for the future map " "in task %s (UID %lld)", futures.size(), domain.get_volume(), get_task_name(), get_unique_id()) - return construct_future_map(domain, futures, true/*internal*/); + return construct_future_map(domain, futures, + domain_deletion, true/*internal*/); } CreationOp *creation_op = runtime->get_available_creation_op(); creation_op->initialize_map(this, futures); const DistributedID did = runtime->get_available_distributed_id(); FutureMapImpl *impl = new FutureMapImpl(this, creation_op, - RtEvent::NO_RT_EVENT, runtime, did, runtime->address_space); + RtEvent::NO_RT_EVENT, domain, runtime, + did, runtime->address_space, domain_deletion); add_to_dependence_queue(creation_op); impl->set_all_futures(futures); return FutureMap(impl); @@ -6248,7 +6412,6 @@ namespace Legion { void InnerContext::progress_unordered_operations(void) //-------------------------------------------------------------------------- { - bool issue_task = false; RtEvent precondition; Operation *op = NULL; { @@ -6256,24 +6419,17 @@ namespace Legion { // If we have any unordered ops and we're not in the middle of // a trace then add them into the queue if (!unordered_ops.empty() && (current_trace == NULL)) - insert_unordered_ops(d_lock); - if (dependence_queue.empty()) + insert_unordered_ops(d_lock, false/*end task*/, true/*progress*/); + if (dependence_queue.empty() || outstanding_dependence) return; - if (!outstanding_dependence) - { - issue_task = true; - outstanding_dependence = true; - precondition = dependence_precondition; - dependence_precondition = RtEvent::NO_RT_EVENT; - op = dependence_queue.front(); - } - } - if (issue_task) - { - DependenceArgs args(op, this); - const LgPriority priority = LG_THROUGHPUT_WORK_PRIORITY; - runtime->issue_runtime_meta_task(args, priority, precondition); + outstanding_dependence = true; + precondition = dependence_precondition; + dependence_precondition = RtEvent::NO_RT_EVENT; + op = dependence_queue.front(); } + DependenceArgs args(op, this); + const LgPriority priority = LG_THROUGHPUT_WORK_PRIORITY; + runtime->issue_runtime_meta_task(args, priority, precondition); } //-------------------------------------------------------------------------- @@ -6295,11 +6451,11 @@ namespace Legion { #endif AutoRuntimeCall call(this); MustEpochOp *epoch_op = runtime->get_available_epoch_op(); - FutureMap result = epoch_op->initialize(this, launcher); #ifdef DEBUG_LEGION log_run.debug("Executing a must epoch in task %s (ID %lld)", get_task_name(), get_unique_id()); #endif + FutureMap result = epoch_op->initialize(this, launcher); // Now find all the parent task regions we need to invalidate std::vector unmapped_regions; if (!runtime->unsafe_launch) @@ -6506,6 +6662,117 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + ApBarrier InnerContext::create_phase_barrier(unsigned arrivals, + ReductionOpID redop, + const void *init_value, + size_t init_size) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Creating application barrier in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + return ApBarrier(Realm::Barrier::create_barrier(arrivals, redop, + init_value, init_size)); + } + + //-------------------------------------------------------------------------- + void InnerContext::destroy_phase_barrier(ApBarrier bar) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Destroying phase barrier in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + destroy_user_barrier(bar); + } + + //-------------------------------------------------------------------------- + PhaseBarrier InnerContext::advance_phase_barrier(PhaseBarrier bar) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Advancing phase barrier in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + PhaseBarrier result = bar; + Runtime::advance_barrier(result); +#ifdef LEGION_SPY + LegionSpy::log_event_dependence(bar.phase_barrier, result.phase_barrier); +#endif + return result; + } + + //-------------------------------------------------------------------------- + void InnerContext::arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, unsigned count) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Arrive dynamic collective in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + Runtime::phase_barrier_arrive(dc, count, ApEvent::NO_AP_EVENT, + buffer, size); + } + + //-------------------------------------------------------------------------- + void InnerContext::defer_dynamic_collective_arrival(DynamicCollective dc, + const Future &f, + unsigned count) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Defer dynamic collective arrival in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + // Record this future as a contribution to the collective + // for future dependence analysis + record_dynamic_collective_contribution(dc, f); + f.impl->contribute_to_collective(dc, count); + } + + //-------------------------------------------------------------------------- + Future InnerContext::get_dynamic_collective_result(DynamicCollective dc) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Get dynamic collective result in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + DynamicCollectiveOp *collective = + runtime->get_available_dynamic_collective_op(); + Future result = collective->initialize(this, dc); + add_to_dependence_queue(collective); + return result; + } + + //-------------------------------------------------------------------------- + DynamicCollective InnerContext::advance_dynamic_collective( + DynamicCollective dc) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Advancing dynamic collective in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + DynamicCollective result = dc; + Runtime::advance_barrier(result); +#ifdef LEGION_SPY + LegionSpy::log_event_dependence(dc.phase_barrier, result.phase_barrier); +#endif + return result; + } + //-------------------------------------------------------------------------- size_t InnerContext::register_new_child_operation(Operation *op, const std::vector *dependences) @@ -6529,14 +6796,24 @@ namespace Legion { } //-------------------------------------------------------------------------- - void InnerContext::insert_unordered_ops(AutoLock &d_lock) + void InnerContext::register_new_internal_operation(InternalOp *op) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(!unordered_ops.empty()); - assert(current_trace == NULL); -#endif - for (std::vector::const_iterator it = + // Nothing to do + } + + //-------------------------------------------------------------------------- + void InnerContext::insert_unordered_ops(AutoLock &d_lock, + const bool end_task, const bool progress) + //-------------------------------------------------------------------------- + { + // If there are no unordered ops then we're done + if (unordered_ops.empty()) + return; + // If we're still in the middle of a trace then don't do any insertions + if (current_trace != NULL) + return; + for (std::list::const_iterator it = unordered_ops.begin(); it != unordered_ops.end(); it++) { (*it)->set_tracking_parent(total_children_count++); @@ -6696,7 +6973,8 @@ namespace Legion { } //-------------------------------------------------------------------------- - void InnerContext::add_to_dependence_queue(Operation *op, bool unordered) + ApEvent InnerContext::add_to_dependence_queue(Operation *op, bool unordered, + bool outermost) //-------------------------------------------------------------------------- { // Launch the task to perform the prepipeline stage for the operation @@ -6730,23 +7008,18 @@ namespace Legion { // If this is unordered, stick it on the list of // unordered ops to be added later and then we're done unordered_ops.push_back(op); - return; + return term_event; } + // Insert any unordered operations into the stream + insert_unordered_ops(d_lock, false/*end task*/, false/*progress*/); + dependence_queue.push_back(op); if (!outstanding_dependence) { -#ifdef DEBUG_LEGION - assert(dependence_queue.empty()); -#endif issue_task = true; outstanding_dependence = true; precondition = dependence_precondition; dependence_precondition = RtEvent::NO_RT_EVENT; } - dependence_queue.push_back(op); - // If we have any unordered ops and we're not in the middle of - // a trace then add them into the queue - if (!unordered_ops.empty() && (current_trace == NULL)) - insert_unordered_ops(d_lock); } if (issue_task) { @@ -6755,13 +7028,14 @@ namespace Legion { } // We disable program order execution when we are replaying a // fixed trace since it might not be sound to block - if (runtime->program_order_execution && !unordered && + if (runtime->program_order_execution && !unordered && outermost && ((current_trace == NULL) || !current_trace->is_fixed())) { begin_task_wait(true/*from runtime*/); term_event.wait(); end_task_wait(); } + return term_event; } //-------------------------------------------------------------------------- @@ -7920,6 +8194,18 @@ namespace Legion { } } + //-------------------------------------------------------------------------- +#ifdef DEBUG_LEGION_COLLECTIVES + MergeCloseOp* InnerContext::get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node) +#else + MergeCloseOp* InnerContext::get_merge_close_op(void) +#endif + //-------------------------------------------------------------------------- + { + return runtime->get_available_merge_close_op(); + } + //-------------------------------------------------------------------------- void InnerContext::record_dynamic_collective_contribution( DynamicCollective dc, const Future &f) @@ -7947,19 +8233,12 @@ namespace Legion { } //-------------------------------------------------------------------------- - Future InnerContext::get_dynamic_collective_result(DynamicCollective dc) + ShardingFunction* InnerContext::find_sharding_function(ShardingID sid) //-------------------------------------------------------------------------- { - AutoRuntimeCall call(this); -#ifdef DEBUG_LEGION - log_run.debug("Get dynamic collective result in task %s (ID %lld)", - get_task_name(), get_unique_id()); -#endif - DynamicCollectiveOp *collective = - runtime->get_available_dynamic_collective_op(); - Future result = collective->initialize(this, dc); - add_to_dependence_queue(collective); - return result; + // Should only be called by inherited classes + assert(false); + return NULL; } //-------------------------------------------------------------------------- @@ -8096,46 +8375,9 @@ namespace Legion { DETAILED_PROFILER(runtime, INVALIDATE_REGION_TREE_CONTEXTS_CALL); // Send messages to invalidate any remote contexts if (!remote_instances.empty()) - { - UniqueID local_uid = get_unique_id(); - Serializer rez; - { - RezCheck z(rez); - rez.serialize(local_uid); - // If we have created requirements figure out what invalidations - // we have to send to the remote context - if (!created_requirements.empty()) - { - std::map to_invalidate; - for (std::map::const_iterator it = - created_requirements.begin(); it != - created_requirements.end(); it++) - { -#ifdef DEBUG_LEGION - assert(returnable_privileges.find(it->first) != - returnable_privileges.end()); -#endif - if (!returnable_privileges[it->first]) - to_invalidate[it->first] = it->second.region; - } - rez.serialize(to_invalidate.size()); - for (std::map::const_iterator it = - to_invalidate.begin(); it != to_invalidate.end(); it++) - { - // Add the size of the original regions to the index - rez.serialize(it->first); - rez.serialize(it->second); - } - } - else - rez.serialize(0); - } - for (std::map::const_iterator it = - remote_instances.begin(); it != remote_instances.end(); it++) - runtime->send_remote_context_release(it->first, rez); - } - // Invalidate all our region contexts - for (unsigned idx = 0; idx < regions.size(); idx++) + invalidate_remote_contexts(); + // Invalidate all our region contexts + for (unsigned idx = 0; idx < regions.size(); idx++) { if (IS_NO_ACCESS(regions[idx])) continue; @@ -8147,56 +8389,55 @@ namespace Legion { regions[idx].region); } if (!created_requirements.empty()) + invalidate_created_requirement_contexts(); + // Clean up our instance top views + if (!instance_top_views.empty()) + clear_instance_top_views(); + // Now we can free our region tree context + runtime->free_region_tree_context(tree_context); + } + + //-------------------------------------------------------------------------- + void InnerContext::invalidate_created_requirement_contexts(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!created_requirements.empty()); +#endif + TaskContext *outermost = find_outermost_local_context(); + const bool is_outermost = (outermost == this); + RegionTreeContext outermost_ctx = outermost->get_context(); + for (std::map::const_iterator it = + created_requirements.begin(); it != + created_requirements.end(); it++) { - TaskContext *outermost = find_outermost_local_context(); - const bool is_outermost = (outermost == this); - RegionTreeContext outermost_ctx = outermost->get_context(); - for (std::map::const_iterator it = - created_requirements.begin(); it != - created_requirements.end(); it++) - { #ifdef DEBUG_LEGION - assert(returnable_privileges.find(it->first) != - returnable_privileges.end()); + assert(returnable_privileges.find(it->first) != + returnable_privileges.end()); #endif - // See if we're a returnable privilege or not - if (returnable_privileges[it->first]) - { - // If we're the outermost context or the requirement was - // deleted, then we can invalidate everything - // Otherwiswe we only invalidate the users - const bool users_only = !is_outermost; - runtime->forest->invalidate_current_context(outermost_ctx, - users_only, it->second.region); - } - else // Not returning so invalidate the full thing - { - runtime->forest->invalidate_current_context(tree_context, - false/*users only*/, it->second.region); - // Little tricky here, this is safe to invaliate the whole - // tree even if we only had privileges on a field because - // if we had privileges on the whole region in this context - // it would have merged the created_requirement and we wouldn't - // have a non returnable privilege requirement in this context - runtime->forest->invalidate_versions(tree_context, - it->second.region); - } + // See if we're a returnable privilege or not + if (returnable_privileges[it->first]) + { + // If we're the outermost context or the requirement was + // deleted, then we can invalidate everything + // Otherwiswe we only invalidate the users + const bool users_only = !is_outermost; + runtime->forest->invalidate_current_context(outermost_ctx, + users_only, it->second.region); } - } - // Clean up our instance top views - if (!instance_top_views.empty()) - { - for (std::map::const_iterator it = - instance_top_views.begin(); it != instance_top_views.end(); it++) + else // Not returning so invalidate the full thing { - it->first->unregister_active_context(this); - if (it->second->remove_base_resource_ref(CONTEXT_REF)) - delete (it->second); + runtime->forest->invalidate_current_context(tree_context, + false/*users only*/, it->second.region); + // Little tricky here, this is safe to invaliate the whole + // tree even if we only had privileges on a field because + // if we had privileges on the whole region in this context + // it would have merged the created_requirement and we wouldn't + // have a non returnable privilege requirement in this context + runtime->forest->invalidate_versions(tree_context, + it->second.region); } - instance_top_views.clear(); - } - // Now we can free our region tree context - runtime->free_region_tree_context(tree_context); + } } //-------------------------------------------------------------------------- @@ -8230,23 +8471,12 @@ namespace Legion { } if (!still_needed.empty()) { - std::set ready_events; const AddressSpaceID local_space = runtime->address_space; for (std::vector::const_iterator it = still_needed.begin(); it != still_needed.end(); it++) { PhysicalManager *manager = targets[*it].get_instance_manager(); - RtEvent ready; - target_views[*it] = - create_instance_top_view(manager, local_space, &ready); - if (ready.exists()) - ready_events.insert(ready); - } - if (!ready_events.empty()) - { - RtEvent wait_on = Runtime::merge_events(ready_events); - if (wait_on.exists()) - wait_on.wait(); + target_views[*it] = create_instance_top_view(manager, local_space); } } } @@ -8265,8 +8495,7 @@ namespace Legion { //-------------------------------------------------------------------------- InstanceView* InnerContext::create_instance_top_view( - PhysicalManager *manager, AddressSpaceID request_source, - RtEvent *ready_event/*=NULL*/) + PhysicalManager *manager, AddressSpaceID request_source) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, CREATE_INSTANCE_TOP_VIEW_CALL); @@ -8447,16 +8676,9 @@ namespace Legion { RtEvent ctx_ready; InnerContext *context = runtime->find_context(context_uid, false, &ctx_ready); - // Find the manager too, we know we are local so it should already - // be registered in the set of distributed IDs - DistributedCollectable *dc = - runtime->find_distributed_collectable(manager_did); -#ifdef DEBUG_LEGION - PhysicalManager *manager = dynamic_cast(dc); - assert(manager != NULL); -#else - PhysicalManager *manager = static_cast(dc); -#endif + RtEvent ready; + PhysicalManager *manager = + runtime->find_or_request_instance_manager(manager_did, ready); // Nasty deadlock case: if the request came from a different node // we have to defer this because we are in the view virtual channel // and we might invoke the update virtual channel, but we already @@ -8464,8 +8686,18 @@ namespace Legion { // the view virtual channel (paging views), so to avoid the cycle // we have to launch a meta-task and record when it is done RemoteCreateViewArgs args(context, manager, target, to_trigger, source); - runtime->issue_runtime_meta_task(args, - LG_LATENCY_DEFERRED_PRIORITY, ctx_ready); + if (ready.exists()) + { + if (ctx_ready.exists()) + runtime->issue_runtime_meta_task(args, LG_LATENCY_DEFERRED_PRIORITY, + Runtime::merge_events(ready, ctx_ready)); + else + runtime->issue_runtime_meta_task(args, LG_LATENCY_DEFERRED_PRIORITY, + ready); + } + else + runtime->issue_runtime_meta_task(args, LG_LATENCY_DEFERRED_PRIORITY, + ctx_ready); } //-------------------------------------------------------------------------- @@ -8587,7 +8819,8 @@ namespace Legion { local_fields_to_delete.begin(); it != local_fields_to_delete.end(); it++) { - FieldAllocatorImpl *allocator = create_field_allocator(it->first); + FieldAllocatorImpl *allocator = + create_field_allocator(it->first, false/*unordered*/); free_fields(allocator, it->first, it->second, false/*unordered*/); } } @@ -8698,8 +8931,15 @@ namespace Legion { // Check to see if we have any unordered operations that we need to inject { AutoLock d_lock(dependence_lock); - if (!unordered_ops.empty()) - insert_unordered_ops(d_lock); + insert_unordered_ops(d_lock, true/*end task*/, false/*progress*/); + if (!dependence_queue.empty() && !outstanding_dependence) + { + outstanding_dependence = true; + DependenceArgs args(dependence_queue.front(), this); + runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_WORK_PRIORITY, dependence_precondition); + dependence_precondition = RtEvent::NO_RT_EVENT; + } } // Mark that we are done executing this operation // We're not actually done until we have registered our pending @@ -8806,9 +9046,10 @@ namespace Legion { } } if (!preconditions.empty()) - single_task->handle_post_mapped(Runtime::merge_events(preconditions)); + single_task->handle_post_mapped(false/*deferral*/, + Runtime::merge_events(preconditions)); else - single_task->handle_post_mapped(); + single_task->handle_post_mapped(false/*deferral*/); } if (need_complete) owner_task->trigger_children_complete(); @@ -8816,6 +9057,63 @@ namespace Legion { owner_task->trigger_children_committed(); } + //-------------------------------------------------------------------------- + void InnerContext::invalidate_remote_contexts(void) + //-------------------------------------------------------------------------- + { + UniqueID local_uid = get_unique_id(); + Serializer rez; + { + RezCheck z(rez); + rez.serialize(local_uid); + // If we have created requirements figure out what invalidations + // we have to send to the remote context + if (!created_requirements.empty()) + { + std::map to_invalidate; + for (std::map::const_iterator it = + created_requirements.begin(); it != + created_requirements.end(); it++) + { +#ifdef DEBUG_LEGION + assert(returnable_privileges.find(it->first) != + returnable_privileges.end()); +#endif + if (!returnable_privileges[it->first]) + to_invalidate[it->first] = it->second.region; + } + rez.serialize(to_invalidate.size()); + for (std::map::const_iterator it = + to_invalidate.begin(); it != to_invalidate.end(); it++) + { + // Add the size of the original regions to the index + rez.serialize(it->first); + rez.serialize(it->second); + } + } + else + rez.serialize(0); + } + for (std::map::const_iterator it = + remote_instances.begin(); it != remote_instances.end(); it++) + runtime->send_remote_context_release(it->first, rez); + } + + //-------------------------------------------------------------------------- + void InnerContext::clear_instance_top_views(void) + //-------------------------------------------------------------------------- + { + for (std::map::const_iterator it = + instance_top_views.begin(); it != instance_top_views.end(); it++) + { + if (it->first->is_owner()) + it->first->unregister_active_context(this); + if (it->second->remove_base_resource_ref(CONTEXT_REF)) + delete (it->second); + } + instance_top_views.clear(); + } + //-------------------------------------------------------------------------- void InnerContext::free_remote_contexts(void) //-------------------------------------------------------------------------- @@ -8961,7 +9259,7 @@ namespace Legion { } if (!wait_events.empty()) { - ApEvent wait_on = Runtime::merge_events(NULL, wait_events); + RtEvent wait_on = Runtime::protect_merge_events(wait_events); wait_on.wait(); } } @@ -9083,6 +9381,8 @@ namespace Legion { "inside of a trace by task %s (UID %lld). Calls to " "'perform_registration_callback' are only permitted outside " "of traces.", get_task_name(), get_unique_id()) + if (effects.has_triggered()) + return; // Dump a mapping fence into the stream that will not be considered // mapped until these effects are done so that we can ensure that // no downstream operations attempt to do anything on remote nodes @@ -9309,7 +9609,7 @@ namespace Legion { //-------------------------------------------------------------------------- void TopLevelContext::pack_remote_context(Serializer &rez, - AddressSpaceID target) + AddressSpaceID target, bool replicate) //-------------------------------------------------------------------------- { rez.serialize(depth); @@ -9340,37 +9640,7489 @@ namespace Legion { RtUserEvent ready_event = Runtime::create_rt_user_event(); Serializer rez; { - RezCheck z(rez); - rez.serialize(context_uid); - rez.serialize(manager); - rez.serialize(tree_id); - expr->pack_expression(rez, owner_space); - rez.serialize(mask); - rez.serialize(handle); - rez.serialize(source); - rez.serialize(ready_event); + RezCheck z(rez); + rez.serialize(context_uid); + rez.serialize(manager); + rez.serialize(tree_id); + expr->pack_expression(rez, owner_space); + rez.serialize(mask); + rez.serialize(handle); + rez.serialize(source); + rez.serialize(ready_event); + } + // Send it to the owner space + runtime->send_compute_equivalence_sets_request(owner_space, rez); + return ready_event; + } + + //-------------------------------------------------------------------------- + InnerContext* TopLevelContext::find_outermost_local_context( + InnerContext *previous) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(previous != NULL); +#endif + return previous; + } + + //-------------------------------------------------------------------------- + InnerContext* TopLevelContext::find_top_context(void) + //-------------------------------------------------------------------------- + { + return this; + } + + ///////////////////////////////////////////////////////////// + // Replicate Context + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplicateContext::ReplicateContext(Runtime *rt, + ShardTask *owner, int d, bool full, + const std::vector &reqs, + const std::vector &parent_indexes, + const std::vector &virt_mapped, + UniqueID ctx_uid, ApEvent exec_fence, + ShardManager *manager) + : InnerContext(rt, owner, d, full, reqs, parent_indexes, virt_mapped, + ctx_uid, exec_fence), owner_shard(owner), shard_manager(manager), + total_shards(shard_manager->total_shards), + next_close_mapped_bar_index(0), next_indirection_bar_index(0), + next_future_map_bar_index(0), index_space_allocator_shard(0), + index_partition_allocator_shard(0), field_space_allocator_shard(0), + field_allocator_shard(0), logical_region_allocator_shard(0), + dynamic_id_allocator_shard(0), next_available_collective_index(0), + trace_recording_collective_id(0), summary_collective_id(0), + next_physical_template_index(0), next_replicate_bar_index(0), + next_trace_bar_index(0), next_summary_bar_index(0), + unordered_ops_counter(0), unordered_ops_epoch(MIN_UNORDERED_OPS_EPOCH) + //-------------------------------------------------------------------------- + { + // Get our allocation barriers + pending_partition_barrier = manager->get_pending_partition_barrier(); + creation_barrier = manager->get_creation_barrier(); + deletion_ready_barrier = manager->get_deletion_ready_barrier(); + deletion_mapping_barrier = manager->get_deletion_mapping_barrier(); + deletion_execution_barrier = manager->get_deletion_execution_barrier(); + inline_mapping_barrier = manager->get_inline_mapping_barrier(); + external_resource_barrier = manager->get_external_resource_barrier(); + mapping_fence_barrier = manager->get_mapping_fence_barrier(); + trace_recording_barrier = manager->get_trace_recording_barrier(); + summary_fence_barrier = manager->get_summary_fence_barrier(); + execution_fence_barrier = manager->get_execution_fence_barrier(); + attach_broadcast_barrier = manager->get_attach_broadcast_barrier(); + attach_reduce_barrier = manager->get_attach_reduce_barrier(); + dependent_partition_barrier = manager->get_dependent_partition_barrier(); + semantic_attach_barrier = manager->get_semantic_attach_barrier(); + inorder_barrier = manager->get_inorder_barrier(); +#ifdef DEBUG_LEGION_COLLECTIVES + collective_check_barrier = manager->get_collective_check_barrier(); + close_check_barrier = manager->get_close_check_barrier(); +#endif + // Configure our collective settings + shard_collective_radix = runtime->legion_collective_radix; + configure_collective_settings(total_shards, owner->shard_id, + shard_collective_radix, shard_collective_log_radix, + shard_collective_stages, shard_collective_participating_shards, + shard_collective_last_radix); + } + + //-------------------------------------------------------------------------- + ReplicateContext::ReplicateContext(const ReplicateContext &rhs) + : InnerContext(*this), owner_shard(NULL), + shard_manager(NULL), total_shards(0) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplicateContext::~ReplicateContext(void) + //-------------------------------------------------------------------------- + { + // We delete the barriers that we created + for (unsigned idx = owner_shard->shard_id; + idx < close_mapped_barriers.size(); idx += total_shards) + { + Realm::Barrier bar = close_mapped_barriers[idx]; + bar.destroy_barrier(); + } + for (unsigned idx = owner_shard->shard_id; + idx < indirection_barriers.size(); idx += total_shards) + { + Realm::Barrier bar = indirection_barriers[idx]; + bar.destroy_barrier(); + } + for (unsigned idx = owner_shard->shard_id; + idx < future_map_barriers.size(); idx += total_shards) + { + Realm::Barrier bar = future_map_barriers[idx]; + bar.destroy_barrier(); + } + while (!pending_index_spaces.empty()) + { + std::pair*,bool> &collective = + pending_index_spaces.front(); + if (collective.second) + { + const ISBroadcast value = collective.first->get_value(false); + runtime->forest->revoke_pending_index_space(value.space_id); + runtime->revoke_pending_distributed_collectable(value.did); + runtime->free_distributed_id(value.did); + } + else + { + // Make sure this collective is done before we delete it + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + done.wait(); + } + delete collective.first; + pending_index_spaces.pop_front(); + } + while (!pending_index_partitions.empty()) + { + std::pair*,ShardID> &collective = + pending_index_partitions.front(); + if (collective.second) + { + const IPBroadcast value = collective.first->get_value(false); + runtime->forest->revoke_pending_partition(value.pid); + runtime->revoke_pending_distributed_collectable(value.did); + runtime->free_distributed_id(value.did); + } + else + { + // Make sure this collective is done before we delete it + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + done.wait(); + } + delete collective.first; + pending_index_partitions.pop_front(); + } + while (!pending_field_spaces.empty()) + { + std::pair*,bool> &collective = + pending_field_spaces.front(); + if (collective.second) + { + const FSBroadcast value = collective.first->get_value(false); + runtime->forest->revoke_pending_field_space(value.space_id); + runtime->revoke_pending_distributed_collectable(value.did); + runtime->free_distributed_id(value.did); + } + else + { + // Make sure this collective is done before we delete it + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + done.wait(); + } + delete collective.first; + pending_field_spaces.pop_front(); + } + while (!pending_fields.empty()) + { + std::pair*,bool> &collective = + pending_fields.front(); + if (!collective.second) + { + // Make sure this collective is done before we delete it + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + done.wait(); + } + delete collective.first; + pending_fields.pop_front(); + } + while (!pending_region_trees.empty()) + { + std::pair*,bool> &collective = + pending_region_trees.front(); + if (collective.second) + { + const LRBroadcast value = collective.first->get_value(false); + runtime->forest->revoke_pending_region_tree(value.tid); + } + else + { + // Make sure this collective is done before we delete it + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + done.wait(); + } + delete collective.first; + pending_region_trees.pop_front(); + } + if (returned_resource_ready_barrier.exists()) + returned_resource_ready_barrier.destroy_barrier(); + if (returned_resource_mapped_barrier.exists()) + returned_resource_mapped_barrier.destroy_barrier(); + if (returned_resource_execution_barrier.exists()) + returned_resource_execution_barrier.destroy_barrier(); + } + + //-------------------------------------------------------------------------- + ReplicateContext& ReplicateContext::operator=(const ReplicateContext &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::perform_global_registration_callbacks( + Realm::DSOReferenceImplementation *dso, RtEvent local_done, + RtEvent global_done, std::set &preconditions) + //-------------------------------------------------------------------------- + { + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_PERFORM_REGISTRATION_CALLBACK); + hasher.hash(dso->dso_name.c_str(), dso->dso_name.size()); + hasher.hash(dso->symbol_name.c_str(), dso->symbol_name.size()); + verify_replicable(hasher, "perform_registration_callback"); + } + shard_manager->perform_global_registration_callbacks(dso, local_done, + global_done, preconditions); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::handle_registration_callback_effects(RtEvent effects) + //-------------------------------------------------------------------------- + { + if (current_trace != NULL) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_PERFORM_REGISTRATION_CALLBACK, + "Illegal call to 'perform_registration_callback' performed " + "inside of a trace by task %s (UID %lld). Calls to " + "'perform_registration_callback' are only permitted outside " + "of traces.", get_task_name(), get_unique_id()) + // Dump a mapping fence into the stream that will not be considered + // mapped until these effects are done so that we can ensure that + // no downstream operations attempt to do anything on remote nodes + // which could need the results of the registration + ReplFenceOp *fence_op = runtime->get_available_repl_fence_op(); + fence_op->initialize_repl_fence(this, FenceOp::MAPPING_FENCE, false); + fence_op->add_mapping_applied_condition(effects); + add_to_dependence_queue(fence_op); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::print_once(FILE *f, const char *message) const + //-------------------------------------------------------------------------- + { + // Only print from shard 0 + if (owner_shard->shard_id == 0) + fprintf(f, "%s", message); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::log_once(Realm::LoggerMessage &message) const + //-------------------------------------------------------------------------- + { + // Deactivate all the messages except shard 0 + if (owner_shard->shard_id != 0) + message.deactivate(); + } + + //-------------------------------------------------------------------------- + ShardID ReplicateContext::get_shard_id(void) const + //-------------------------------------------------------------------------- + { + return owner_shard->shard_id; + } + + //-------------------------------------------------------------------------- + size_t ReplicateContext::get_num_shards(void) const + //-------------------------------------------------------------------------- + { + return total_shards; + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::consensus_match(const void *input, void *output, + size_t num_elements, size_t element_size) + //-------------------------------------------------------------------------- + { + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CONSENSUS_MATCH); + verify_replicable(hasher, "consensus_match"); + } + ApUserEvent complete = Runtime::create_ap_user_event(NULL); + Future result = runtime->help_create_future(complete); + switch (element_size) + { + case 1: + { + ConsensusMatchExchange *collective = + new ConsensusMatchExchange(this, COLLECTIVE_LOC_89, + result, output, complete); + if (collective->match_elements_async(input, num_elements)) + delete collective; + break; + } + case 2: + { + ConsensusMatchExchange *collective = + new ConsensusMatchExchange(this, COLLECTIVE_LOC_89, + result, output, complete); + if (collective->match_elements_async(input, num_elements)) + delete collective; + break; + } + case 4: + { + ConsensusMatchExchange *collective = + new ConsensusMatchExchange(this, COLLECTIVE_LOC_89, + result, output, complete); + if (collective->match_elements_async(input, num_elements)) + delete collective; + break; + } + case 8: + { + ConsensusMatchExchange *collective = + new ConsensusMatchExchange(this, COLLECTIVE_LOC_89, + result, output, complete); + if (collective->match_elements_async(input, num_elements)) + delete collective; + break; + } + default: + REPORT_LEGION_FATAL(LEGION_FATAL_UNSUPPORTED_CONSENSUS_SIZE, + "Unsupported size %zd for consensus match in %s (UID %lld)", + element_size, get_task_name(), get_unique_id()) + } + return result; + } + + //-------------------------------------------------------------------------- + VariantID ReplicateContext::register_variant( + const TaskVariantRegistrar ®istrar, const void *user_data, + size_t user_data_size, const CodeDescriptor &desc, bool ret, + VariantID vid, bool check_task_id) + //-------------------------------------------------------------------------- + { + // If we're inside a registration callback we don't care + if (inside_registration_callback) + return TaskContext::register_variant(registrar, user_data, + user_data_size, desc, ret, vid, check_task_id); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_REGISTER_TASK_VARIANT); + hasher.hash(registrar.task_id); + hasher.hash(registrar.global_registration); + if (registrar.task_variant_name != NULL) + hasher.hash(registrar.task_variant_name, + strlen(registrar.task_variant_name)); + Serializer rez; + registrar.execution_constraints.serialize(rez); + registrar.layout_constraints.serialize(rez); + hasher.hash(rez.get_buffer(), rez.get_used_bytes()); + for (std::set::const_iterator it = + registrar.generator_tasks.begin(); it != + registrar.generator_tasks.end(); it++) + hasher.hash(*it); + hasher.hash(registrar.leaf_variant); + hasher.hash(registrar.inner_variant); + hasher.hash(registrar.idempotent_variant); + hasher.hash(registrar.replicable_variant); + if (user_data != NULL) + hasher.hash(user_data, user_data_size); + hasher.hash(vid); + verify_replicable(hasher, "register_task_variant"); + } + VariantID result; + if (owner_shard->shard_id == dynamic_id_allocator_shard) + { + ValueBroadcast collective(this, COLLECTIVE_LOC_17); + // Have this shard do the registration, and then broadcast the + // resulting variant to all the other shards + result = runtime->register_variant(registrar, user_data, user_data_size, + desc, ret, vid, check_task_id, false/*check context*/); + collective.broadcast(result); + } + else + { + ValueBroadcast collective(this, dynamic_id_allocator_shard, + COLLECTIVE_LOC_17); + result = collective.get_value(); + } + if (++dynamic_id_allocator_shard == total_shards) + dynamic_id_allocator_shard = 0; + return result; + } + + //-------------------------------------------------------------------------- + TraceID ReplicateContext::generate_dynamic_trace_id(void) + //-------------------------------------------------------------------------- + { + // If we're inside a registration callback we don't care + if (inside_registration_callback) + return TaskContext::generate_dynamic_trace_id(); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_GENERATE_DYNAMIC_TRACE_ID); + verify_replicable(hasher, "generate_dynamic_trace_id"); + } + // Otherwise have one shard make it and broadcast it to everyone else + TraceID result; + if (owner_shard->shard_id == dynamic_id_allocator_shard) + { + ValueBroadcast collective(this, COLLECTIVE_LOC_9); + result = runtime->generate_dynamic_trace_id(false/*check context*/); + collective.broadcast(result); + } + else + { + ValueBroadcast collective(this, dynamic_id_allocator_shard, + COLLECTIVE_LOC_9); + result = collective.get_value(); + } + if (++dynamic_id_allocator_shard == total_shards) + dynamic_id_allocator_shard = 0; + return result; + } + + //-------------------------------------------------------------------------- + MapperID ReplicateContext::generate_dynamic_mapper_id(void) + //-------------------------------------------------------------------------- + { + // If we're inside a registration callback we don't care + if (inside_registration_callback) + return TaskContext::generate_dynamic_mapper_id(); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_GENERATE_DYNAMIC_MAPPER_ID); + verify_replicable(hasher, "generate_dynamic_mapper_id"); + } + // Otherwise have one shard make it and broadcast it to everyone else + MapperID result; + if (owner_shard->shard_id == dynamic_id_allocator_shard) + { + ValueBroadcast collective(this, COLLECTIVE_LOC_10); + result = runtime->generate_dynamic_mapper_id(false/*check context*/); + collective.broadcast(result); + } + else + { + ValueBroadcast collective(this, dynamic_id_allocator_shard, + COLLECTIVE_LOC_10); + result = collective.get_value(); + } + if (++dynamic_id_allocator_shard == total_shards) + dynamic_id_allocator_shard = 0; + return result; + } + + //-------------------------------------------------------------------------- + ProjectionID ReplicateContext::generate_dynamic_projection_id(void) + //-------------------------------------------------------------------------- + { + // If we're inside a registration callback we don't care + if (inside_registration_callback) + return TaskContext::generate_dynamic_projection_id(); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_GENERATE_DYNAMIC_PROJECTION_ID); + verify_replicable(hasher, "generate_dynamic_projection_id"); + } + // Otherwise have one shard make it and broadcast it to everyone else + ProjectionID result; + if (owner_shard->shard_id == dynamic_id_allocator_shard) + { + ValueBroadcast collective(this, COLLECTIVE_LOC_11); + result = + runtime->generate_dynamic_projection_id(false/*check context*/); + collective.broadcast(result); + } + else + { + ValueBroadcast collective(this,dynamic_id_allocator_shard, + COLLECTIVE_LOC_11); + result = collective.get_value(); + } + if (++dynamic_id_allocator_shard == total_shards) + dynamic_id_allocator_shard = 0; + return result; + } + + //-------------------------------------------------------------------------- + ShardingID ReplicateContext::generate_dynamic_sharding_id(void) + //-------------------------------------------------------------------------- + { + // If we're inside a registration callback we don't care + if (inside_registration_callback) + return TaskContext::generate_dynamic_sharding_id(); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_GENERATE_DYNAMIC_SHARDING_ID); + verify_replicable(hasher, "generate_dynamic_sharding_id"); + } + // Otherwise have one shard make it and broadcast it to everyone else + ShardingID result; + if (owner_shard->shard_id == dynamic_id_allocator_shard) + { + ValueBroadcast collective(this, COLLECTIVE_LOC_12); + result = runtime->generate_dynamic_sharding_id(false/*check context*/); + collective.broadcast(result); + } + else + { + ValueBroadcast collective(this,dynamic_id_allocator_shard, + COLLECTIVE_LOC_12); + result = collective.get_value(); + } + if (++dynamic_id_allocator_shard == total_shards) + dynamic_id_allocator_shard = 0; + return result; + } + + //-------------------------------------------------------------------------- + TaskID ReplicateContext::generate_dynamic_task_id(void) + //-------------------------------------------------------------------------- + { + // If we're inside a registration callback we don't care + if (inside_registration_callback) + return TaskContext::generate_dynamic_task_id(); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_GENERATE_DYNAMIC_TASK_ID); + verify_replicable(hasher, "generate_dynamic_task_id"); + } + // Otherwise have one shard make it and broadcast it to everyone else + TaskID result; + if (owner_shard->shard_id == dynamic_id_allocator_shard) + { + ValueBroadcast collective(this, COLLECTIVE_LOC_13); + result = runtime->generate_dynamic_task_id(false/*check context*/); + collective.broadcast(result); + } + else + { + ValueBroadcast collective(this, dynamic_id_allocator_shard, + COLLECTIVE_LOC_13); + result = collective.get_value(); + } + if (++dynamic_id_allocator_shard == total_shards) + dynamic_id_allocator_shard = 0; + return result; + } + + //-------------------------------------------------------------------------- + ReductionOpID ReplicateContext::generate_dynamic_reduction_id(void) + //-------------------------------------------------------------------------- + { + // If we're inside a registration callback we don't care + if (inside_registration_callback) + return TaskContext::generate_dynamic_reduction_id(); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_GENERATE_DYNAMIC_REDUCTION_ID); + verify_replicable(hasher, "generate_dynamic_reduction_id"); + } + // Otherwise have one shard make it and broadcast it to everyone else + ReductionOpID result; + if (owner_shard->shard_id == dynamic_id_allocator_shard) + { + ValueBroadcast collective(this, COLLECTIVE_LOC_14); + result = runtime->generate_dynamic_reduction_id(false/*check context*/); + collective.broadcast(result); + } + else + { + ValueBroadcast collective(this, + dynamic_id_allocator_shard, COLLECTIVE_LOC_14); + result = collective.get_value(); + } + if (++dynamic_id_allocator_shard == total_shards) + dynamic_id_allocator_shard = 0; + return result; + } + + //-------------------------------------------------------------------------- + CustomSerdezID ReplicateContext::generate_dynamic_serdez_id(void) + //-------------------------------------------------------------------------- + { + // If we're inside a registration callback we don't care + if (inside_registration_callback) + return TaskContext::generate_dynamic_serdez_id(); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_GENERATE_DYNAMIC_SERDEZ_ID); + verify_replicable(hasher, "generate_dynamic_serdez_id"); + } + // Otherwise have one shard make it and broadcast it to everyone else + CustomSerdezID result; + if (owner_shard->shard_id == dynamic_id_allocator_shard) + { + ValueBroadcast collective(this, COLLECTIVE_LOC_16); + result = runtime->generate_dynamic_serdez_id(false/*check context*/); + collective.broadcast(result); + } + else + { + ValueBroadcast collective(this, + dynamic_id_allocator_shard, COLLECTIVE_LOC_16); + result = collective.get_value(); + } + if (++dynamic_id_allocator_shard == total_shards) + dynamic_id_allocator_shard = 0; + return result; + } + + //-------------------------------------------------------------------------- + bool ReplicateContext::perform_semantic_attach(bool &global) + //-------------------------------------------------------------------------- + { + if (inside_registration_callback) + return TaskContext::perform_semantic_attach(global); + // Before we do anything else here, we need to make sure that all + // the shards are done reading before we attempt to mutate the value + Runtime::phase_barrier_arrive(semantic_attach_barrier, 1/*count*/); + const RtEvent wait_on = semantic_attach_barrier; + advance_replicate_barrier(semantic_attach_barrier, total_shards); + // Check to see if we can downgrade this to a local_only update + if (global && shard_manager->is_total_sharding()) + global = false; + // Wait until all the reads of the semantic info are done + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + if (global) + { + // If we're still global then just have shard 0 do this for now + if (owner_shard->shard_id == 0) + return true; + post_semantic_attach(); + return false; + } + else + { + // See if we're the local shard to perform the attach operation + if (shard_manager->perform_semantic_attach()) + return true; + post_semantic_attach(); + return false; + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::post_semantic_attach(void) + //-------------------------------------------------------------------------- + { + if (inside_registration_callback) + return; + Runtime::phase_barrier_arrive(semantic_attach_barrier, 1/*count*/); + const RtEvent wait_on = semantic_attach_barrier; + advance_replicate_barrier(semantic_attach_barrier, total_shards); + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::verify_replicable(Murmur3Hasher &hasher, + const char *func_name) + //-------------------------------------------------------------------------- + { + uint64_t hash[2]; + hasher.finalize(hash); + VerifyReplicableExchange exchange(COLLECTIVE_LOC_82, this); + const VerifyReplicableExchange::ShardHashes &hashes = + exchange.exchange(hash); + // If all shards had the same hashes then we are done + if (hashes.size() == 1) + return; + const std::pair key(hash[0],hash[1]); + const VerifyReplicableExchange::ShardHashes::const_iterator + finder = hashes.find(key); +#ifdef DEBUG_LEGION + assert(finder != hashes.end()); +#endif + // See if we are one of the lowest hashes and report the error + // We'll let the other shards continue to avoid printing out + // too many error messages, they'll be killed soon enough + if (finder->second == owner_shard->shard_id) + REPORT_LEGION_ERROR(ERROR_CONTROL_REPLICATION_VIOLATION, + "Detected control replication violation when invoking %s in " + "task %s (UID %lld) on shard %d. The hash summary for the function " + "does not align with the hash summaries from other call sites.", + func_name, get_task_name(), get_unique_id(), owner_shard->shard_id) + } + + //-------------------------------------------------------------------------- + /*static*/ void ReplicateContext::help_complete_future(Future &f, + const void *result, size_t result_size, bool own) + //-------------------------------------------------------------------------- + { + f.impl->set_result(result, result_size, own); + } + + //-------------------------------------------------------------------------- + RtEvent ReplicateContext::compute_equivalence_sets(VersionManager *manager, + RegionTreeID tree_id, IndexSpace handle, + IndexSpaceExpression *expr, const FieldMask &mask, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + // This one is very similar to the InnerContext version with the + // exception that we round robin tree ids across shards for the + // contol replication context +#ifdef DEBUG_LEGION + assert(handle.exists()); +#endif + EquivalenceSet *root = NULL; + if (expr->is_empty()) + { + // Special case for empty expression + // In this case we don't need to bother having the + // same equivalence set for all shards since it doesn't matter + { + const std::pair + key(tree_id, expr->expr_id); + AutoLock tree_lock(tree_set_lock); + // Check to see if we already have an empty equivalence set + // and if not make it + std::map, + EquivalenceSet*>::const_iterator finder = + empty_equivalence_sets.find(key); + if (finder == empty_equivalence_sets.end()) + { + const AddressSpaceID local_space = runtime->address_space; + IndexSpaceNode *node = runtime->forest->get_node(handle); + root = new EquivalenceSet(runtime, + runtime->get_available_distributed_id(), + local_space, local_space, expr, node, + true/*register now*/); + empty_equivalence_sets[key] = root; + root->add_base_resource_ref(CONTEXT_REF); + } + else + root = finder->second; + } + // Now that we have the empty equivalence set, either record it + // or send it back to the source node for the response + if (source != runtime->address_space) + { + // Not local so we need to send a message + RtUserEvent recorded_event = Runtime::create_rt_user_event(); + Serializer rez; + { + RezCheck z(rez); + rez.serialize(root->did); + rez.serialize(mask); + rez.serialize(manager); + rez.serialize(recorded_event); + } + runtime->send_equivalence_set_ray_trace_response(source, rez); + return recorded_event; + } + else + manager->record_equivalence_set(root, mask); + return RtEvent::NO_RT_EVENT; + } + else + { + AutoLock tree_lock(tree_set_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + tree_equivalence_sets.find(tree_id); + if (finder != tree_equivalence_sets.end()) + root = finder->second; + } + if (root == NULL) + { + // We don't have one yet check to see if we need to send a + // request for it yet or not based on which shard should + // be owning this tree ID + const ShardID tree_shard = tree_id % shard_manager->total_shards; + if (tree_shard != owner_shard->shard_id) + { + RtEvent wait_on; + bool send_message = false; + { + AutoLock tree_lock(tree_set_lock); + // First see if we lost the race + std::map::const_iterator finder = + tree_equivalence_sets.find(tree_id); + if (finder == tree_equivalence_sets.end()) + { + // Don't have it yet, see if we need to send a message + std::map::const_iterator + request_finder = pending_tree_requests.find(tree_id); + if (request_finder == pending_tree_requests.end()) + { + RtUserEvent request_event = Runtime::create_rt_user_event(); + pending_tree_requests[tree_id] = request_event; + wait_on = request_event; + send_message = true; + } + else // Message was already send just wait for it + wait_on = request_finder->second; + } + else // Already have it + root = finder->second; + } + // If we didn't find it already then do our thing + if (root == NULL) + { + if (send_message) + { + Serializer rez; + rez.serialize(shard_manager->repl_id); + rez.serialize(tree_shard); + rez.serialize(tree_id); + rez.serialize(this); + rez.serialize(runtime->address_space); + shard_manager->send_equivalence_set_request(tree_shard, rez); + } + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + AutoLock tree_lock(tree_set_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + tree_equivalence_sets.find(tree_id); +#ifdef DEBUG_LEGION + // It better be here at this point + assert(finder != tree_equivalence_sets.end()); +#endif + root = finder->second; + } +#ifdef DEBUG_LEGION + assert(root != NULL); +#endif + } + else + root = find_or_create_top_equivalence_set(tree_id); + } +#ifdef DEBUG_LEGION + assert(root != NULL); +#endif + RtUserEvent ready = Runtime::create_rt_user_event(); + root->ray_trace_equivalence_sets(manager, expr, mask, + handle, source, ready); + return ready; + } + + //-------------------------------------------------------------------------- + InnerContext* ReplicateContext::find_parent_physical_context(unsigned index, + LogicalRegion handle) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(regions.size() == virtual_mapped.size()); + assert(regions.size() == parent_req_indexes.size()); +#endif + if (index < virtual_mapped.size()) + { + // See if it is virtual mapped + if (virtual_mapped[index]) + return find_parent_context()->find_parent_physical_context( + parent_req_indexes[index], handle); + else // We mapped a physical instance so we're it + return this; + } + else // We created it + { + // Check to see if this has returnable privileges or not + // If they are not returnable, then we can just be the + // context for the handling the meta-data management, + // otherwise we can only support this currently if we are + // the context for the top-level task + // If you change this then also make sure that you + // change invalidate_region_tree_contexts to match + AutoLock priv_lock(privilege_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + returnable_privileges.find(index); + if ((finder == returnable_privileges.end()) || finder->second) + { + if (owner_task->get_depth() > 0) + REPORT_LEGION_FATAL(LEGION_FATAL_CTRL_REPL_RETURN_PRIV, + "Returnable privileges are not currently " + "supported for control replicated tasks " + "that are are not the top-level task such " + "as task %s (UID %lld)", + owner_task->get_task_name(), + owner_task->get_unique_id()) + } + return this; + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::invalidate_region_tree_contexts(void) + //-------------------------------------------------------------------------- + { + // This does mostly the same thing as the InnerContext version + // but handles created requirements differently since we know + // that we kept those things in our context + DETAILED_PROFILER(runtime, INVALIDATE_REGION_TREE_CONTEXTS_CALL); + if (!remote_instances.empty()) + invalidate_remote_contexts(); + // Invalidate all our region contexts + for (unsigned idx = 0; idx < regions.size(); idx++) + { + if (IS_NO_ACCESS(regions[idx])) + continue; + runtime->forest->invalidate_current_context(tree_context, + false/*users only*/, + regions[idx].region); + if (!virtual_mapped[idx]) + runtime->forest->invalidate_versions(tree_context, + regions[idx].region); + } + if (!created_requirements.empty()) + invalidate_created_requirement_contexts(); + // Cannot clear our instance top view references until we are deleted + // as we might still need to help out our other sibling shards + + // Now we can free our region tree context + runtime->free_region_tree_context(tree_context); + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::create_index_space(const Domain &domain, + TypeTag type_tag) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_INDEX_SPACE); + Serializer rez; + rez.serialize(domain); + hasher.hash(rez.get_buffer(), rez.get_used_bytes()); + hasher.hash(type_tag); + verify_replicable(hasher, "create_index_space"); + } + // Seed this with the first index space broadcast + if (pending_index_spaces.empty()) + increase_pending_index_spaces(1/*count*/, false/*double*/); + IndexSpace handle; + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_index_spaces.front(); + if (collective.second) + { + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid, type_tag); + double_buffer = value.double_buffer; + std::set applied; + IndexSpaceNode *node = + runtime->forest->create_index_space(handle, &domain, value.did, + false/*notify remote*/, value.expr_id, ApEvent::NO_AP_EVENT, + creation_barrier, &applied); + // Now we can update the creation set + node->update_creation_set(shard_manager->get_mapping()); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + runtime->forest->revoke_pending_index_space(value.space_id); + runtime->revoke_pending_distributed_collectable(value.did); +#ifdef DEBUG_LEGION + log_index.debug("Creating index space %x in task%s (ID %lld)", + handle.id, get_task_name(), get_unique_id()); +#endif + if (runtime->legion_spy_enabled) + LegionSpy::log_top_index_space(handle.id); + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid, type_tag); + double_buffer = value.double_buffer; +#ifdef DEBUG_LEGION + assert(handle.exists()); +#endif + std::set applied; + runtime->forest->create_index_space(handle, &domain, value.did, + false/*notify remote*/, value.expr_id, ApEvent::NO_AP_EVENT, + creation_barrier, &applied); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + } + delete collective.first; + pending_index_spaces.pop_front(); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + // Record this in our context + register_index_space_creation(handle); + // Get new handles in flight for the next time we need them + // Always add a new one to replace the old one, but double the number + // in flight if we're not hiding the latency + increase_pending_index_spaces(double_buffer ? + pending_index_spaces.size() + 1 : 1, double_next && !double_buffer); + return handle; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::increase_pending_index_spaces(unsigned count, + bool double_next) + //-------------------------------------------------------------------------- + { + for (unsigned idx = 0; idx < count; idx++) + { + if (owner_shard->shard_id == index_space_allocator_shard) + { + const IndexSpaceID space_id = runtime->get_unique_index_space_id(); + const DistributedID did = runtime->get_available_distributed_id(); + // We're the owner, so make it locally and then broadcast it + runtime->forest->record_pending_index_space(space_id); + runtime->record_pending_distributed_collectable(did); + // Do our arrival on this generation, should be the last one + ValueBroadcast *collective = + new ValueBroadcast(this, COLLECTIVE_LOC_3); + collective->broadcast(ISBroadcast(space_id, + runtime->get_unique_index_tree_id(), + runtime->get_unique_index_space_expr_id(), did, double_next)); + pending_index_spaces.push_back( + std::pair*,bool>(collective, true)); + } + else + { + ValueBroadcast *collective = + new ValueBroadcast(this, index_space_allocator_shard, + COLLECTIVE_LOC_3); + register_collective(collective); + pending_index_spaces.push_back( + std::pair*,bool>(collective, false)); + } + index_space_allocator_shard++; + if (index_space_allocator_shard == total_shards) + index_space_allocator_shard = 0; + double_next = false; + } + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::create_index_space(const Future &future, + TypeTag type_tag) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_INDEX_SPACE); + const Domain *domain = static_cast( + future.impl->get_untyped_result(true,NULL,true/*internal*/)); + Serializer rez; + rez.serialize(*domain); + hasher.hash(rez.get_buffer(), rez.get_used_bytes()); + hasher.hash(type_tag); + verify_replicable(hasher, "create_index_space"); + } + // Seed this with the first index space broadcast + if (pending_index_spaces.empty()) + increase_pending_index_spaces(1/*count*/, false/*double*/); + IndexSpace handle; + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_index_spaces.front(); + IndexSpaceNode *node = NULL; + // Get a new creation operation + CreationOp *creator_op = runtime->get_available_creation_op(); + const ApEvent ready = creator_op->get_completion_event(); + if (collective.second) + { + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid, type_tag); + double_buffer = value.double_buffer; + std::set applied; + node = runtime->forest->create_index_space(handle, NULL, value.did, + false/*notify remote*/, value.expr_id, ready, + creation_barrier, &applied); + // Now we can update the creation set + node->update_creation_set(shard_manager->get_mapping()); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + runtime->forest->revoke_pending_index_space(value.space_id); + runtime->revoke_pending_distributed_collectable(value.did); +#ifdef DEBUG_LEGION + log_index.debug("Creating index space %x in task%s (ID %lld)", + handle.id, get_task_name(), get_unique_id()); +#endif + if (runtime->legion_spy_enabled) + LegionSpy::log_top_index_space(handle.id); + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid, type_tag); + double_buffer = value.double_buffer; +#ifdef DEBUG_LEGION + assert(handle.exists()); +#endif + std::set applied; + node = runtime->forest->create_index_space(handle, NULL, value.did, + false/*notify remote*/, value.expr_id, ready, + creation_barrier, &applied); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + } + creator_op->initialize_index_space(this, node, future, + shard_manager->is_first_local_shard(owner_shard), + &(shard_manager->get_mapping())); + add_to_dependence_queue(creator_op); + delete collective.first; + pending_index_spaces.pop_front(); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + // Record this in our context + register_index_space_creation(handle); + // Get new handles in flight for the next time we need them + // Always add a new one to replace the old one, but double the number + // in flight if we're not hiding the latency + increase_pending_index_spaces(double_buffer ? + pending_index_spaces.size() + 1 : 1, double_next && !double_buffer); + return handle; + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::union_index_spaces( + const std::vector &spaces) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_UNION_INDEX_SPACES); + for (std::vector::const_iterator it = + spaces.begin(); it != spaces.end(); it++) + hasher.hash(*it); + verify_replicable(hasher, "union_index_spaces"); + } + if (spaces.empty()) + return IndexSpace::NO_SPACE; + bool none_exists = true; + for (std::vector::const_iterator it = + spaces.begin(); it != spaces.end(); it++) + { + if (none_exists && it->exists()) + none_exists = false; + if (spaces[0].get_type_tag() != it->get_type_tag()) + REPORT_LEGION_ERROR(ERROR_DYNAMIC_TYPE_MISMATCH, + "Dynamic type mismatch in 'union_index_spaces' " + "performed in task %s (UID %lld)", + get_task_name(), get_unique_id()) + } + if (none_exists) + return IndexSpace::NO_SPACE; + // Seed this with the first index space broadcast + if (pending_index_spaces.empty()) + increase_pending_index_spaces(1/*count*/, false/*double*/); + IndexSpace handle; + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_index_spaces.front(); + if (collective.second) + { + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid,spaces[0].get_type_tag()); + double_buffer = value.double_buffer; + std::set applied; + IndexSpaceNode *node = + runtime->forest->create_union_space(handle, value.did, spaces, + creation_barrier, false/*notify remote*/, value.expr_id, &applied); + // Now we can update the creation set + node->update_creation_set(shard_manager->get_mapping()); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + runtime->forest->revoke_pending_index_space(value.space_id); + runtime->revoke_pending_distributed_collectable(value.did); +#ifdef DEBUG_LEGION + log_index.debug("Creating index space %x in task%s (ID %lld)", + handle.id, get_task_name(), get_unique_id()); +#endif + if (runtime->legion_spy_enabled) + LegionSpy::log_top_index_space(handle.id); + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid,spaces[0].get_type_tag()); + double_buffer = value.double_buffer; +#ifdef DEBUG_LEGION + assert(handle.exists()); +#endif + std::set applied; + runtime->forest->create_union_space(handle, value.did, spaces, + creation_barrier, false/*notify remote*/, value.expr_id, &applied); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + } + delete collective.first; + pending_index_spaces.pop_front(); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + // Record this in our context + register_index_space_creation(handle); + // Get new handles in flight for the next time we need them + // Always add a new one to replace the old one, but double the number + // in flight if we're not hiding the latency + increase_pending_index_spaces(double_buffer ? + pending_index_spaces.size() + 1 : 1, double_next && !double_buffer); + return handle; + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::intersect_index_spaces( + const std::vector &spaces) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_INTERSECT_INDEX_SPACES); + for (std::vector::const_iterator it = + spaces.begin(); it != spaces.end(); it++) + hasher.hash(*it); + verify_replicable(hasher, "intersect_index_spaces"); + } + if (spaces.empty()) + return IndexSpace::NO_SPACE; + bool none_exists = true; + for (std::vector::const_iterator it = + spaces.begin(); it != spaces.end(); it++) + { + if (none_exists && it->exists()) + none_exists = false; + if (spaces[0].get_type_tag() != it->get_type_tag()) + REPORT_LEGION_ERROR(ERROR_DYNAMIC_TYPE_MISMATCH, + "Dynamic type mismatch in 'intersect_index_spaces' " + "performed in task %s (UID %lld)", + get_task_name(), get_unique_id()) + } + if (none_exists) + return IndexSpace::NO_SPACE; + // Seed this with the first index space broadcast + if (pending_index_spaces.empty()) + increase_pending_index_spaces(1/*count*/, false/*double*/); + IndexSpace handle; + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_index_spaces.front(); + if (collective.second) + { + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid,spaces[0].get_type_tag()); + double_buffer = value.double_buffer; + std::set applied; + IndexSpaceNode *node = + runtime->forest->create_intersection_space(handle, value.did, spaces, + creation_barrier, false/*notify remote*/, value.expr_id, &applied); + // Now we can update the creation set + node->update_creation_set(shard_manager->get_mapping()); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + runtime->forest->revoke_pending_index_space(value.space_id); + runtime->revoke_pending_distributed_collectable(value.did); +#ifdef DEBUG_LEGION + log_index.debug("Creating index space %x in task%s (ID %lld)", + handle.id, get_task_name(), get_unique_id()); +#endif + if (runtime->legion_spy_enabled) + LegionSpy::log_top_index_space(handle.id); + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid,spaces[0].get_type_tag()); + double_buffer = value.double_buffer; +#ifdef DEBUG_LEGION + assert(handle.exists()); +#endif + std::set applied; + runtime->forest->create_intersection_space(handle, value.did, spaces, + creation_barrier, false/*notify remote*/, value.expr_id, &applied); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + } + delete collective.first; + pending_index_spaces.pop_front(); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + // Record this in our context + register_index_space_creation(handle); + // Get new handles in flight for the next time we need them + // Always add a new one to replace the old one, but double the number + // in flight if we're not hiding the latency + increase_pending_index_spaces(double_buffer ? + pending_index_spaces.size() + 1 : 1, double_next && !double_buffer); + return handle; + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::subtract_index_spaces( + IndexSpace left, IndexSpace right) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_SUBTRACT_INDEX_SPACES); + hasher.hash(left); + hasher.hash(right); + verify_replicable(hasher, "subtract_index_spaces"); + } + if (!left.exists()) + return IndexSpace::NO_SPACE; + if (right.exists() && left.get_type_tag() != right.get_type_tag()) + REPORT_LEGION_ERROR(ERROR_DYNAMIC_TYPE_MISMATCH, + "Dynamic type mismatch in 'create_difference_spaces' " + "performed in task %s (UID %lld)", + get_task_name(), get_unique_id()) + // Seed this with the first index space broadcast + if (pending_index_spaces.empty()) + increase_pending_index_spaces(1/*count*/, false/*double*/); + IndexSpace handle; + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_index_spaces.front(); + if (collective.second) + { + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid, left.get_type_tag()); + double_buffer = value.double_buffer; + std::set applied; + IndexSpaceNode *node = + runtime->forest->create_difference_space(handle, value.did, left, + right,creation_barrier,false/*notify remote*/,value.expr_id,&applied); + // Now we can update the creation set + node->update_creation_set(shard_manager->get_mapping()); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + runtime->forest->revoke_pending_index_space(value.space_id); + runtime->revoke_pending_distributed_collectable(value.did); +#ifdef DEBUG_LEGION + log_index.debug("Creating index space %x in task%s (ID %lld)", + handle.id, get_task_name(), get_unique_id()); +#endif + if (runtime->legion_spy_enabled) + LegionSpy::log_top_index_space(handle.id); + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const ISBroadcast value = collective.first->get_value(false); + handle = IndexSpace(value.space_id, value.tid, left.get_type_tag()); + double_buffer = value.double_buffer; +#ifdef DEBUG_LEGION + assert(handle.exists()); +#endif + std::set applied; + runtime->forest->create_difference_space(handle, value.did, left, right, + creation_barrier, false/*notify remote*/, value.expr_id, &applied); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + } + delete collective.first; + pending_index_spaces.pop_front(); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + // Record this in our context + register_index_space_creation(handle); + // Get new handles in flight for the next time we need them + // Always add a new one to replace the old one, but double the number + // in flight if we're not hiding the latency + increase_pending_index_spaces(double_buffer ? + pending_index_spaces.size() + 1 : 1, double_next && !double_buffer); + return handle; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::create_shared_ownership(IndexSpace handle) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_SHARED_OWNERSHIP); + hasher.hash(handle); + verify_replicable(hasher, "create_shared_ownership"); + } + if (!handle.exists()) + return; + // Check to see if this is a top-level index space, if not then + // we shouldn't even be destroying it + if (!runtime->forest->is_top_level_index_space(handle)) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARED_OWNERSHIP, + "Illegal call to create shared ownership for index space %x in " + "task %s (UID %lld) which is not a top-level index space. Legion " + "only permits top-level index spaces to have shared ownership.", + handle.get_id(), get_task_name(), get_unique_id()) + if (shard_manager->is_total_sharding() && + shard_manager->is_first_local_shard(owner_shard)) + runtime->create_shared_ownership(handle, true/*total sharding*/); + else if (owner_shard->shard_id == 0) + runtime->create_shared_ownership(handle); + AutoLock priv_lock(privilege_lock); + std::map::iterator finder = + created_index_spaces.find(handle); + if (finder != created_index_spaces.end()) + finder->second++; + else + created_index_spaces[handle] = 1; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::destroy_index_space(IndexSpace handle, + const bool unordered, const bool recurse) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication && !unordered) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_DESTROY_INDEX_SPACE); + hasher.hash(handle); + hasher.hash(recurse); + verify_replicable(hasher, "destroy_index_space"); + } + if (!handle.exists()) + return; +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_index.debug("Destroying index space %x in task %s (ID %lld)", + handle.id, get_task_name(), get_unique_id()); +#endif + // Check to see if this is a top-level index space, if not then + // we shouldn't even be destroying it + if (!runtime->forest->is_top_level_index_space(handle)) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_RESOURCE_DESTRUCTION, + "Illegal call to destroy index space %x in task %s (UID %lld) " + "which is not a top-level index space. Legion only permits " + "top-level index spaces to be destroyed.", handle.get_id(), + get_task_name(), get_unique_id()) + // Check to see if this is one that we should be allowed to destory + std::vector sub_partitions; + { + AutoLock priv_lock(privilege_lock); + std::map::iterator finder = + created_index_spaces.find(handle); + if (finder == created_index_spaces.end()) + { + // If we didn't make the index space in this context, just + // record it and keep going, it will get handled later + deleted_index_spaces.push_back(std::make_pair(handle,recurse)); + return; + } + else + { +#ifdef DEBUG_LEGION + assert(finder->second > 0); +#endif + if (--finder->second == 0) + created_index_spaces.erase(finder); + else + return; + } + if (recurse) + { + // Also remove any index partitions for this index space tree + for (std::map::iterator it = + created_index_partitions.begin(); it != + created_index_partitions.end(); /*nothing*/) + { + if (it->first.get_tree_id() == handle.get_tree_id()) + { + sub_partitions.push_back(it->first); +#ifdef DEBUG_LEGION + assert(it->second > 0); +#endif + if (--it->second == 0) + { + std::map::iterator to_delete = it++; + created_index_partitions.erase(to_delete); + } + else + it++; + } + else + it++; + } + } + } + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_index_space_deletion(this,handle,sub_partitions,unordered); + op->initialize_replication(this, deletion_ready_barrier, + deletion_mapping_barrier, deletion_execution_barrier, + shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + add_to_dependence_queue(op, unordered); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::create_shared_ownership(IndexPartition handle) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_SHARED_OWNERSHIP); + hasher.hash(handle); + verify_replicable(hasher, "create_shared_ownership"); + } + if (!handle.exists()) + return; + if (shard_manager->is_total_sharding() && + shard_manager->is_first_local_shard(owner_shard)) + runtime->create_shared_ownership(handle, true/*total sharding*/); + else if (owner_shard->shard_id == 0) + runtime->create_shared_ownership(handle); + AutoLock priv_lock(privilege_lock); + std::map::iterator finder = + created_index_partitions.find(handle); + if (finder != created_index_partitions.end()) + finder->second++; + else + created_index_partitions[handle] = 1; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::destroy_index_partition(IndexPartition handle, + const bool unordered, const bool recurse) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication && !unordered) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_DESTROY_INDEX_PARTITION); + hasher.hash(handle); + hasher.hash(recurse); + verify_replicable(hasher, "destroy_index_partition"); + } + if (!handle.exists()) + return; +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_index.debug("Destroying index partition %x in task %s (ID %lld)", + handle.id, get_task_name(), get_unique_id()); +#endif + std::vector sub_partitions; + { + AutoLock priv_lock(privilege_lock); + std::map::iterator finder = + created_index_partitions.find(handle); + if (finder != created_index_partitions.end()) + { +#ifdef DEBUG_LEGION + assert(finder->second > 0); +#endif + if (--finder->second == 0) + created_index_partitions.erase(finder); + else + return; + if (recurse) + { + // Remove any other partitions that this partition dominates + for (std::map::iterator it = + created_index_partitions.begin(); it != + created_index_partitions.end(); /*nothing*/) + { + if ((handle.get_tree_id() == it->first.get_tree_id()) && + runtime->forest->is_dominated_tree_only(it->first, handle)) + { + sub_partitions.push_back(it->first); +#ifdef DEBUG_LEGION + assert(it->second > 0); +#endif + if (--it->second == 0) + { + std::map::iterator to_delete = it++; + created_index_partitions.erase(to_delete); + } + else + it++; + } + else + it++; + } + } + } + else + { + // If we didn't make the partition, record it and keep going + deleted_index_partitions.push_back(std::make_pair(handle,recurse)); + return; + } + } + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_index_part_deletion(this, handle, + sub_partitions, unordered); + op->initialize_replication(this, deletion_ready_barrier, + deletion_mapping_barrier, deletion_execution_barrier, + shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + add_to_dependence_queue(op, unordered); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::increase_pending_partitions(unsigned count, + bool double_next) + //-------------------------------------------------------------------------- + { + for (unsigned idx = 0; idx < count; idx++) + { + if (owner_shard->shard_id == index_partition_allocator_shard) + { + const IndexPartitionID pid = runtime->get_unique_index_partition_id(); + const DistributedID did = runtime->get_available_distributed_id(); + // We're the owner, so make it locally and then broadcast it + runtime->forest->record_pending_partition(pid); + runtime->record_pending_distributed_collectable(did); + // Do our arrival on this generation, should be the last one + ValueBroadcast *collective = + new ValueBroadcast(this, COLLECTIVE_LOC_7); + collective->broadcast(IPBroadcast(pid, did, double_next)); + pending_index_partitions.push_back( + std::pair*,ShardID>(collective, + index_partition_allocator_shard)); + } + else + { + ValueBroadcast *collective = + new ValueBroadcast(this,index_partition_allocator_shard, + COLLECTIVE_LOC_7); + register_collective(collective); + pending_index_partitions.push_back( + std::pair*,ShardID>(collective, + index_partition_allocator_shard)); + } + index_partition_allocator_shard++; + if (index_partition_allocator_shard == total_shards) + index_partition_allocator_shard = 0; + double_next = false; + } + } + + //-------------------------------------------------------------------------- + bool ReplicateContext::create_shard_partition(IndexPartition &pid, + IndexSpace parent, IndexSpace color_space, PartitionKind part_kind, + LegionColor partition_color, bool color_generated, + ValueBroadcast *disjoint_result/*=NULL*/, + ApBarrier partition_ready /*=ApBarrier::NO_AP_BARRIER*/) + //-------------------------------------------------------------------------- + { + if (pending_index_partitions.empty()) + increase_pending_partitions(1/*count*/, false/*double*/); + bool double_next = false; + bool double_buffer = false; + std::pair*,ShardID> &collective = + pending_index_partitions.front(); + const bool is_owner = (collective.second == owner_shard->shard_id); + if (is_owner) + { + const IPBroadcast value = collective.first->get_value(false); + pid.id = value.pid; + double_buffer = value.double_buffer; + // Have to do our registration before broadcasting + RtEvent safe_event = runtime->forest->create_pending_partition_shard( + collective.second, this, pid, parent, + color_space, partition_color, + part_kind,value.did,disjoint_result, + partition_ready.exists() ? + partition_ready : + pending_partition_barrier, + shard_manager->get_mapping(), + creation_barrier, partition_ready); + // Broadcast the color if we have to generate it + if (color_generated) + { +#ifdef DEBUG_LEGION + assert(partition_color != INVALID_COLOR); // we should have an ID +#endif + ValueBroadcast color_collective(this, COLLECTIVE_LOC_8); + color_collective.broadcast(partition_color); + } + // Signal that we're done our creation + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, safe_event); + runtime->forest->revoke_pending_partition(value.pid); + runtime->revoke_pending_distributed_collectable(value.did); + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const IPBroadcast value = collective.first->get_value(false); + pid.id = value.pid; + double_buffer = value.double_buffer; +#ifdef DEBUG_LEGION + assert(pid.exists()); +#endif + // If we need a color then we can get that too + if (color_generated) + { + ValueBroadcast color_collective(this, collective.second, + COLLECTIVE_LOC_8); + partition_color = color_collective.get_value(); +#ifdef DEBUG_LEGION + assert(partition_color != INVALID_COLOR); +#endif + } + // Do our registration + RtEvent safe_event = runtime->forest->create_pending_partition_shard( + collective.second, this, pid, parent, + color_space, partition_color, + part_kind, value.did, disjoint_result, + partition_ready.exists() ? + partition_ready : + pending_partition_barrier, + shard_manager->get_mapping(), + creation_barrier, partition_ready); + // Signal that we're done our creation + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, safe_event); + } + // Clean up the collective + delete collective.first; + pending_index_partitions.pop_front(); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + // Get new handles in flight for the next time we need them + // Always add a new one to replace the old one, but double the number + // in flight if we're not hiding the latency + increase_pending_partitions(double_buffer ? + pending_index_partitions.size() + 1 : 1, double_next && !double_buffer); + return is_owner; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_equal_partition( + IndexSpace parent, + IndexSpace color_space, + size_t granularity, + Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_EQUAL_PARTITION); + hasher.hash(parent); + hasher.hash(color_space); + hasher.hash(granularity); + hasher.hash(color); + verify_replicable(hasher, "create_equal_partition"); + } + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + LegionColor partition_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + partition_color = color; + else + color_generated = true; + if (create_shard_partition(pid, parent,color_space, + LEGION_DISJOINT_COMPLETE_KIND, partition_color, color_generated)) + log_index.debug("Creating equal partition %d with parent index space %x" + " in task %s (ID %lld)", pid.id, parent.id, + get_task_name(), get_unique_id()); + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_equal_partition(this, pid, granularity); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // Trigger the pending partition barrier and advance it + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_weights( + IndexSpace parent, + const FutureMap &weights, + IndexSpace color_space, + size_t granularity, Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_WEIGHTS); + hasher.hash(parent); +#ifdef DEBUG_LEGION + assert(weights.impl != NULL); + ReplFutureMapImpl *impl = + dynamic_cast(weights.impl); + assert(impl != NULL); +#else + ReplFutureMapImpl *impl = + static_cast(weights.impl); +#endif + hasher.hash(impl->op_ctx_index); + hasher.hash(color_space); + hasher.hash(granularity); + hasher.hash(color); + verify_replicable(hasher, "create_partition_by_weights"); + } + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + LegionColor partition_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + partition_color = color; + else + color_generated = true; + if (create_shard_partition(pid, parent,color_space, + LEGION_DISJOINT_COMPLETE_KIND, partition_color, color_generated)) + log_index.debug("Creating equal partition %d with parent index space %x" + " in task %s (ID %lld)", pid.id, parent.id, + get_task_name(), get_unique_id()); + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_weight_partition(this, pid, weights, granularity); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // Trigger the pending partition barrier and advance it + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_union( + IndexSpace parent, + IndexPartition handle1, + IndexPartition handle2, + IndexSpace color_space, + PartitionKind kind, Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_UNION); + hasher.hash(parent); + hasher.hash(handle1); + hasher.hash(handle2); + hasher.hash(color_space); + hasher.hash(kind); + hasher.hash(color); + verify_replicable(hasher, "create_partition_by_union"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, kind) +#ifdef DEBUG_LEGION + if (parent.get_tree_id() != handle1.get_tree_id()) + REPORT_LEGION_ERROR(ERROR_INDEX_TREE_MISMATCH, + "IndexPartition %d is not part of the same " + "index tree as IndexSpace %d in create " + "partition by union!", handle1.id, parent.id) + if (parent.get_tree_id() != handle2.get_tree_id()) + REPORT_LEGION_ERROR(ERROR_INDEX_TREE_MISMATCH, + "IndexPartition %d is not part of the same " + "index tree as IndexSpace %d in create " + "partition by union!", handle2.id, parent.id) +#endif + LegionColor partition_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + partition_color = color; + else + color_generated = true; + // If either partition is aliased the result is aliased + if ((kind == LEGION_COMPUTE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + { + // If one of these partitions is aliased then the result is aliased + IndexPartNode *p1 = runtime->forest->get_node(handle1); + if (p1->is_disjoint(true/*from app*/)) + { + IndexPartNode *p2 = runtime->forest->get_node(handle2); + if (!p2->is_disjoint(true/*from app*/)) + { + if (kind == LEGION_COMPUTE_KIND) + kind = LEGION_ALIASED_KIND; + else if (kind == LEGION_COMPUTE_COMPLETE_KIND) + kind = LEGION_ALIASED_COMPLETE_KIND; + else + kind = LEGION_ALIASED_INCOMPLETE_KIND; + } + } + else + { + if (kind == LEGION_COMPUTE_KIND) + kind = LEGION_ALIASED_KIND; + else if (kind == LEGION_COMPUTE_COMPLETE_KIND) + kind = LEGION_ALIASED_COMPLETE_KIND; + else + kind = LEGION_ALIASED_INCOMPLETE_KIND; + } + } + ValueBroadcast *disjoint_result = NULL; + if ((kind == LEGION_COMPUTE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_61); + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + if (create_shard_partition(pid, parent, color_space, kind, + partition_color, color_generated, disjoint_result)) + log_index.debug("Creating union partition %d with parent index " + "space %x in task %s (ID %lld)", pid.id, parent.id, + get_task_name(), get_unique_id()); + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_union_partition(this, pid, handle1, handle2); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_intersection( + IndexSpace parent, + IndexPartition handle1, + IndexPartition handle2, + IndexSpace color_space, + PartitionKind kind, Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_INTERSECTION); + hasher.hash(parent); + hasher.hash(handle1); + hasher.hash(handle2); + hasher.hash(color_space); + hasher.hash(kind); + hasher.hash(color); + verify_replicable(hasher, "create_partition_by_intersection"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, kind) +#ifdef DEBUG_LEGION + if (parent.get_tree_id() != handle1.get_tree_id()) + REPORT_LEGION_ERROR(ERROR_INDEX_TREE_MISMATCH, + "IndexPartition %d is not part of the same " + "index tree as IndexSpace %d in create partition by " + "intersection!", handle1.id, parent.id) + if (parent.get_tree_id() != handle2.get_tree_id()) + REPORT_LEGION_ERROR(ERROR_INDEX_TREE_MISMATCH, + "IndexPartition %d is not part of the same " + "index tree as IndexSpace %d in create partition by " + "intersection!", handle2.id, parent.id) +#endif + LegionColor partition_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + partition_color = color; + else + color_generated = true; + // If either partition is disjoint then the result is disjoint + if ((kind == LEGION_COMPUTE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + { + IndexPartNode *p1 = runtime->forest->get_node(handle1); + if (!p1->is_disjoint(true/*from app*/)) + { + IndexPartNode *p2 = runtime->forest->get_node(handle2); + if (p2->is_disjoint(true/*from app*/)) + { + if (kind == LEGION_COMPUTE_KIND) + kind = LEGION_DISJOINT_KIND; + else if (kind == LEGION_COMPUTE_COMPLETE_KIND) + kind = LEGION_DISJOINT_COMPLETE_KIND; + else + kind = LEGION_DISJOINT_INCOMPLETE_KIND; + } + } + else + { + if (kind == LEGION_COMPUTE_KIND) + kind = LEGION_DISJOINT_KIND; + else if (kind == LEGION_COMPUTE_COMPLETE_KIND) + kind = LEGION_DISJOINT_COMPLETE_KIND; + else + kind = LEGION_DISJOINT_INCOMPLETE_KIND; + } + } + ValueBroadcast *disjoint_result = NULL; + if ((kind == LEGION_COMPUTE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_62); + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + if (create_shard_partition(pid, parent, color_space, kind, + partition_color, color_generated, disjoint_result)) + log_index.debug("Creating intersection partition %d with parent " + "index space %x in task %s (ID %lld)", pid.id, + parent.id, get_task_name(), get_unique_id()); + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_intersection_partition(this, pid, handle1, handle2); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_intersection( + IndexSpace parent, + IndexPartition partition, + PartitionKind kind, Color color, + bool dominates) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_INTERSECTION); + hasher.hash(parent); + hasher.hash(partition); + hasher.hash(kind); + hasher.hash(color); + hasher.hash(dominates); + verify_replicable(hasher, "create_partition_by_intersection"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, kind) +#ifdef DEBUG_LEGION + if (parent.get_type_tag() != partition.get_type_tag()) + REPORT_LEGION_ERROR(ERROR_INDEXPARTITION_NOT_SAME_INDEX_TREE, + "IndexPartition %d does not have the same type as the " + "parent index space %x in task %s (UID %lld)", partition.id, + parent.id, get_task_name(), get_unique_id()) +#endif + LegionColor partition_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + partition_color = color; + else + color_generated = true; + IndexPartNode *part_node = runtime->forest->get_node(partition); + // See if we can determine disjointness if we weren't told + if ((kind == LEGION_COMPUTE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + { + if (part_node->is_disjoint(true/*from app*/)) + { + if (kind == LEGION_COMPUTE_KIND) + kind = LEGION_DISJOINT_KIND; + else if (kind == LEGION_COMPUTE_COMPLETE_KIND) + kind = LEGION_DISJOINT_COMPLETE_KIND; + else + kind = LEGION_DISJOINT_INCOMPLETE_KIND; + } + } + ValueBroadcast *disjoint_result = NULL; + if ((kind == LEGION_COMPUTE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_62); + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + if (create_shard_partition(pid, parent, part_node->color_space->handle, + kind, partition_color, color_generated, disjoint_result)) + log_index.debug("Creating intersection partition %d with parent " + "index space %x in task %s (ID %lld)", pid.id, + parent.id, get_task_name(), get_unique_id()); + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_intersection_partition(this,pid,partition,dominates); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_difference( + IndexSpace parent, + IndexPartition handle1, + IndexPartition handle2, + IndexSpace color_space, + PartitionKind kind, + Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_DIFFERENCE); + hasher.hash(parent); + hasher.hash(handle1); + hasher.hash(handle2); + hasher.hash(color_space); + hasher.hash(kind); + hasher.hash(color); + verify_replicable(hasher, "create_partition_by_difference"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, kind) +#ifdef DEBUG_LEGION + if (parent.get_tree_id() != handle1.get_tree_id()) + REPORT_LEGION_ERROR(ERROR_INDEX_TREE_MISMATCH, + "IndexPartition %d is not part of the same " + "index tree as IndexSpace %d in create " + "partition by difference!", + handle1.id, parent.id) + if (parent.get_tree_id() != handle2.get_tree_id()) + REPORT_LEGION_ERROR(ERROR_INDEX_TREE_MISMATCH, + "IndexPartition %d is not part of the same " + "index tree as IndexSpace %d in create " + "partition by difference!", + handle2.id, parent.id) +#endif + LegionColor partition_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + partition_color = color; + else + color_generated = true; + // If the left-hand-side is disjoint the result is disjoint + if ((kind == LEGION_COMPUTE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + { + IndexPartNode *p1 = runtime->forest->get_node(handle1); + if (p1->is_disjoint(true/*from app*/)) + { + if (kind == LEGION_COMPUTE_KIND) + kind = LEGION_DISJOINT_KIND; + else if (kind == LEGION_COMPUTE_COMPLETE_KIND) + kind = LEGION_DISJOINT_COMPLETE_KIND; + else + kind = LEGION_DISJOINT_INCOMPLETE_KIND; + } + } + ValueBroadcast *disjoint_result = NULL; + if ((kind == LEGION_COMPUTE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_63); + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + if (create_shard_partition(pid, parent, color_space, kind, + partition_color, color_generated, disjoint_result)) + log_index.debug("Creating difference partition %d with parent " + "index space %x in task %s (ID %lld)", pid.id, + parent.id, get_task_name(), get_unique_id()); + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_difference_partition(this, pid, handle1, handle2); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + Color ReplicateContext::create_cross_product_partitions( + IndexPartition handle1, + IndexPartition handle2, + std::map &handles, + PartitionKind kind, + Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_CROSS_PRODUCT_PARTITIONS); + hasher.hash(handle1); + hasher.hash(handle2); + hasher.hash(kind); + hasher.hash(color); + verify_replicable(hasher, "create_cross_product_partitions"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, kind) +#ifdef DEBUG_LEGION + log_index.debug("Creating cross product partitions in task %s (ID %lld)", + get_task_name(), get_unique_id()); + if (handle1.get_tree_id() != handle2.get_tree_id()) + REPORT_LEGION_ERROR(ERROR_INDEX_TREE_MISMATCH, + "IndexPartition %d is not part of the same " + "index tree as IndexPartition %d in create " + "cross product partitions!", + handle1.id, handle2.id) +#endif + LegionColor partition_color = INVALID_COLOR; + if (color != LEGION_AUTO_GENERATE_ID) + partition_color = color; + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + ApEvent term_event = part_op->get_completion_event(); + // We need an owner node to decide which color everyone is going to use + if (owner_shard->shard_id == index_partition_allocator_shard) + { + // Do the call on the owner node + std::set safe_events; + runtime->forest->create_pending_cross_product(this, handle1, handle2, + handles, kind, partition_color, + term_event, safe_events, + owner_shard->shard_id, total_shards); + // We need to wait on the safe event here to make sure effects + // have been broadcast before letting the other shard to their part + if (!safe_events.empty()) + { + const RtEvent wait_on = Runtime::merge_events(safe_events); + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + } + // Now broadcast the chosen color to all the other shards + ValueBroadcast color_collective(this, COLLECTIVE_LOC_15); + color_collective.broadcast(partition_color); + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + // Wait for the creation to be done + creation_barrier.wait(); + } + else + { + // Get the color result from the owner node + ValueBroadcast color_collective(this, + index_partition_allocator_shard, COLLECTIVE_LOC_15); + partition_color = color_collective.get_value(); +#ifdef DEBUG_LEGION + assert(partition_color != INVALID_COLOR); +#endif + // Now we can do the call from this node + std::set safe_events; + runtime->forest->create_pending_cross_product(this, handle1, handle2, + handles, kind, partition_color, + term_event, safe_events, + owner_shard->shard_id, total_shards); + // Signal that we're done with our creation + RtEvent safe_event; + if (!safe_events.empty()) + safe_event = Runtime::merge_events(safe_events); + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, safe_event); + // Also have to wait for creation to finish on all shards because + // any shard can handle requests for any cross-product partition + creation_barrier.wait(); + } + advance_replicate_barrier(creation_barrier, total_shards); + part_op->initialize_cross_product(this, handle1, handle2,partition_color); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // If we have any handles then we need to perform an exchange so + // that all the shards have all the names for the handles they need + if (!handles.empty()) + { + CrossProductCollective collective(this, COLLECTIVE_LOC_36); + collective.exchange_partitions(handles); + } + // Update our allocation shard + index_partition_allocator_shard++; + if (index_partition_allocator_shard == total_shards) + index_partition_allocator_shard = 0; + if (runtime->verify_partitions) + { + Domain color_space = runtime->get_index_partition_color_space(handle1); + // This code will only work if the color space has type coord_t + TypeTag type_tag; + switch (color_space.get_dim()) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + type_tag = NT_TemplateHelper::encode_tag(); \ + assert(handle1.get_type_tag() == type_tag); \ + break; \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + for (Domain::DomainPointIterator itr(color_space); itr; itr++) + { + IndexSpace subspace; + switch (color_space.get_dim()) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + const Point p(itr.p); \ + subspace = runtime->get_index_subspace(handle1, &p, type_tag); \ + break; \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + IndexPartition part = + runtime->get_index_partition(subspace, partition_color); + verify_partition(part, verify_kind, __func__); + } + } + return partition_color; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::create_association(LogicalRegion domain, + LogicalRegion domain_parent, + FieldID domain_fid, + IndexSpace range, + MapperID id, MappingTagID tag) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_ASSOCIATION); + hasher.hash(domain); + hasher.hash(domain_parent); + hasher.hash(domain_fid); + hasher.hash(range); + hasher.hash(id); + hasher.hash(tag); + verify_replicable(hasher, "create_association"); + } + ReplDependentPartitionOp *part_op = + runtime->get_available_repl_dependent_partition_op(); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_index.debug("Creating association in task %s (ID %lld)", + get_task_name(), get_unique_id()); + part_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_37)); +#endif + part_op->initialize_by_association(this, domain, domain_parent, + domain_fid, range, id, tag, dependent_partition_barrier); + // Now figure out if we need to unmap and re-map any inline mappings + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(part_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around create_association call " + "in task %s (UID %lld).", get_task_name(), get_unique_id()); + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(part_op); + // Remap any unmapped regions + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_restricted_partition( + IndexSpace parent, + IndexSpace color_space, + const void *transform, + size_t transform_size, + const void *extent, + size_t extent_size, + PartitionKind part_kind, + Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_RESTRICTED_PARTITION); + hasher.hash(parent); + hasher.hash(color_space); + hasher.hash(transform, transform_size); + hasher.hash(extent, extent_size); + hasher.hash(part_kind); + hasher.hash(color); + verify_replicable(hasher, "create_restricted_partition"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, part_kind) + LegionColor part_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + part_color = color; + else + color_generated = true; + ValueBroadcast *disjoint_result = NULL; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_64); + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + if (create_shard_partition(pid, parent, color_space, part_kind, + part_color, color_generated, disjoint_result)) + log_index.debug("Creating restricted partition in task %s (ID %lld)", + get_task_name(), get_unique_id()); + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_restricted_partition(this, pid, transform, + transform_size, extent, extent_size); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // Now update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_DOMAIN); + hasher.hash(parent); + Serializer rez; + for (std::map::const_iterator it = + domains.begin(); it != domains.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + hasher.hash(rez.get_buffer(), rez.get_used_bytes()); + hasher.hash(color_space); + hasher.hash(perform_intersections); + hasher.hash(part_kind); + hasher.hash(color); + verify_replicable(hasher, "create_partition_by_domain"); + } + Domain fm_domain; + RtUserEvent deletion_precondition; + // Have to include all the points in the domain computation + switch (color_space.get_dim()) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + std::vector > points(domains.size());\ + unsigned index = 0; \ + for (std::map::const_iterator it = \ + domains.begin(); it != domains.end(); it++) \ + { \ + const Point point = it->first; \ + points[index++] = point; \ + } \ + Realm::IndexSpace space(points); \ + const DomainT domaint(space); \ + fm_domain = domaint; \ + if (!space.dense()) \ + { \ + deletion_precondition = Runtime::create_rt_user_event(); \ + space.destroy(deletion_precondition); \ + } \ + break; \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + const DistributedID did = runtime->get_available_distributed_id(); + FutureMap future_map(new FutureMapImpl(this, runtime, fm_domain, did, + runtime->address_space, RtEvent::NO_RT_EVENT, true/*reg now*/, + deletion_precondition)); + // Prune out every N-th one for this shard and then pass through + // the subset to the normal InnerContext variation of this + ShardID shard = 0; + std::map shard_futures; + for (std::map::const_iterator it = + domains.begin(); it != domains.end(); it++) + { + if (shard++ == owner_shard->shard_id) + shard_futures[it->first] = Future::from_untyped_pointer( + runtime->external, &it->second, sizeof(it->second)); + if (shard == total_shards) + shard = 0; + } + future_map.impl->set_all_futures(shard_futures); + return create_partition_by_domain(parent, future_map, color_space, + perform_intersections, part_kind,color); + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_domain( + IndexSpace parent, + const FutureMap &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_DOMAIN); + hasher.hash(parent); +#ifdef DEBUG_LEGION + assert(domains.impl != NULL); + ReplFutureMapImpl *impl = + dynamic_cast(domains.impl); + assert(impl != NULL); +#else + ReplFutureMapImpl *impl = + static_cast(domains.impl); +#endif + hasher.hash(impl->op_ctx_index); + hasher.hash(color_space); + hasher.hash(perform_intersections); + hasher.hash(part_kind); + hasher.hash(color); + verify_replicable(hasher, "create_partition_by_domain"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, part_kind) + LegionColor part_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + part_color = color; + else + color_generated = true; + ValueBroadcast *disjoint_result = NULL; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_76); + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + if (create_shard_partition(pid, parent, color_space, part_kind, + part_color, color_generated, disjoint_result)) + log_index.debug("Creating partition by domain in task %s (ID %lld)", + get_task_name(), get_unique_id()); + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_by_domain(this, pid, domains, perform_intersections); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + // Now update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_field( + LogicalRegion handle, + LogicalRegion parent_priv, + FieldID fid, + IndexSpace color_space, + Color color, + MapperID id, MappingTagID tag, + PartitionKind part_kind) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_FIELD); + hasher.hash(handle); + hasher.hash(parent_priv); + hasher.hash(fid); + hasher.hash(color_space); + hasher.hash(color); + hasher.hash(id); + hasher.hash(tag); + hasher.hash(part_kind); + verify_replicable(hasher, "create_partition_by_field"); + } + // Partition by field is disjoint by construction + PartitionKind verify_kind = LEGION_DISJOINT_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, part_kind) + IndexSpace parent = handle.get_index_space(); + LegionColor part_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + part_color = color; + else + color_generated = true; + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + if (create_shard_partition(pid, parent, color_space, part_kind, + part_color, color_generated)) + log_index.debug("Creating partition by field in task %s (ID %lld)", + get_task_name(), get_unique_id()); + // Allocate the partition operation + ReplDependentPartitionOp *part_op = + runtime->get_available_repl_dependent_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_by_field(this, index_partition_allocator_shard, + pending_partition_barrier, pid, handle, + parent_priv, fid, id, tag, + dependent_partition_barrier); +#ifdef DEBUG_LEGION + part_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_38)); +#endif + // Now figure out if we need to unmap and re-map any inline mappings + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(part_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around create_partition_by_field call " + "in task %s (UID %lld).", get_task_name(), get_unique_id()); + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + // Remap any unmapped regions + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_image( + IndexSpace handle, + LogicalPartition projection, + LogicalRegion parent, + FieldID fid, + IndexSpace color_space, + PartitionKind part_kind, + Color color, + MapperID id, + MappingTagID tag) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_IMAGE); + hasher.hash(handle); + hasher.hash(projection); + hasher.hash(parent); + hasher.hash(fid); + hasher.hash(color_space); + hasher.hash(part_kind); + hasher.hash(color); + hasher.hash(id); + hasher.hash(tag); + verify_replicable(hasher, "create_partition_by_image"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, part_kind) + LegionColor part_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + part_color = color; + else + color_generated = true; + ValueBroadcast *disjoint_result = NULL; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_65); + IndexPartition pid(0/*temp*/, handle.get_tree_id(),handle.get_type_tag()); + if (create_shard_partition(pid, handle, color_space, part_kind, + part_color, color_generated, disjoint_result)) + log_index.debug("Creating partition by image in task %s (ID %lld)", + get_task_name(), get_unique_id()); + // Allocate the partition operation + ReplDependentPartitionOp *part_op = + runtime->get_available_repl_dependent_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_by_image(this, +#ifndef SHARD_BY_IMAGE + index_partition_allocator_shard, +#endif + pending_partition_barrier, + pid, projection, parent, fid, id, tag, + owner_shard->shard_id, total_shards, + dependent_partition_barrier); +#ifdef DEBUG_LEGION + part_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_39)); +#endif + // Now figure out if we need to unmap and re-map any inline mappings + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(part_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around create_partition_by_image call " + "in task %s (UID %lld).", get_task_name(), get_unique_id()); + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + // Remap any unmapped regions + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_image_range( + IndexSpace handle, + LogicalPartition projection, + LogicalRegion parent, + FieldID fid, + IndexSpace color_space, + PartitionKind part_kind, + Color color, + MapperID id, + MappingTagID tag) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_IMAGE_RANGE); + hasher.hash(handle); + hasher.hash(projection); + hasher.hash(parent); + hasher.hash(fid); + hasher.hash(color_space); + hasher.hash(part_kind); + hasher.hash(color); + hasher.hash(id); + hasher.hash(tag); + verify_replicable(hasher, "create_partition_by_image_range"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, part_kind) + LegionColor part_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + part_color = color; + else + color_generated = true; + ValueBroadcast *disjoint_result = NULL; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_66); + IndexPartition pid(0/*temp*/, handle.get_tree_id(),handle.get_type_tag()); + if (create_shard_partition(pid, handle, color_space, part_kind, + part_color, color_generated, disjoint_result)) + log_index.debug("Creating partition by image range in task %s " + "(ID %lld)", get_task_name(), get_unique_id()); + // Allocate the partition operation + ReplDependentPartitionOp *part_op = + runtime->get_available_repl_dependent_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_by_image_range(this, +#ifndef SHARD_BY_IMAGE + index_partition_allocator_shard, +#endif + pending_partition_barrier, + pid, projection, parent, fid, id, tag, + owner_shard->shard_id, total_shards, + dependent_partition_barrier); +#ifdef DEBUG_LEGION + part_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_40)); +#endif + // Now figure out if we need to unmap and re-map any inline mappings + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(part_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around create_partition_by_image_range call " + "in task %s (UID %lld).", get_task_name(), get_unique_id()); + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + // Remap any unmapped regions + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_preimage( + IndexPartition projection, + LogicalRegion handle, + LogicalRegion parent, + FieldID fid, + IndexSpace color_space, + PartitionKind part_kind, + Color color, + MapperID id, MappingTagID tag) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_PREIMAGE); + hasher.hash(projection); + hasher.hash(handle); + hasher.hash(parent); + hasher.hash(fid); + hasher.hash(color_space); + hasher.hash(part_kind); + hasher.hash(color); + hasher.hash(id); + hasher.hash(tag); + verify_replicable(hasher, "create_partition_by_preimage"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, part_kind) + LegionColor part_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + part_color = color; + else + color_generated = true; + // If the source of the preimage is disjoint then the result is disjoint + // Note this only applies here and not to range + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + { + IndexPartNode *p = runtime->forest->get_node(projection); + if (p->is_disjoint(true/*from app*/)) + { + if (part_kind == LEGION_COMPUTE_KIND) + part_kind = LEGION_DISJOINT_KIND; + else if (part_kind == LEGION_COMPUTE_COMPLETE_KIND) + part_kind = LEGION_DISJOINT_COMPLETE_KIND; + else + part_kind = LEGION_DISJOINT_INCOMPLETE_KIND; + } + } + ValueBroadcast *disjoint_result = NULL; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_67); + IndexPartition pid(0/*temp*/, + handle.get_index_space().get_tree_id(), parent.get_type_tag()); + if (create_shard_partition(pid, handle.get_index_space(), color_space, + part_kind, part_color, color_generated, disjoint_result)) + log_index.debug("Creating partition by preimage in task %s (ID %lld)", + get_task_name(), get_unique_id()); + // Allocate the partition operation + ReplDependentPartitionOp *part_op = + runtime->get_available_repl_dependent_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_by_preimage(this, index_partition_allocator_shard, + pending_partition_barrier, + pid, projection, handle, + parent, fid, id, tag, + dependent_partition_barrier); +#ifdef DEBUG_LEGION + part_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_41)); +#endif + // Now figure out if we need to unmap and re-map any inline mappings + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(part_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around create_partition_by_preimage call " + "in task %s (UID %lld).", get_task_name(), get_unique_id()); + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + // Remap any unmapped regions + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_partition_by_preimage_range( + IndexPartition projection, + LogicalRegion handle, + LogicalRegion parent, + FieldID fid, + IndexSpace color_space, + PartitionKind part_kind, + Color color, + MapperID id, MappingTagID tag) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PARTITION_BY_PREIMAGE_RANGE); + hasher.hash(projection); + hasher.hash(handle); + hasher.hash(parent); + hasher.hash(fid); + hasher.hash(color_space); + hasher.hash(part_kind); + hasher.hash(color); + hasher.hash(id); + hasher.hash(tag); + verify_replicable(hasher, "create_partition_by_preimage_range"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, part_kind) + LegionColor part_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + part_color = color; + else + color_generated = true; + ValueBroadcast *disjoint_result = NULL; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_68); + IndexPartition pid(0/*temp*/, + handle.get_index_space().get_tree_id(), parent.get_type_tag()); + if (create_shard_partition(pid, handle.get_index_space(), color_space, + part_kind, part_color, color_generated, disjoint_result)) + log_index.debug("Creating partition by preimage range in task %s " + "(ID %lld)", get_task_name(), get_unique_id()); + // Allocate the partition operation + ReplDependentPartitionOp *part_op = + runtime->get_available_repl_dependent_partition_op(); + const ApEvent term_event = part_op->get_completion_event(); + part_op->initialize_by_preimage_range(this, + index_partition_allocator_shard, + pending_partition_barrier, + pid, projection, handle, + parent, fid, id, tag, + dependent_partition_barrier); +#ifdef DEBUG_LEGION + part_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_42)); +#endif + // Now figure out if we need to unmap and re-map any inline mappings + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(part_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around create_partition_by_preimage_range call " + "in task %s (UID %lld).", get_task_name(), get_unique_id()); + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(part_op); + // Update the pending partition barrier + Runtime::phase_barrier_arrive(pending_partition_barrier, + 1/*count*/, term_event); + advance_replicate_barrier(pending_partition_barrier, total_shards); + // Remap any unmapped regions + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + if (runtime->verify_partitions) + verify_partition(pid, verify_kind, __func__); + return pid; + } + + //-------------------------------------------------------------------------- + IndexPartition ReplicateContext::create_pending_partition( + IndexSpace parent, + IndexSpace color_space, + PartitionKind part_kind, + Color color) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_PENDING_PARTITION); + hasher.hash(parent); + hasher.hash(color_space); + hasher.hash(part_kind); + hasher.hash(color); + verify_replicable(hasher, "create_pending_partition"); + } + PartitionKind verify_kind = LEGION_COMPUTE_KIND; + if (runtime->verify_partitions) + SWAP_PART_KINDS(verify_kind, part_kind) + LegionColor part_color = INVALID_COLOR; + bool color_generated = false; + if (color != LEGION_AUTO_GENERATE_ID) + part_color = color; + else + color_generated = true; + ValueBroadcast *disjoint_result = NULL; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + disjoint_result = new ValueBroadcast(this, + pending_index_partitions.empty() ? index_partition_allocator_shard : + pending_index_partitions.front().second, COLLECTIVE_LOC_69); + ApBarrier partition_ready; + if (owner_shard->shard_id == index_partition_allocator_shard) + { + // We have to make a barrier to be used for this partition + size_t color_space_size = + runtime->forest->get_domain_volume(color_space); + partition_ready = + ApBarrier(Realm::Barrier::create_barrier(color_space_size)); + ValueBroadcast bar_collective(this, COLLECTIVE_LOC_30); + bar_collective.broadcast(partition_ready); + } + else + { + ValueBroadcast bar_collective(this, + index_partition_allocator_shard, COLLECTIVE_LOC_30); + partition_ready = bar_collective.get_value(); + } + // Update our allocation shard + index_partition_allocator_shard++; + if (index_partition_allocator_shard == total_shards) + index_partition_allocator_shard = 0; + IndexPartition pid(0/*temp*/,parent.get_tree_id(),parent.get_type_tag()); + if (create_shard_partition(pid, parent, color_space, part_kind, + part_color, color_generated, disjoint_result, partition_ready)) + log_index.debug("Creating pending partition in task %s (ID %lld)", + get_task_name(), get_unique_id()); + if (runtime->verify_partitions) + { + // We can't block to check this here because the user needs + // control back in order to fill in the pieces of the partitions + // so just launch a meta-task to check it when we can + VerifyPartitionArgs args(this, pid, verify_kind, __func__); + runtime->issue_runtime_meta_task(args, LG_LOW_PRIORITY, + Runtime::protect_event(partition_ready)); + } + return pid; + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::create_index_space_union( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + const std::vector &handles) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_INDEX_SPACE_UNION); + hasher.hash(parent); + hasher.hash(realm_color, color_size); + hasher.hash(type_tag); + for (std::vector::const_iterator it = + handles.begin(); it != handles.end(); it++) + hasher.hash(*it); + verify_replicable(hasher, "create_index_space_union"); + } +#ifdef DEBUG_LEGION + log_index.debug("Creating index space union in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); + part_op->initialize_index_space_union(this, result, handles); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + return result; + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::create_index_space_union( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + IndexPartition handle) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_INDEX_SPACE_UNION); + hasher.hash(parent); + hasher.hash(realm_color, color_size); + hasher.hash(type_tag); + hasher.hash(handle); + verify_replicable(hasher, "create_index_space_union"); + } +#ifdef DEBUG_LEGION + log_index.debug("Creating index space union in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); + part_op->initialize_index_space_union(this, result, handle); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + return result; + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::create_index_space_intersection( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + const std::vector &handles) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_INDEX_SPACE_INTERSECTION); + hasher.hash(parent); + hasher.hash(realm_color, color_size); + hasher.hash(type_tag); + for (std::vector::const_iterator it = + handles.begin(); it != handles.end(); it++) + hasher.hash(*it); + verify_replicable(hasher, "create_index_space_intersection"); + } +#ifdef DEBUG_LEGION + log_index.debug("Creating index space intersection in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); + part_op->initialize_index_space_intersection(this, result, handles); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + return result; + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::create_index_space_intersection( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + IndexPartition handle) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_INDEX_SPACE_INTERSECTION); + hasher.hash(parent); + hasher.hash(realm_color, color_size); + hasher.hash(type_tag); + hasher.hash(handle); + verify_replicable(hasher, "create_index_space_intersection"); + } +#ifdef DEBUG_LEGION + log_index.debug("Creating index space intersection in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); + part_op->initialize_index_space_intersection(this, result, handle); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + return result; + } + + //-------------------------------------------------------------------------- + IndexSpace ReplicateContext::create_index_space_difference( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + IndexSpace initial, + const std::vector &handles) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_INDEX_SPACE_DIFFERENCE); + hasher.hash(parent); + hasher.hash(realm_color, color_size); + hasher.hash(type_tag); + hasher.hash(initial); + for (std::vector::const_iterator it = + handles.begin(); it != handles.end(); it++) + hasher.hash(*it); + verify_replicable(hasher, "create_index_space_difference"); + } +#ifdef DEBUG_LEGION + log_index.debug("Creating index space difference in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + ReplPendingPartitionOp *part_op = + runtime->get_available_repl_pending_partition_op(); + IndexSpace result = + runtime->forest->get_index_subspace(parent, realm_color, type_tag); + part_op->initialize_index_space_difference(this, result, initial,handles); + // Now we can add the operation to the queue + add_to_dependence_queue(part_op); + return result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::verify_partition(IndexPartition pid, + PartitionKind kind, const char *function_name) + //-------------------------------------------------------------------------- + { + IndexPartNode *node = runtime->forest->get_node(pid); + // Check containment first + if (node->total_children == node->max_linearized_color) + { + for (LegionColor color = owner_shard->shard_id; + color < node->total_children; color+=total_shards) + { + IndexSpaceNode *child_node = node->get_child(color); + IndexSpaceExpression *diff = + runtime->forest->subtract_index_spaces(child_node, node->parent); + if (!diff->is_empty()) + { + const DomainPoint bad = + node->color_space->delinearize_color_to_point(color); + switch (bad.get_dim()) + { + case 1: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0]) + case 2: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1]) + case 3: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2]) + case 4: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3]) + case 5: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4]) + case 6: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4], bad[5]) + case 7: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4], bad[5], bad[6]) + case 8: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld,%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4], bad[5], bad[6], + bad[7]) + case 9: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld,%lld,%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4], bad[5], bad[6], + bad[7], bad[8]) + default: + assert(false); + } + } + } + } + else + { + ColorSpaceIterator *itr = + node->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < owner_shard->shard_id; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } + while (itr->is_valid()) + { + const LegionColor color = itr->yield_color(); + IndexSpaceNode *child_node = node->get_child(color); + IndexSpaceExpression *diff = + runtime->forest->subtract_index_spaces(child_node, node->parent); + if (!diff->is_empty()) + { + const DomainPoint bad = + node->color_space->delinearize_color_to_point(color); + switch (bad.get_dim()) + { + case 1: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0]) + case 2: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1]) + case 3: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2]) + case 4: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3]) + case 5: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4]) + case 6: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4], bad[5]) + case 7: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4], bad[5], bad[6]) + case 8: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld,%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4], bad[5], bad[6], + bad[7]) + case 9: + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partition function %s in %s (UID %lld) has " + "non-dominated child sub-region at color (%lld,%lld," + "%lld,%lld,%lld,%lld,%lld,%lld,%lld).", + function_name, get_task_name(), get_unique_id(), + bad[0], bad[1], bad[2], bad[3], bad[4], bad[5], bad[6], + bad[7], bad[8]) + default: + assert(false); + } + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } + } + } + delete itr; + } + // Only need to do the rest of this on shard 0 + if (owner_shard->shard_id > 0) + return; + // Check disjointness + if ((kind == LEGION_DISJOINT_KIND) || + (kind == LEGION_DISJOINT_COMPLETE_KIND) || + (kind == LEGION_DISJOINT_INCOMPLETE_KIND)) + { + if (!node->is_disjoint(true/*from application*/)) + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partitioning function %s in %s (UID %lld) specified " + "partition was %s but the partition is aliased.", + function_name, get_task_name(), get_unique_id(), + (kind == LEGION_DISJOINT_KIND) ? "DISJOINT_KIND" : + (kind == LEGION_DISJOINT_COMPLETE_KIND) ? + "DISJOINT_COMPLETE_KIND" : "DISJOINT_INCOMPLETE_KIND") + } + else if ((kind == LEGION_ALIASED_KIND) || + (kind == LEGION_ALIASED_COMPLETE_KIND) || + (kind == LEGION_ALIASED_INCOMPLETE_KIND)) + { + if (node->is_disjoint(true/*from application*/)) + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partitioning function %s in %s (UID %lld) specified " + "partition was %s but the partition is disjoint.", + function_name, get_task_name(), get_unique_id(), + (kind == LEGION_ALIASED_KIND) ? "ALIASED_KIND" : + (kind == LEGION_ALIASED_COMPLETE_KIND) ? "ALIASED_COMPLETE_KIND" : + "ALIASED_INCOMPLETE_KIND") + } + // Check completeness + if ((kind == LEGION_DISJOINT_COMPLETE_KIND) || + (kind == LEGION_ALIASED_COMPLETE_KIND) || + (kind == LEGION_COMPUTE_COMPLETE_KIND)) + { + if (!node->is_complete(true/*from application*/)) + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partitioning function %s in %s (UID %lld) specified " + "partition was %s but the partition is incomplete.", + function_name, get_task_name(), get_unique_id(), + (kind == LEGION_DISJOINT_COMPLETE_KIND) ? "DISJOINT_COMPLETE_KIND" + : (kind == LEGION_ALIASED_COMPLETE_KIND) ? "ALIASED_COMPLETE_KIND" : + "COMPUTE_COMPLETE_KIND") + } + else if ((kind == LEGION_DISJOINT_INCOMPLETE_KIND) || + (kind == LEGION_ALIASED_INCOMPLETE_KIND) || + (kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + { + if (node->is_complete(true/*from application*/)) + REPORT_LEGION_ERROR(ERROR_PARTITION_VERIFICATION, + "Call to partitioning function %s in %s (UID %lld) specified " + "partition was %s but the partition is complete.", + function_name, get_task_name(), get_unique_id(), + (kind == LEGION_DISJOINT_INCOMPLETE_KIND) ? + "DISJOINT_INCOMPLETE_KIND" : + (kind == LEGION_ALIASED_INCOMPLETE_KIND) ? + "ALIASED_INCOMPLETE_KIND" : "COMPUTE_INCOMPLETE_KIND") + } + } + + //-------------------------------------------------------------------------- + FieldSpace ReplicateContext::create_field_space(void) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_FIELD_SPACE); + verify_replicable(hasher, "create_field_space"); + } + // Seed this with the first field space broadcast + if (pending_field_spaces.empty()) + increase_pending_field_spaces(1/*count*/, false/*double*/); + FieldSpace space; + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_field_spaces.front(); + ShardMapping &shard_mapping = shard_manager->get_mapping(); + if (collective.second) + { + const FSBroadcast value = collective.first->get_value(false); + space = FieldSpace(value.space_id); + double_buffer = value.double_buffer; + // Need to register this before broadcasting + std::set applied; + FieldSpaceNode *node = runtime->forest->create_field_space(space, + value.did, false/*notify remote*/, creation_barrier, &applied, + &shard_mapping); + // Now we can update the creation set + node->update_creation_set(shard_mapping); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + runtime->forest->revoke_pending_field_space(value.space_id); + runtime->revoke_pending_distributed_collectable(value.did); +#ifdef DEBUG_LEGION + log_field.debug("Creating field space %x in task %s (ID %lld)", + space.id, get_task_name(), get_unique_id()); +#endif + if (runtime->legion_spy_enabled) + LegionSpy::log_field_space(space.id); + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const FSBroadcast value = collective.first->get_value(false); + space = FieldSpace(value.space_id); + double_buffer = value.double_buffer; +#ifdef DEBUG_LEGION + assert(space.exists()); +#endif + std::set applied; + runtime->forest->create_field_space(space, value.did, + false/*notify remote*/, creation_barrier, &applied, &shard_mapping); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + } + delete collective.first; + pending_field_spaces.pop_front(); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + // Record this in our context + register_field_space_creation(space); + // Get new handles in flight for the next time we need them + // Always add a new one to replace the old one, but double the number + // in flight if we're not hiding the latency + increase_pending_field_spaces(double_buffer ? + pending_field_spaces.size() + 1 : 1, double_next && !double_buffer); + return space; + } + + //-------------------------------------------------------------------------- + FieldSpace ReplicateContext::create_field_space( + const std::vector &sizes, + std::vector &resulting_fields, + CustomSerdezID serdez_id) + //-------------------------------------------------------------------------- + { + FieldSpace result = create_field_space(); + allocate_fields(result, sizes, resulting_fields,false/*local*/,serdez_id); + return result; + } + + //-------------------------------------------------------------------------- + FieldSpace ReplicateContext::create_field_space( + const std::vector &sizes, + std::vector &resulting_fields, + CustomSerdezID serdez_id) + //-------------------------------------------------------------------------- + { + FieldSpace result = create_field_space(); + allocate_fields(result, sizes, resulting_fields,false/*local*/,serdez_id); + return result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::increase_pending_field_spaces(unsigned count, + bool double_next) + //-------------------------------------------------------------------------- + { + for (unsigned idx = 0; idx < count; idx++) + { + if (owner_shard->shard_id == field_space_allocator_shard) + { + const FieldSpaceID space = runtime->get_unique_field_space_id(); + const DistributedID did = runtime->get_available_distributed_id(); + // We're the owner, so make it locally and then broadcast it + runtime->forest->record_pending_field_space(space); + runtime->record_pending_distributed_collectable(did); + // Do our arrival on this generation, should be the last one + ValueBroadcast *collective = + new ValueBroadcast(this, COLLECTIVE_LOC_31); + collective->broadcast(FSBroadcast(space, did, double_next)); + pending_field_spaces.push_back( + std::pair*,bool>(collective, true)); + } + else + { + ValueBroadcast *collective = + new ValueBroadcast(this, field_space_allocator_shard, + COLLECTIVE_LOC_31); + register_collective(collective); + pending_field_spaces.push_back( + std::pair*,bool>(collective, false)); + } + field_space_allocator_shard++; + if (field_space_allocator_shard == total_shards) + field_space_allocator_shard = 0; + double_next = false; + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::create_shared_ownership(FieldSpace handle) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (!handle.exists()) + return; + if (shard_manager->is_total_sharding() && + shard_manager->is_first_local_shard(owner_shard)) + runtime->create_shared_ownership(handle, true/*total sharding*/); + else if (owner_shard->shard_id == 0) + runtime->create_shared_ownership(handle); + AutoLock priv_lock(privilege_lock); + std::map::iterator finder = + created_field_spaces.find(handle); + if (finder != created_field_spaces.end()) + finder->second++; + else + created_field_spaces[handle] = 1; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::destroy_field_space(FieldSpace handle, + const bool unordered) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication && !unordered) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_DESTROY_FIELD_SPACE); + hasher.hash(handle); + verify_replicable(hasher, "destroy_field_space"); + } + if (!handle.exists()) + return; +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_field.debug("Destroying field space %x in task %s (ID %lld)", + handle.id, get_task_name(), get_unique_id()); +#endif + // Check to see if this is one that we should be allowed to destory + { + AutoLock priv_lock(privilege_lock); + std::map::iterator finder = + created_field_spaces.find(handle); + if (finder != created_field_spaces.end()) + { +#ifdef DEBUG_LEGION + assert(finder->second > 0); +#endif + if (--finder->second == 0) + created_field_spaces.erase(finder); + else + return; + // Count how many regions are still using this field space + // that still need to be deleted before we can remove the + // list of created fields + std::set latent_regions; + for (std::map::const_iterator it = + created_regions.begin(); it != created_regions.end(); it++) + if (it->first.get_field_space() == handle) + latent_regions.insert(it->first); + for (std::map::const_iterator it = + local_regions.begin(); it != local_regions.end(); it++) + if (it->first.get_field_space() == handle) + latent_regions.insert(it->first); + if (latent_regions.empty()) + { + // No remaining regions so we can remove any created fields now + for (std::set >::iterator it = + created_fields.begin(); it != + created_fields.end(); /*nothing*/) + { + if (it->first == handle) + { + std::set >::iterator + to_delete = it++; + created_fields.erase(to_delete); + } + else + it++; + } + } + else + latent_field_spaces[handle] = latent_regions; + } + else + { + // If we didn't make this field space, record the deletion + // and keep going. It will be handled by the context that + // made the field space + deleted_field_spaces.push_back(handle); + return; + } + } + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_field_space_deletion(this, handle, unordered); + op->initialize_replication(this, deletion_ready_barrier, + deletion_mapping_barrier, deletion_execution_barrier, + shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + add_to_dependence_queue(op, unordered); + } + + //-------------------------------------------------------------------------- + FieldID ReplicateContext::allocate_field(FieldSpace space,size_t field_size, + FieldID fid, bool local, + CustomSerdezID serdez_id) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_ALLOCATE_FIELD); + hasher.hash(space); + hasher.hash(field_size); + hasher.hash(fid); + hasher.hash(local); + hasher.hash(serdez_id); + verify_replicable(hasher, "allocate_field"); + } + if (local) + REPORT_LEGION_FATAL(LEGION_FATAL_UNIMPLEMENTED_FEATURE, + "Local field creation is not currently supported " + "for control replication with task %s (UID %lld)", + get_task_name(), get_unique_id()) + if (fid == LEGION_AUTO_GENERATE_ID) + { + if (pending_fields.empty()) + increase_pending_fields(1/*count*/, false/*double*/); + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_fields.front(); + if (collective.second) + { + const FIDBroadcast value = collective.first->get_value(false); + fid = value.field_id; + double_buffer = value.double_buffer; + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const FIDBroadcast value = collective.first->get_value(false); + fid = value.field_id; + double_buffer = value.double_buffer; + } + delete collective.first; + pending_fields.pop_front(); + increase_pending_fields(double_buffer ? pending_fields.size() + 1 : 1, + double_next && !double_buffer); + } + else if (fid >= LEGION_MAX_APPLICATION_FIELD_ID) + REPORT_LEGION_ERROR(ERROR_TASK_ATTEMPTED_ALLOCATE_FIELD, + "Task %s (ID %lld) attempted to allocate a field with " + "ID %d which exceeds the LEGION_MAX_APPLICATION_FIELD_ID" + " bound set in legion_config.h", get_task_name(), + get_unique_id(), fid) + std::map >::const_iterator finder = + field_allocator_owner_shards.find(space); +#ifdef DEBUG_LEGION + assert(finder != field_allocator_owner_shards.end()); +#endif + RtEvent precondition; + // This deduplicates multiple shards on the same node + if (finder->second.second) + { + const bool non_owner = (finder->second.first != owner_shard->shard_id); + precondition = runtime->forest->allocate_field(space, field_size, fid, + serdez_id, non_owner); + if (runtime->legion_spy_enabled && !non_owner) + LegionSpy::log_field_creation(space.id, fid, field_size); + } + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, precondition); + // Launch the creation op in this context to act as a fence to ensure + // that the allocations are done on all shard nodes before anyone else + // tries to use them or their meta-data + CreationOp *creator_op = runtime->get_available_creation_op(); + creator_op->initialize_fence(this, creation_barrier); + add_to_dependence_queue(creator_op); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + register_field_creation(space, fid, local); + return fid; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::increase_pending_fields(unsigned count, + bool double_next) + //-------------------------------------------------------------------------- + { + for (unsigned idx = 0; idx < count; idx++) + { + if (owner_shard->shard_id == field_allocator_shard) + { + const FieldID fid = runtime->get_unique_field_id(); + // Do our arrival on this generation, should be the last one + ValueBroadcast *collective = + new ValueBroadcast(this, COLLECTIVE_LOC_33); + collective->broadcast(FIDBroadcast(fid, double_next)); + pending_fields.push_back( + std::pair*,bool>(collective, true)); + } + else + { + ValueBroadcast *collective = + new ValueBroadcast(this, field_allocator_shard, + COLLECTIVE_LOC_33); + register_collective(collective); + pending_fields.push_back( + std::pair*,bool>(collective, false)); + } + field_allocator_shard++; + if (field_allocator_shard == total_shards) + field_allocator_shard = 0; + double_next = false; + } + } + + //-------------------------------------------------------------------------- + FieldID ReplicateContext::allocate_field(FieldSpace space, + const Future &field_size, + FieldID fid, bool local, + CustomSerdezID serdez_id) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_ALLOCATE_FIELD); + hasher.hash(space); + const size_t *size = static_cast( + field_size.impl->get_untyped_result(true,NULL,true/*internal*/)); + hasher.hash(*size); + hasher.hash(fid); + hasher.hash(local); + hasher.hash(serdez_id); + verify_replicable(hasher, "allocate_field"); + } + if (local) + REPORT_LEGION_FATAL(LEGION_FATAL_UNIMPLEMENTED_FEATURE, + "Local field creation is not currently supported " + "for control replication with task %s (UID %lld)", + get_task_name(), get_unique_id()) + if (fid == LEGION_AUTO_GENERATE_ID) + { + if (pending_fields.empty()) + increase_pending_fields(1/*count*/, false/*double*/); + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_fields.front(); + if (collective.second) + { + const FIDBroadcast value = collective.first->get_value(false); + fid = value.field_id; + double_buffer = value.double_buffer; + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const FIDBroadcast value = collective.first->get_value(false); + fid = value.field_id; + double_buffer = value.double_buffer; + } + delete collective.first; + pending_fields.pop_front(); + increase_pending_fields(double_buffer ? pending_fields.size() + 1 : 1, + double_next && !double_buffer); + } + else if (fid >= LEGION_MAX_APPLICATION_FIELD_ID) + REPORT_LEGION_ERROR(ERROR_TASK_ATTEMPTED_ALLOCATE_FIELD, + "Task %s (ID %lld) attempted to allocate a field with " + "ID %d which exceeds the LEGION_MAX_APPLICATION_FIELD_ID" + " bound set in legion_config.h", get_task_name(), + get_unique_id(), fid) + if (field_size.impl == NULL) + REPORT_LEGION_ERROR(ERROR_REQUEST_FOR_EMPTY_FUTURE, + "Invalid empty future passed to field allocation for field %d " + "in task %s (UID %lld)", fid, get_task_name(), get_unique_id()) + std::map >::const_iterator finder = + field_allocator_owner_shards.find(space); +#ifdef DEBUG_LEGION + assert(finder != field_allocator_owner_shards.end()); +#endif + // Get a new creation operation + CreationOp *creator_op = runtime->get_available_creation_op(); + // This deduplicates multiple shards on the same node + if (finder->second.second) + { + const ApEvent ready = creator_op->get_completion_event(); + const bool owner = (finder->second.first == owner_shard->shard_id); + RtEvent precondition; + FieldSpaceNode *node = runtime->forest->allocate_field(space, ready, + fid, serdez_id, precondition, !owner); + Runtime::phase_barrier_arrive(creation_barrier,1/*count*/,precondition); + creator_op->initialize_field(this, node, fid, field_size, + precondition, owner); + } + else + { + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + creator_op->initialize_fence(this, creation_barrier); + } + // Launch the creation op in this context to act as a fence to ensure + // that the allocations are done on all shard nodes before anyone else + // tries to use them or their meta-data + add_to_dependence_queue(creator_op); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + register_field_creation(space, fid, local); + return fid; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::free_field(FieldAllocatorImpl *allocator, + FieldSpace space, FieldID fid, const bool unordered) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_FREE_FIELD); + hasher.hash(space); + hasher.hash(fid); + verify_replicable(hasher, "free_field"); + } + { + AutoLock priv_lock(privilege_lock,1,false/*exclusive*/); + const std::pair key(space, fid); + // This field will actually be removed in analyze_destroy_fields + std::set >::const_iterator finder = + created_fields.find(key); + if (finder == created_fields.end()) + { + std::map,bool>::iterator + local_finder = local_fields.find(key); + if (local_finder == local_fields.end()) + { + // If we didn't make this field, record the deletion and + // then have a later context handle it + deleted_fields.push_back(key); + return; + } + else + local_finder->second = true; + } + } + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_field_deletion(this, space, fid, unordered, allocator); + op->initialize_replication(this, deletion_ready_barrier, + deletion_mapping_barrier, deletion_execution_barrier, + shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + add_to_dependence_queue(op, unordered); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::allocate_fields(FieldSpace space, + const std::vector &sizes, + std::vector &resulting_fields, + bool local, CustomSerdezID serdez_id) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_ALLOCATE_FIELDS); + hasher.hash(space); + for (std::vector::const_iterator it = + sizes.begin(); it != sizes.end(); it++) + hasher.hash(*it); + for (std::vector::const_iterator it = + resulting_fields.begin(); it != resulting_fields.end(); it++) + hasher.hash(*it); + hasher.hash(local); + hasher.hash(serdez_id); + verify_replicable(hasher, "allocate_fields"); + } + if (local) + REPORT_LEGION_FATAL(LEGION_FATAL_UNIMPLEMENTED_FEATURE, + "Local field creation is not currently supported " + "for control replication with task %s (UID %lld)", + get_task_name(), get_unique_id()) + if (resulting_fields.size() < sizes.size()) + resulting_fields.resize(sizes.size(), LEGION_AUTO_GENERATE_ID); + for (unsigned idx = 0; idx < resulting_fields.size(); idx++) + { + if (resulting_fields[idx] == LEGION_AUTO_GENERATE_ID) + { + if (pending_fields.empty()) + increase_pending_fields(1/*count*/, false/*double*/); + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_fields.front(); + if (collective.second) + { + const FIDBroadcast value = collective.first->get_value(false); + resulting_fields[idx] = value.field_id; + double_buffer = value.double_buffer; + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const FIDBroadcast value = collective.first->get_value(false); + resulting_fields[idx] = value.field_id; + double_buffer = value.double_buffer; + } + delete collective.first; + pending_fields.pop_front(); + increase_pending_fields(double_buffer ? pending_fields.size() + 1 : 1, + double_next && !double_buffer); + } + else if (resulting_fields[idx] >= LEGION_MAX_APPLICATION_FIELD_ID) + REPORT_LEGION_ERROR(ERROR_TASK_ATTEMPTED_ALLOCATE_FIELD, + "Task %s (ID %lld) attempted to allocate a field with " + "ID %d which exceeds the LEGION_MAX_APPLICATION_FIELD_ID " + "bound set in legion_config.h", get_task_name(), + get_unique_id(), resulting_fields[idx]) + } + std::map >::const_iterator finder = + field_allocator_owner_shards.find(space); +#ifdef DEBUG_LEGION + assert(finder != field_allocator_owner_shards.end()); +#endif + RtEvent precondition; + // This deduplicates multiple shards on the same node + if (finder->second.second) + { + const bool non_owner = (finder->second.first != owner_shard->shard_id); + precondition = runtime->forest->allocate_fields(space, sizes, + resulting_fields, serdez_id, non_owner); + if (runtime->legion_spy_enabled && !non_owner) + for (unsigned idx = 0; idx < resulting_fields.size(); idx++) + LegionSpy::log_field_creation(space.id, + resulting_fields[idx], sizes[idx]); + } + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, precondition); + // Launch the creation op in this context to act as a fence to ensure + // that the allocations are done on all shard nodes before anyone else + // tries to use them or their meta-data + CreationOp *creator_op = runtime->get_available_creation_op(); + creator_op->initialize_fence(this, creation_barrier); + add_to_dependence_queue(creator_op); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + register_all_field_creations(space, local, resulting_fields); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::allocate_fields(FieldSpace space, + const std::vector &sizes, + std::vector &resulting_fields, + bool local, CustomSerdezID serdez_id) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_ALLOCATE_FIELDS); + hasher.hash(space); + for (std::vector::const_iterator it = + sizes.begin(); it != sizes.end(); it++) + { + const size_t *size = static_cast( + it->impl->get_untyped_result(true,NULL,true/*internal*/)); + hasher.hash(*size); + } + for (std::vector::const_iterator it = + resulting_fields.begin(); it != resulting_fields.end(); it++) + hasher.hash(*it); + hasher.hash(local); + hasher.hash(serdez_id); + verify_replicable(hasher, "allocate_fields"); + } + if (local) + REPORT_LEGION_FATAL(LEGION_FATAL_UNIMPLEMENTED_FEATURE, + "Local field creation is not currently supported " + "for control replication with task %s (UID %lld)", + get_task_name(), get_unique_id()) + for (unsigned idx = 0; idx < resulting_fields.size(); idx++) + { + if (resulting_fields[idx] == LEGION_AUTO_GENERATE_ID) + { + if (pending_fields.empty()) + increase_pending_fields(1/*count*/, false/*double*/); + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_fields.front(); + if (collective.second) + { + const FIDBroadcast value = collective.first->get_value(false); + resulting_fields[idx] = value.field_id; + double_buffer = value.double_buffer; + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const FIDBroadcast value = collective.first->get_value(false); + resulting_fields[idx] = value.field_id; + double_buffer = value.double_buffer; + } + delete collective.first; + pending_fields.pop_front(); + increase_pending_fields(double_buffer ? pending_fields.size() + 1 : 1, + double_next && !double_buffer); + } +#ifdef DEBUG_LEGION + else if (resulting_fields[idx] >= LEGION_MAX_APPLICATION_FIELD_ID) + REPORT_LEGION_ERROR(ERROR_TASK_ATTEMPTED_ALLOCATE_FIELD, + "Task %s (ID %lld) attempted to allocate a field with " + "ID %d which exceeds the LEGION_MAX_APPLICATION_FIELD_ID " + "bound set in legion_config.h", get_task_name(), + get_unique_id(), resulting_fields[idx]) +#endif + } + for (unsigned idx = 0; idx < sizes.size(); idx++) + if (sizes[idx].impl == NULL) + REPORT_LEGION_ERROR(ERROR_REQUEST_FOR_EMPTY_FUTURE, + "Invalid empty future passed to field allocation for field %d " + "in task %s (UID %lld)", resulting_fields[idx], + get_task_name(), get_unique_id()) + std::map >::const_iterator finder = + field_allocator_owner_shards.find(space); +#ifdef DEBUG_LEGION + assert(finder != field_allocator_owner_shards.end()); +#endif + // Get a new creation operation + CreationOp *creator_op = runtime->get_available_creation_op(); + // This deduplicates multiple shards on the same node + if (finder->second.second) + { + const ApEvent ready = creator_op->get_completion_event(); + const bool owner = (finder->second.first == owner_shard->shard_id); + RtEvent precondition; + FieldSpaceNode *node = runtime->forest->allocate_fields(space, ready, + resulting_fields, serdez_id, precondition, !owner); + Runtime::phase_barrier_arrive(creation_barrier,1/*count*/,precondition); + creator_op->initialize_fields(this, node, resulting_fields, + sizes, precondition, owner); + } + else + { + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + creator_op->initialize_fence(this, creation_barrier); + } + // Launch the creation op in this context to act as a fence to ensure + // that the allocations are done on all shard nodes before anyone else + // tries to use them or their meta-data + add_to_dependence_queue(creator_op); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + register_all_field_creations(space, local, resulting_fields); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::free_fields(FieldAllocatorImpl *allocator, + FieldSpace space, + const std::set &to_free, + const bool unordered) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication && !unordered) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_FREE_FIELDS); + hasher.hash(space); + for (std::set::const_iterator it = + to_free.begin(); it != to_free.end(); it++) + hasher.hash(*it); + verify_replicable(hasher, "free_fields"); + } + std::set free_now; + { + AutoLock priv_lock(privilege_lock,1,false/*exclusive*/); + // These fields will actually be removed in analyze_destroy_fields + for (std::set::const_iterator it = + to_free.begin(); it != to_free.end(); it++) + { + const std::pair key(space, *it); + std::set >::const_iterator finder = + created_fields.find(key); + if (finder == created_fields.end()) + { + std::map,bool>::iterator + local_finder = local_fields.find(key); + if (local_finder != local_fields.end()) + { + local_finder->second = true; + free_now.insert(*it); + } + else + deleted_fields.push_back(key); + } + else + free_now.insert(*it); + } + } + if (free_now.empty()) + return; + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_field_deletions(this, space, free_now, + unordered, allocator); + op->initialize_replication(this, deletion_ready_barrier, + deletion_mapping_barrier, deletion_execution_barrier, + shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + add_to_dependence_queue(op, unordered); + } + + //-------------------------------------------------------------------------- + LogicalRegion ReplicateContext::create_logical_region( + RegionTreeForest *forest, + IndexSpace index_space, + FieldSpace field_space, + bool task_local) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_LOGICAL_REGION); + hasher.hash(index_space); + hasher.hash(field_space); + hasher.hash(task_local); + verify_replicable(hasher, "create_logical_region"); + } + // Seed this with the first field space broadcast + if (pending_region_trees.empty()) + increase_pending_region_trees(1/*count*/, false/*double*/); + LogicalRegion handle(0/*temp*/, index_space, field_space); + bool double_next = false; + bool double_buffer = false; + std::pair*,bool> &collective = + pending_region_trees.front(); + if (collective.second) + { + const LRBroadcast value = collective.first->get_value(false); + handle.tree_id = value.tid; + double_buffer = value.double_buffer; + std::set applied; + // Have to register this before doing the broadcast + RegionNode *node = + forest->create_logical_region(handle, false/*notify remote*/, + creation_barrier, &applied); + // Now we can update the creation set + node->update_creation_set(shard_manager->get_mapping()); + // Arrive on the creation barrier + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + runtime->forest->revoke_pending_region_tree(value.tid); +#ifdef DEBUG_LEGION + log_region.debug("Creating logical region in task %s (ID %lld) with " + "index space %x and field space %x in new tree %d", + get_task_name(), get_unique_id(), index_space.id, + field_space.id, handle.tree_id); +#endif + if (runtime->legion_spy_enabled) + LegionSpy::log_top_region(index_space.id, field_space.id, + handle.tree_id); + } + else + { + const RtEvent done = collective.first->get_done_event(); + if (!done.has_triggered()) + { + double_next = true; + done.wait(); + } + const LRBroadcast value = collective.first->get_value(false); + handle.tree_id = value.tid; + double_buffer = value.double_buffer; +#ifdef DEBUG_LEGION + assert(handle.exists()); +#endif + std::set applied; + forest->create_logical_region(handle, false/*notify remote*/, + creation_barrier, &applied); + // Signal that we are done our creation + if (!applied.empty()) + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/, + Runtime::merge_events(applied)); + else + Runtime::phase_barrier_arrive(creation_barrier, 1/*count*/); + } + delete collective.first; + pending_region_trees.pop_front(); + // Advance the creation barrier so that we know when it is ready + advance_replicate_barrier(creation_barrier, total_shards); + // Register the creation of a top-level region with the context + register_region_creation(handle, task_local); + // Get new handles in flight for the next time we need them + // Always add a new one to replace the old one, but double the number + // in flight if we're not hiding the latency + increase_pending_region_trees(double_buffer ? + pending_region_trees.size() + 1 : 1, double_next && !double_buffer); + return handle; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::increase_pending_region_trees(unsigned count, + bool double_next) + //-------------------------------------------------------------------------- + { + for (unsigned idx = 0; idx < count; idx++) + { + if (owner_shard->shard_id == logical_region_allocator_shard) + { + const RegionTreeID tid = runtime->get_unique_region_tree_id(); + // We're the owner, so make it locally and then broadcast it + runtime->forest->record_pending_region_tree(tid); + // Do our arrival on this generation, should be the last one + ValueBroadcast *collective = + new ValueBroadcast(this, COLLECTIVE_LOC_34); + collective->broadcast(LRBroadcast(tid, double_next)); + pending_region_trees.push_back( + std::pair*,bool>(collective, true)); + } + else + { + ValueBroadcast *collective = + new ValueBroadcast(this,logical_region_allocator_shard, + COLLECTIVE_LOC_34); + register_collective(collective); + pending_region_trees.push_back( + std::pair*,bool>(collective, false)); + } + logical_region_allocator_shard++; + if (logical_region_allocator_shard == total_shards) + logical_region_allocator_shard = 0; + double_next = false; + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::create_shared_ownership(LogicalRegion handle) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_SHARED_OWNERSHIP); + hasher.hash(handle); + verify_replicable(hasher, "create_shared_ownership"); + } + if (!handle.exists()) + return; + if (!runtime->forest->is_top_level_region(handle)) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARED_OWNERSHIP, + "Illegal call to create shared ownership for logical region " + "(%x,%x,%x in task %s (UID %lld) which is not a top-level logical " + "region. Legion only permits top-level logical regions to have " + "shared ownerships.", handle.index_space.id, handle.field_space.id, + handle.tree_id, get_task_name(), get_unique_id()) + if (shard_manager->is_total_sharding() && + shard_manager->is_first_local_shard(owner_shard)) + runtime->create_shared_ownership(handle, true/*total sharding*/); + else if (owner_shard->shard_id == 0) + runtime->create_shared_ownership(handle); + AutoLock priv_lock(privilege_lock); + std::map::iterator finder = + created_regions.find(handle); + if (finder != created_regions.end()) + finder->second++; + else + created_regions[handle] = 1; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::destroy_logical_region(LogicalRegion handle, + const bool unordered) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication && !unordered) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_DESTROY_LOGICAL_REGION); + hasher.hash(handle); + verify_replicable(hasher, "destroy_logical_region"); + } + if (!handle.exists()) + return; +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_region.debug("Deleting logical region (%x,%x) in task %s (ID %lld)", + handle.index_space.id, handle.field_space.id, + get_task_name(), get_unique_id()); +#endif + // Check to see if this is a top-level logical region, if not then + // we shouldn't even be destroying it + if (!runtime->forest->is_top_level_region(handle)) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_RESOURCE_DESTRUCTION, + "Illegal call to destroy logical region (%x,%x,%x in task %s " + "(UID %lld) which is not a top-level logical region. Legion only " + "permits top-level logical regions to be destroyed.", + handle.index_space.id, handle.field_space.id, handle.tree_id, + get_task_name(), get_unique_id()) + // Check to see if this is one that we should be allowed to destory + { + AutoLock priv_lock(privilege_lock,1,false/*exclusive*/); + std::map::iterator finder = + created_regions.find(handle); + if (finder == created_regions.end()) + { + // Check to see if it is a local region + std::map::iterator local_finder = + local_regions.find(handle); + // Mark that this region is deleted, safe even though this + // is a read-only lock because we're not changing the structure + // of the map + if (local_finder == local_regions.end()) + { + // Record the deletion for later and propagate it up + deleted_regions.push_back(handle); + return; + } + else + local_finder->second = true; + } + else + { + if (finder->second == 0) + { + REPORT_LEGION_WARNING(LEGION_WARNING_DUPLICATE_DELETION, + "Duplicate deletions were performed for region (%x,%x,%x) " + "in task tree rooted by %s", handle.index_space.id, + handle.field_space.id, handle.tree_id, get_task_name()) + return; + } + if (--finder->second > 0) + return; + // Don't remove anything from created regions yet, we still might + // need it as part of the logical dependence analysis for earlier + // operations, but the reference count is zero so we're protected + } + } + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_logical_region_deletion(this, handle, unordered); + op->initialize_replication(this, deletion_ready_barrier, + deletion_mapping_barrier, deletion_execution_barrier, + shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + add_to_dependence_queue(op, unordered); + } + + //-------------------------------------------------------------------------- + FieldAllocatorImpl* ReplicateContext::create_field_allocator( + FieldSpace handle, bool unordered) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_CREATE_FIELD_ALLOCATOR); + hasher.hash(handle); + verify_replicable(hasher, "create_field_allocator"); + } + { + AutoLock priv_lock(privilege_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + field_allocators.find(handle); + if (finder != field_allocators.end()) + return finder->second; + } + // Didn't find it, so have to make, retake the lock in exclusive mode + AutoLock priv_lock(privilege_lock); + // Check to see if we lost the race + std::map::const_iterator finder = + field_allocators.find(handle); + if (finder != field_allocators.end()) + return finder->second; + // Check to see which shard (if any) owns this field space + const AddressSpaceID owner_space = + FieldSpaceNode::get_owner_space(handle, runtime); + // Figure out which shard is the owner + bool found = false; + std::pair owner(0,false); + const ShardMapping &mapping = shard_manager->get_mapping(); + for (unsigned idx = 0; idx < mapping.size(); idx++) + { + if (mapping[idx] != owner_space) + continue; + owner.first = idx; + found = true; + break; + } + // Pick a shard to be the owner if we don't have a local shard + if (!found) + { + if (unordered) + { + // This next part is unsafe to perform in a control replicated + // context if we are unordered, so just make a fresh allocator + const RtEvent ready = + runtime->forest->create_field_space_allocator(handle); + // Don't have one so make a new one + FieldAllocatorImpl *result = + new FieldAllocatorImpl(handle, NULL, ready); + // DO NOT SAVE THIS! + return result; + } + owner.first = field_allocator_shard++; + if (field_allocator_shard == total_shards) + field_allocator_shard = 0; + } + if (owner_space == runtime->address_space) + owner.second = (owner.first == owner_shard->shard_id); + else + owner.second = shard_manager->is_first_local_shard(owner_shard); +#ifdef DEBUG_LEGION + assert(field_allocator_owner_shards.find(handle) == + field_allocator_owner_shards.end()); +#endif + field_allocator_owner_shards[handle] = owner; + RtEvent ready; + if (owner.second) + ready = runtime->forest->create_field_space_allocator(handle, + true/*sharded context*/, (owner.first == owner_shard->shard_id)); + // Don't have one so make a new one + FieldAllocatorImpl *result = new FieldAllocatorImpl(handle, this, ready); + // Save it for later + field_allocators[handle] = result; + return result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::destroy_field_allocator(FieldSpace handle) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (runtime->safe_control_replication) + { + Murmur3Hasher hasher; + hasher.hash(REPLICATE_DESTROY_FIELD_ALLOCATOR); + hasher.hash(handle); + verify_replicable(hasher, "destroy_field_allocator"); + } + bool found = false; + std::pair result; + { + AutoLock priv_lock(privilege_lock); + // Check to see if we still have one + std::map::iterator finder = + field_allocators.find(handle); + if (finder != field_allocators.end()) + { + found = true; + field_allocators.erase(finder); + std::map >::iterator owner_finder = + field_allocator_owner_shards.find(handle); +#ifdef DEBUG_LEGION + assert(owner_finder != field_allocator_owner_shards.end()); +#endif + result = owner_finder->second; + + field_allocator_owner_shards.erase(owner_finder); + } + } + if (found) + { + if (result.second) + runtime->forest->destroy_field_space_allocator(handle, + true/*sharded*/, (result.first == owner_shard->shard_id)); + } + else + runtime->forest->destroy_field_space_allocator(handle); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::insert_unordered_ops(AutoLock &d_lock, + const bool end_task, const bool progress) + //-------------------------------------------------------------------------- + { + // If we have a trace then we're definitely not inserting operations + if (current_trace != NULL) + return; + // For control replication, we need to have an algorithm to determine + // when the shards try to sync up to insert operations that doesn't + // rely on knowing if or when any one shard has unordered ops + // We employ a sampling based algorithm here with exponential backoff + // to detect when we are doing unordered ops since it's likely a + // binary state where either we are or we aren't doing unordered ops + if (!end_task) + { +#ifdef DEBUG_LEGION + assert(unordered_ops_counter < unordered_ops_epoch); +#endif + // If we're doing progress then we can skip this check and + // reset the counter back to zero since we're doing an exchange + if (!progress) + { + if (++unordered_ops_counter < unordered_ops_epoch) + return; + } + else + unordered_ops_counter = 0; + } + // If we're at the end of the task and we don't have any unordered ops + // then nobody else should have any either so we are done + else if (unordered_ops.empty()) + return; + // If we make it here then all the shards are agreed that they are + // going to do the sync up and will exchange information + // We're going to release the lock so we need to grab a local copy + // of all our unordered operations. + std::list local_unordered; + local_unordered.swap(unordered_ops); + // Now we can release the lock and do the exchange + d_lock.release(); + UnorderedExchange exchange(this, COLLECTIVE_LOC_88); + std::vector ready_ops; + const bool any_unordered_ops = + exchange.exchange_unordered_ops(local_unordered, ready_ops); + // Reacquire the lock and handle the operations + d_lock.reacquire(); + if (!ready_ops.empty()) + { + for (std::vector::const_iterator it = + ready_ops.begin(); it != ready_ops.end(); it++) + { + (*it)->set_tracking_parent(total_children_count++); + dependence_queue.push_back(*it); + } + __sync_fetch_and_add(&outstanding_children_count, ready_ops.size()); + if (ready_ops.size() != local_unordered.size()) + { + // For any operations which we aren't in the ready ops + // then we need to put them back on the unordered list + for (std::list::const_reverse_iterator it = + local_unordered.rbegin(); it != local_unordered.rend(); it++) + { + bool found = false; + for (unsigned idx = 0; idx < ready_ops.size(); idx++) + { + if (ready_ops[idx] != (*it)) + continue; + found = true; + break; + } + if (!found) + unordered_ops.push_front(*it); + } + } + } + else if (!local_unordered.empty()) + { + // Put all our of items back on the unordered list + if (!unordered_ops.empty()) + unordered_ops.insert(unordered_ops.begin(), + local_unordered.begin(), local_unordered.end()); + else + unordered_ops.swap(local_unordered); + } + if (!end_task) + { + // Reset the count + unordered_ops_counter = 0; + // Check to see how to adjust the epoch size + if (!any_unordered_ops) + { + // If there were no ready unordered ops then we double the epoch + if (unordered_ops_epoch < MAX_UNORDERED_OPS_EPOCH) + unordered_ops_epoch *= 2; + } + else // Otherwise reset to min epoch size + unordered_ops_epoch = MIN_UNORDERED_OPS_EPOCH; +#ifdef DEBUG_LEGION + assert(MIN_UNORDERED_OPS_EPOCH <= unordered_ops_epoch); + assert(unordered_ops_epoch <= MAX_UNORDERED_OPS_EPOCH); +#endif + } + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::execute_task(const TaskLauncher &launcher) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + // Quick out for predicate false + if (launcher.predicate == Predicate::FALSE_PRED) + { + if (launcher.predicate_false_future.impl != NULL) + return launcher.predicate_false_future; + // Otherwise check to see if we have a value + FutureImpl *result = new FutureImpl(runtime, true/*register*/, + runtime->get_available_distributed_id(), + runtime->address_space, ApEvent::NO_AP_EVENT); + if (launcher.predicate_false_result.get_size() > 0) + result->set_result(launcher.predicate_false_result.get_ptr(), + launcher.predicate_false_result.get_size(), + false/*own*/); + else + { + // We need to check to make sure that the task actually + // does expect to have a void return type + TaskImpl *impl = runtime->find_or_create_task_impl(launcher.task_id); + if (impl->returns_value()) + REPORT_LEGION_ERROR(ERROR_MISSING_DEFAULT_PREDICATE_RESULT, + "Predicated task launch for task %s in parent " + "task %s (UID %lld) has non-void return type " + "but no default value for its future if the task " + "predicate evaluates to false. Please set either " + "the 'predicate_false_result' or " + "'predicate_false_future' fields of the " + "TaskLauncher struct.", impl->get_name(), + get_task_name(), get_unique_id()) + result->set_result(NULL, 0, false/*own*/); + } + return Future(result); + } + // If we're doing a local-function task then we can run that with just + // a normal individual task in each shard since it is safe to duplicate + if (launcher.local_function_task) + return InnerContext::execute_task(launcher); + ReplIndividualTask *task = + runtime->get_available_repl_individual_task(); + Future result = task->initialize_task(this, launcher); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_task.debug("Registering new single task with unique id %lld " + "and task %s (ID %lld) with high level runtime in " + "addresss space %d", + task->get_unique_id(), task->get_task_name(), + task->get_unique_id(), runtime->address_space); + task->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_43)); +#endif + // Now initialize the particular information for replication + task->initialize_replication(this); + if (launcher.enable_inlining && !launcher.silence_warnings) + REPORT_LEGION_WARNING(LEGION_WARNING_INLINING_NOT_SUPPORTED, + "Inlining is not currently supported for replicated tasks " + "such as %s (UID %lld)", get_task_name(), get_unique_id()) + execute_task_launch(task, false/*index*/, current_trace, + launcher.silence_warnings, false/*no inlining*/); + return result; + } + + //-------------------------------------------------------------------------- + FutureMap ReplicateContext::execute_index_space( + const IndexTaskLauncher &launcher) + //-------------------------------------------------------------------------- + { + if (launcher.must_parallelism) + { + // Turn around and use a must epoch launcher + MustEpochLauncher epoch_launcher(launcher.map_id, launcher.tag); + epoch_launcher.add_index_task(launcher); + FutureMap result = execute_must_epoch(epoch_launcher); + return result; + } + AutoRuntimeCall call(this); + if (launcher.launch_domain.exists() && + (launcher.launch_domain.get_volume() == 0)) + { + log_run.warning("Ignoring empty index task launch in task %s (ID %lld)", + get_task_name(), get_unique_id()); + return FutureMap(); + } + // Quick out for predicate false + if (launcher.predicate == Predicate::FALSE_PRED) + { + Domain launch_domain = launcher.launch_domain; + if (!launch_domain.exists()) + runtime->forest->find_launch_space_domain(launcher.launch_space, + launch_domain); + FutureMapImpl *result = new FutureMapImpl(this, runtime, + launch_domain, runtime->get_available_distributed_id(), + runtime->address_space, RtEvent::NO_RT_EVENT); + if (launcher.predicate_false_future.impl != NULL) + { + ApEvent ready_event = + launcher.predicate_false_future.impl->get_ready_event(); + if (ready_event.has_triggered()) + { + const void *f_result = + launcher.predicate_false_future.impl->get_untyped_result(); + size_t f_result_size = + launcher.predicate_false_future.impl->get_untyped_size(); + for (Domain::DomainPointIterator itr(launcher.launch_domain); + itr; itr++) + { + Future f = result->get_future(itr.p, true/*internal*/); + f.impl->set_result(f_result, f_result_size, false/*own*/); + } + } + else + { + // Otherwise launch a task to complete the future map, + // add the necessary references to prevent premature + // garbage collection by the runtime + result->add_base_gc_ref(DEFERRED_TASK_REF); + launcher.predicate_false_future.impl->add_base_gc_ref( + FUTURE_HANDLE_REF); + TaskOp::DeferredFutureMapSetArgs args(result, + launcher.predicate_false_future.impl, + launcher.launch_domain, owner_task); + runtime->issue_runtime_meta_task(args, LG_LATENCY_DEFERRED_PRIORITY, + Runtime::protect_event(ready_event)); + } + return FutureMap(result); + } + if (launcher.predicate_false_result.get_size() == 0) + { + // Check to make sure the task actually does expect to + // have a void return type + TaskImpl *impl = runtime->find_or_create_task_impl(launcher.task_id); + if (impl->returns_value()) + REPORT_LEGION_ERROR(ERROR_MISSING_DEFAULT_PREDICATE_RESULT, + "Predicated index task launch for task %s " + "in parent task %s (UID %lld) has non-void " + "return type but no default value for its " + "future if the task predicate evaluates to " + "false. Please set either the " + "'predicate_false_result' or " + "'predicate_false_future' fields of the " + "IndexTaskLauncher struct.", impl->get_name(), + get_task_name(), get_unique_id()) + // Just initialize all the futures + for (Domain::DomainPointIterator itr(launcher.launch_domain); + itr; itr++) + result->get_future(itr.p, true/*internal*/); + } + else + { + const void *ptr = launcher.predicate_false_result.get_ptr(); + size_t ptr_size = launcher.predicate_false_result.get_size(); + for (Domain::DomainPointIterator itr(launcher.launch_domain); + itr; itr++) + { + Future f = result->get_future(itr.p, true/*internal*/); + f.impl->set_result(ptr, ptr_size, false/*own*/); + } + } + return FutureMap(result); + } + IndexSpace launch_space = launcher.launch_space; + if (!launch_space.exists()) + launch_space = find_index_launch_space(launcher.launch_domain); + ReplIndexTask *task = runtime->get_available_repl_index_task(); + FutureMap result = task->initialize_task(this, launcher, launch_space); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_task.debug("Registering new index space task with unique id " + "%lld and task %s (ID %lld) with high level runtime in " + "address space %d", + task->get_unique_id(), task->get_task_name(), + task->get_unique_id(), runtime->address_space); + task->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_44)); +#endif + task->initialize_replication(this); + if (launcher.enable_inlining && !launcher.silence_warnings) + REPORT_LEGION_WARNING(LEGION_WARNING_INLINING_NOT_SUPPORTED, + "Inlining is not currently supported for replicated tasks " + "such as %s (UID %lld)", get_task_name(), get_unique_id()) + execute_task_launch(task, true/*index*/, current_trace, + launcher.silence_warnings, false/*no inlining*/); + return result; + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::execute_index_space( + const IndexTaskLauncher &launcher, ReductionOpID redop, bool deterministic) + //-------------------------------------------------------------------------- + { + if (launcher.must_parallelism) + REPORT_LEGION_FATAL(LEGION_FATAL_UNIMPLEMENTED_FEATURE, + "Task %s (UID %lld) requested an index space launch with must " + "parallelism (aka a MustEpochLaunch) that needs a reduction of " + "all future values. This feature is not currently implemented.", + get_task_name(), get_unique_id()) + AutoRuntimeCall call(this); + // Quick out for predicate false + if (launcher.predicate == Predicate::FALSE_PRED) + { + if (launcher.predicate_false_future.impl != NULL) + return launcher.predicate_false_future; + // Otherwise check to see if we have a value + FutureImpl *result = new FutureImpl(runtime, true/*register*/, + runtime->get_available_distributed_id(), + runtime->address_space, ApEvent::NO_AP_EVENT); + if (launcher.predicate_false_result.get_size() > 0) + result->set_result(launcher.predicate_false_result.get_ptr(), + launcher.predicate_false_result.get_size(), + false/*own*/); + else + { + // We need to check to make sure that the task actually + // does expect to have a void return type + TaskImpl *impl = runtime->find_or_create_task_impl(launcher.task_id); + if (impl->returns_value()) + REPORT_LEGION_ERROR(ERROR_MISSING_DEFAULT_PREDICATE_RESULT, + "Predicated index task launch for task %s " + "in parent task %s (UID %lld) has non-void " + "return type but no default value for its " + "future if the task predicate evaluates to " + "false. Please set either the " + "'predicate_false_result' or " + "'predicate_false_future' fields of the " + "IndexTaskLauncher struct.", impl->get_name(), + get_task_name(), get_unique_id()) + result->set_result(NULL, 0, false/*own*/); + } + return Future(result); + } + if (launcher.launch_domain.exists() && + (launcher.launch_domain.get_volume() == 0)) + { + log_run.warning("Ignoring empty index task launch in task %s (ID %lld)", + get_task_name(), get_unique_id()); + return Future(); + } + IndexSpace launch_space = launcher.launch_space; + if (!launch_space.exists()) + launch_space = find_index_launch_space(launcher.launch_domain); + ReplIndexTask *task = runtime->get_available_repl_index_task(); + Future result = task->initialize_task(this, launcher, launch_space, + redop, deterministic); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_task.debug("Registering new index space task with unique id " + "%lld and task %s (ID %lld) with high level runtime in " + "address space %d", + task->get_unique_id(), task->get_task_name(), + task->get_unique_id(), runtime->address_space); + task->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_45)); +#endif + task->initialize_replication(this); + if (launcher.enable_inlining && !launcher.silence_warnings) + REPORT_LEGION_WARNING(LEGION_WARNING_INLINING_NOT_SUPPORTED, + "Inlining is not currently supported for replicated tasks " + "such as %s (UID %lld)", get_task_name(), get_unique_id()) + execute_task_launch(task, true/*index*/, current_trace, + launcher.silence_warnings, false/*no inlining*/); + return result; + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::reduce_future_map(const FutureMap &future_map, + ReductionOpID redop, bool deterministic) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (future_map.impl == NULL) + return Future(); + // Check to see if this is just a normal future map, if so then + // we can just do the standard thing here + if (!future_map.impl->is_replicate_future_map()) + return InnerContext::reduce_future_map(future_map,redop,deterministic); + ReplAllReduceOp *all_reduce_op = + runtime->get_available_repl_all_reduce_op(); + Future result = + all_reduce_op->initialize(this, future_map, redop, deterministic); + all_reduce_op->initialize_replication(this); + add_to_dependence_queue(all_reduce_op); + return result; + } + + //-------------------------------------------------------------------------- + PhysicalRegion ReplicateContext::map_region(const InlineLauncher &launcher) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (IS_NO_ACCESS(launcher.requirement)) + return PhysicalRegion(); + ReplMapOp *map_op = runtime->get_available_repl_map_op(); + PhysicalRegion result = map_op->initialize(this, launcher); +#ifdef DEBUG_LEGION + log_run.debug("Registering a map operation for region " + "(%x,%x,%x) in task %s (ID %lld)", + launcher.requirement.region.index_space.id, + launcher.requirement.region.field_space.id, + launcher.requirement.region.tree_id, + get_task_name(), get_unique_id()); +#endif + map_op->initialize_replication(this, inline_mapping_barrier); + + bool parent_conflict = false, inline_conflict = false; + const int index = + has_conflicting_regions(map_op, parent_conflict, inline_conflict); + if (parent_conflict) + REPORT_LEGION_ERROR(ERROR_ATTEMPTED_INLINE_MAPPING_REGION, + "Attempted an inline mapping of region " + "(%x,%x,%x) that conflicts with mapped region " + "(%x,%x,%x) at index %d of parent task %s " + "(ID %lld) that would ultimately result in " + "deadlock. Instead you receive this error message.", + launcher.requirement.region.index_space.id, + launcher.requirement.region.field_space.id, + launcher.requirement.region.tree_id, + regions[index].region.index_space.id, + regions[index].region.field_space.id, + regions[index].region.tree_id, + index, get_task_name(), get_unique_id()) + if (inline_conflict) + REPORT_LEGION_ERROR(ERROR_ATTEMPTED_INLINE_MAPPING_REGION, + "Attempted an inline mapping of region (%x,%x,%x) " + "that conflicts with previous inline mapping in " + "task %s (ID %lld) that would ultimately result in " + "deadlock. Instead you receive this error message.", + launcher.requirement.region.index_space.id, + launcher.requirement.region.field_space.id, + launcher.requirement.region.tree_id, + get_task_name(), get_unique_id()) + register_inline_mapped_region(result); + add_to_dependence_queue(map_op); + return result; + } + + //-------------------------------------------------------------------------- + ApEvent ReplicateContext::remap_region(PhysicalRegion region) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + // Check to see if the region is already mapped, + // if it is then we are done + if (region.is_mapped()) + return ApEvent::NO_AP_EVENT; + ReplMapOp *map_op = runtime->get_available_repl_map_op(); + map_op->initialize(this, region); + map_op->initialize_replication(this, inline_mapping_barrier); + register_inline_mapped_region(region); + const ApEvent result = map_op->get_completion_event(); + add_to_dependence_queue(map_op); + return result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::fill_fields(const FillLauncher &launcher) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + ReplFillOp *fill_op = runtime->get_available_repl_fill_op(); + fill_op->initialize(this, launcher); +#ifdef DEBUG_LEGION + log_run.debug("Registering a fill operation in task %s (ID %lld)", + get_task_name(), get_unique_id()); + fill_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_51)); +#endif + fill_op->initialize_replication(this); + // Check to see if we need to do any unmappings and remappings + // before we can issue this copy operation + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(fill_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings && !launcher.silence_warnings) + { + REPORT_LEGION_WARNING(LEGION_WARNING_RUNTIME_UNMAPPING_REMAPPING, + "WARNING: Runtime is unmapping and remapping " + "physical regions around fill_fields call in task %s (UID %lld).", + get_task_name(), get_unique_id()); + } + // Unmap any regions which are conflicting + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(fill_op); + // Remap any regions which we unmapped + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::fill_fields(const IndexFillLauncher &launcher) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (launcher.launch_domain.exists() && + (launcher.launch_domain.get_volume() == 0)) + { + log_run.warning("Ignoring empty index space fill in task %s (ID %lld)", + get_task_name(), get_unique_id()); + return; + } + IndexSpace launch_space = launcher.launch_space; + if (!launch_space.exists()) + launch_space = find_index_launch_space(launcher.launch_domain); + ReplIndexFillOp *fill_op = + runtime->get_available_repl_index_fill_op(); + fill_op->initialize(this, launcher, launch_space); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Registering an index fill operation in task %s " + "(ID %lld)", get_task_name(), get_unique_id()); + fill_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_46)); +#endif + fill_op->initialize_replication(this); + // Check to see if we need to do any unmappings and remappings + // before we can issue this copy operation + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(fill_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings && !launcher.silence_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around fill_fields call in task %s (UID %lld).", + get_task_name(), get_unique_id()); + // Unmap any regions which are conflicting + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(fill_op); + // Remap any regions which we unmapped + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::issue_copy(const CopyLauncher &launcher) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + ReplCopyOp *copy_op = runtime->get_available_repl_copy_op(); + copy_op->initialize(this, launcher); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Registering a copy operation in task %s (ID %lld)", + get_task_name(), get_unique_id()); + copy_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_47)); +#endif + copy_op->initialize_replication(this); + // Check to see if we need to do any unmappings and remappings + // before we can issue this copy operation + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(copy_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings && !launcher.silence_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around issue_copy_operation call in " + "task %s (UID %lld).", get_task_name(), get_unique_id()); + // Unmap any regions which are conflicting + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(copy_op); + // Remap any regions which we unmapped + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::issue_copy(const IndexCopyLauncher &launcher) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (launcher.launch_domain.exists() && + (launcher.launch_domain.get_volume() == 0)) + { + log_run.warning("Ignoring empty index space copy in task %s " + "(ID %lld)", get_task_name(), get_unique_id()); + return; + } + IndexSpace launch_space = launcher.launch_space; + if (!launch_space.exists()) + launch_space = find_index_launch_space(launcher.launch_domain); + ReplIndexCopyOp *copy_op = + runtime->get_available_repl_index_copy_op(); + copy_op->initialize(this, launcher, launch_space); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Registering an index copy operation in task %s " + "(ID %lld)", get_task_name(), get_unique_id()); + copy_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_48)); +#endif + copy_op->initialize_replication(this, indirection_barriers, + next_indirection_bar_index); + // Check to see if we need to do any unmappings and remappings + // before we can issue this copy operation + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + find_conflicting_regions(copy_op, unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings && !launcher.silence_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around issue_copy_operation call in " + "task %s (UID %lld).", get_task_name(), get_unique_id()); + // Unmap any regions which are conflicting + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Issue the copy operation + add_to_dependence_queue(copy_op); + // Remap any regions which we unmapped + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::issue_acquire(const AcquireLauncher &launcher) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_REPLICATE_TASK_VIOLATION, + "Acquire operations are not currently supported in control " + "replication contexts for task %s (UID %lld). It may be " + "supported in the future.", + get_task_name(), get_unique_id()) + } + + //-------------------------------------------------------------------------- + void ReplicateContext::issue_release(const ReleaseLauncher &launcher) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_REPLICATE_TASK_VIOLATION, + "Release operations are not currently supported in control " + "replication contexts for task %s (UID %lld). It may be " + "supported in the future.", + get_task_name(), get_unique_id()) + } + + //-------------------------------------------------------------------------- + PhysicalRegion ReplicateContext::attach_resource( + const AttachLauncher &launcher) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + if (launcher.restricted) + REPORT_LEGION_ERROR(ERROR_REPLICATE_TASK_VIOLATION, + "Attach operations in control replication context %s (UID %lld) " + "requested a restriction. Restrictions are only permitted for " + "attach operations in non-control-replicated contexts currently.", + get_task_name(), get_unique_id()); + ReplAttachOp *attach_op = runtime->get_available_repl_attach_op(); + PhysicalRegion result = attach_op->initialize(this, launcher); + attach_op->initialize_replication(this, external_resource_barrier, + attach_broadcast_barrier, attach_reduce_barrier); + + bool parent_conflict = false, inline_conflict = false; + int index = has_conflicting_regions(attach_op, + parent_conflict, inline_conflict); + if (parent_conflict) + REPORT_LEGION_ERROR(ERROR_ATTEMPTED_ATTACH_HDF5, + "Attempted an attach hdf5 file operation on region " + "(%x,%x,%x) that conflicts with mapped region " + "(%x,%x,%x) at index %d of parent task %s (ID %lld) " + "that would ultimately result in deadlock. Instead you " + "receive this error message. Try unmapping the region " + "before invoking attach_hdf5 on file %s", + launcher.handle.index_space.id, + launcher.handle.field_space.id, + launcher.handle.tree_id, + regions[index].region.index_space.id, + regions[index].region.field_space.id, + regions[index].region.tree_id, index, + get_task_name(), get_unique_id(), launcher.file_name) + if (inline_conflict) + REPORT_LEGION_ERROR(ERROR_ATTEMPTED_ATTACH_HDF5, + "Attempted an attach hdf5 file operation on region " + "(%x,%x,%x) that conflicts with previous inline " + "mapping in task %s (ID %lld) " + "that would ultimately result in deadlock. Instead you " + "receive this error message. Try unmapping the region " + "before invoking attach_hdf5 on file %s", + launcher.handle.index_space.id, + launcher.handle.field_space.id, + launcher.handle.tree_id, get_task_name(), + get_unique_id(), launcher.file_name) + // If we're counting this region as mapped we need to register it + if (launcher.mapped) + register_inline_mapped_region(result); + add_to_dependence_queue(attach_op); + return result; + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::detach_resource(PhysicalRegion region, + const bool flush, const bool unordered) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); + ReplDetachOp *op = runtime->get_available_repl_detach_op(); + Future result = op->initialize_detach(this, region, flush, unordered); + op->initialize_replication(this, external_resource_barrier); + // If the region is still mapped, then unmap it + if (region.is_mapped()) + { + unregister_inline_mapped_region(region); + region.impl->unmap_region(); + } + add_to_dependence_queue(op, unordered); + return result; + } + + //-------------------------------------------------------------------------- + FutureMap ReplicateContext::execute_must_epoch( + const MustEpochLauncher &launcher) + //-------------------------------------------------------------------------- + { +#ifdef SAFE_MUST_EPOCH_LAUNCHES + // See the comment in InnerContext::execute_must_epoch for why this + // particular call is here for safe must epoch launches + // Also see github issue #659 + issue_execution_fence(); +#endif + AutoRuntimeCall call(this); + ReplMustEpochOp *epoch_op = runtime->get_available_repl_epoch_op(); + FutureMap result = epoch_op->initialize(this, launcher); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Executing a must epoch in task %s (ID %lld)", + get_task_name(), get_unique_id()); + epoch_op->set_sharding_collective(new ShardingGatherCollective(this, + 0/*owner shard*/, COLLECTIVE_LOC_49)); +#endif + epoch_op->initialize_replication(this); + // Now find all the parent task regions we need to invalidate + std::vector unmapped_regions; + if (!runtime->unsafe_launch) + epoch_op->find_conflicted_regions(unmapped_regions); + if (!unmapped_regions.empty()) + { + if (runtime->runtime_warnings && !launcher.silence_warnings) + log_run.warning("WARNING: Runtime is unmapping and remapping " + "physical regions around issue_release call in " + "task %s (UID %lld).", get_task_name(), get_unique_id()); + for (unsigned idx = 0; idx < unmapped_regions.size(); idx++) + unmapped_regions[idx].impl->unmap_region(); + } + // Now we can issue the must epoch + add_to_dependence_queue(epoch_op); + // Remap any unmapped regions + if (!unmapped_regions.empty()) + remap_unmapped_regions(current_trace, unmapped_regions); + return result; + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::issue_timing_measurement( + const TimingLauncher &launcher) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Issuing a timing measurement in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + ReplTimingOp *timing_op = runtime->get_available_repl_timing_op(); + Future result = timing_op->initialize(this, launcher); + ValueBroadcast *timing_collective = + new ValueBroadcast(this, 0/*shard 0 is always the owner*/, + COLLECTIVE_LOC_35); + timing_op->set_timing_collective(timing_collective); + add_to_dependence_queue(timing_op); + return result; + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::issue_mapping_fence(void) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Issuing a mapping fence in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + ReplFenceOp *fence_op = runtime->get_available_repl_fence_op(); + Future result = + fence_op->initialize_repl_fence(this, FenceOp::MAPPING_FENCE, true); + add_to_dependence_queue(fence_op); + return result; + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::issue_execution_fence(void) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Issuing an execution fence in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + ReplFenceOp *fence_op = runtime->get_available_repl_fence_op(); + Future result = + fence_op->initialize_repl_fence(this, FenceOp::EXECUTION_FENCE, true); + add_to_dependence_queue(fence_op); + return result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::begin_trace(TraceID tid, bool logical_only, + bool static_trace, const std::set *trees, bool deprecated) + //-------------------------------------------------------------------------- + { + if (runtime->no_tracing) return; + if (runtime->no_physical_tracing) logical_only = true; + + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Beginning a trace in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + // No need to hold the lock here, this is only ever called + // by the one thread that is running the task. + if (current_trace != NULL) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_NESTED_TRACE, + "Illegal nested trace with ID %d attempted in " + "task %s (ID %lld)", tid, get_task_name(), get_unique_id()) + std::map::const_iterator finder = traces.find(tid); + LegionTrace *trace = NULL; + if (finder == traces.end()) + { + // Trace does not exist yet, so make one and record it + if (static_trace) + trace = new StaticTrace(tid, this, logical_only, trees); + else + trace = new DynamicTrace(tid, this, logical_only); + if (!deprecated) + traces[tid] = trace; + trace->add_reference(); + } + else + trace = finder->second; + +#ifdef DEBUG_LEGION + assert(trace != NULL); +#endif + trace->clear_blocking_call(); + + // Issue a begin op + ReplTraceBeginOp *begin = runtime->get_available_repl_begin_op(); + begin->initialize_begin(this, trace); + add_to_dependence_queue(begin); + + if (!logical_only) + { + // Issue a replay op + ReplTraceReplayOp *replay = runtime->get_available_repl_replay_op(); + replay->initialize_replay(this, trace); + add_to_dependence_queue(replay); + } + + // Now mark that we are starting a trace + current_trace = trace; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::end_trace(TraceID tid, bool deprecated) + //-------------------------------------------------------------------------- + { + if (runtime->no_tracing) return; + + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + log_run.debug("Ending a trace in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + if (current_trace == NULL) + REPORT_LEGION_ERROR(ERROR_UMATCHED_END_TRACE, + "Unmatched end trace for ID %d in task %s (ID %lld)", + tid, get_task_name(), get_unique_id()) + else if (!deprecated && (current_trace->tid != tid)) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_END_TRACE_CALL, + "Illegal end trace call on trace ID %d that does not match " + "the current trace ID %d in task %s (UID %lld)", tid, + current_trace->tid, get_task_name(), get_unique_id()) + const bool has_blocking_call = current_trace->has_blocking_call(); + if (current_trace->is_fixed()) + { + // Already fixed, dump a complete trace op into the stream + ReplTraceCompleteOp *complete_op = + runtime->get_available_repl_trace_op(); + complete_op->initialize_complete(this, has_blocking_call); + // Make a summary collective ID here if we don't have one in case + // we need to regenerate the summary barrier during the dependence + // analysis stage of the pipeline + if (current_trace->has_physical_trace() && (summary_collective_id == 0)) + summary_collective_id = get_next_collective_index(COLLECTIVE_LOC_98); + add_to_dependence_queue(complete_op); + } + else + { + // Not fixed yet, dump a capture trace op into the stream + ReplTraceCaptureOp *capture_op = + runtime->get_available_repl_capture_op(); + capture_op->initialize_capture(this, has_blocking_call, deprecated); + // Make a trace collective ID here if we don't have one in case + // we need to regenerate the trace barrier during the dependence + // analysis stage of the pipeline + if (trace_recording_collective_id == 0) + trace_recording_collective_id = + get_next_collective_index(COLLECTIVE_LOC_99); + // Mark that the current trace is now fixed + current_trace->fix_trace(); + add_to_dependence_queue(capture_op); + } + // We no longer have a trace that we're executing + current_trace = NULL; + } + + //-------------------------------------------------------------------------- + ApEvent ReplicateContext::add_to_dependence_queue(Operation *op, + bool unordered, bool outermost) + //-------------------------------------------------------------------------- + { + // We disable program order execution when we are replaying a + // fixed trace since it might not be sound to block + if (runtime->program_order_execution && !unordered && + ((current_trace == NULL) || !current_trace->is_fixed())) + { +#ifdef DEBUG_LEGION + assert(inorder_barrier.exists()); +#endif + ApEvent term_event = + InnerContext::add_to_dependence_queue(op,unordered,false/*outermost*/); + Runtime::phase_barrier_arrive(inorder_barrier, 1/*count*/, term_event); + term_event = inorder_barrier; + advance_replicate_barrier(inorder_barrier, total_shards); + if (outermost) + { + begin_task_wait(true/*from runtime*/); + term_event.wait(); + end_task_wait(); + } + return term_event; + } + else + return InnerContext::add_to_dependence_queue(op, unordered, outermost); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::record_dynamic_collective_contribution( + DynamicCollective dc, const Future &f) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_REPLICATE_TASK_VIOLATION, + "Illegal dynamic collective operation used in " + "control replicated task %s (UID %lld)", + get_task_name(), get_unique_id()) + } + + //-------------------------------------------------------------------------- + void ReplicateContext::find_collective_contributions(DynamicCollective dc, + std::vector &contributions) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_REPLICATE_TASK_VIOLATION, + "Illegal dynamic collective operation used in " + "control replicated task %s (UID %lld)", + get_task_name(), get_unique_id()) + } + + //-------------------------------------------------------------------------- + ApBarrier ReplicateContext::create_phase_barrier(unsigned arrivals, + ReductionOpID redop, + const void *init_value, + size_t init_size) + //-------------------------------------------------------------------------- + { + ValueBroadcast bar_collective(this, 0/*origin*/, + COLLECTIVE_LOC_71); + // Shard 0 will make the barrier and broadcast it + if (owner_shard->shard_id == 0) + { + ApBarrier result = InnerContext::create_phase_barrier(arrivals, redop, + init_value, init_size); + bar_collective.broadcast(result); + return result; + } + else + return bar_collective.get_value(); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::destroy_phase_barrier(ApBarrier bar) + //-------------------------------------------------------------------------- + { + // Shard 0 has to wait for all the other shards to get here + // too before it can do the deletion + ShardSyncTree sync_point(this, 0/*origin*/, COLLECTIVE_LOC_72); + if (owner_shard->shard_id == 0) + InnerContext::destroy_phase_barrier(bar); + } + + //-------------------------------------------------------------------------- + PhaseBarrier ReplicateContext::advance_phase_barrier(PhaseBarrier bar) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Advancing phase barrier in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + PhaseBarrier result = bar; + Runtime::advance_barrier(result); +#ifdef LEGION_SPY + if (owner_shard->shard_id == 0) + LegionSpy::log_event_dependence(bar.phase_barrier,result.phase_barrier); +#endif + return result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, unsigned count) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_REPLICATE_TASK_VIOLATION, + "Illegal dynamic collective arrival performed in " + "control replicated task %s (UID %lld)", + get_task_name(), get_unique_id()) + } + + //-------------------------------------------------------------------------- + void ReplicateContext::defer_dynamic_collective_arrival( + DynamicCollective dc, + const Future &f, + unsigned count) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_REPLICATE_TASK_VIOLATION, + "Illegal defer dynamic collective arrival performed in " + "control replicated task %s (UID %lld)", + get_task_name(), get_unique_id()) + } + + //-------------------------------------------------------------------------- + Future ReplicateContext::get_dynamic_collective_result(DynamicCollective dc) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_REPLICATE_TASK_VIOLATION, + "Illegal get dynamic collective result performed in " + "control replicated task %s (UID %lld)", + get_task_name(), get_unique_id()) + return Future(); + } + + //-------------------------------------------------------------------------- + DynamicCollective ReplicateContext::advance_dynamic_collective( + DynamicCollective dc) + //-------------------------------------------------------------------------- + { + AutoRuntimeCall call(this); +#ifdef DEBUG_LEGION + if (owner_shard->shard_id == 0) + log_run.debug("Advancing dynamic collective in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + DynamicCollective result = dc; + Runtime::advance_barrier(result); +#ifdef LEGION_SPY + if (owner_shard->shard_id == 0) + LegionSpy::log_event_dependence(dc.phase_barrier, result.phase_barrier); +#endif + return result; + } + + //-------------------------------------------------------------------------- +#ifdef DEBUG_LEGION_COLLECTIVES + MergeCloseOp* ReplicateContext::get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node) +#else + MergeCloseOp* ReplicateContext::get_merge_close_op(void) +#endif + //-------------------------------------------------------------------------- + { + ReplMergeCloseOp *result = runtime->get_available_repl_merge_close_op(); + // Get the mapped barrier for the close operation + const unsigned close_index = next_close_mapped_bar_index++; + if (next_close_mapped_bar_index == close_mapped_barriers.size()) + next_close_mapped_bar_index = 0; + RtBarrier &mapped_bar = close_mapped_barriers[close_index]; +#ifdef DEBUG_LEGION_COLLECTIVES + CloseCheckReduction::RHS barrier(user, mapped_bar, + node, false/*read only*/); + Runtime::phase_barrier_arrive(close_check_barrier, 1/*count*/, + RtEvent::NO_RT_EVENT, &barrier, sizeof(barrier)); + close_check_barrier.wait(); + CloseCheckReduction::RHS actual_barrier; + bool ready = Runtime::get_barrier_result(close_check_barrier, + &actual_barrier, sizeof(actual_barrier)); + assert(ready); + assert(actual_barrier == barrier); + advance_replicate_barrier(close_check_barrier, total_shards); +#endif + result->set_repl_close_info(mapped_bar); + // Advance the phase for the next time through + advance_replicate_barrier(mapped_bar, total_shards); + return result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::pack_remote_context(Serializer &rez, + AddressSpaceID target, bool replicate) + //-------------------------------------------------------------------------- + { + // Do the normal inner pack with replicate true + InnerContext::pack_remote_context(rez, target, true/*replicate*/); + // Then pack our additional information + rez.serialize(total_shards); + rez.serialize(shard_manager->repl_id); + } + + //-------------------------------------------------------------------------- + ShardingFunction* ReplicateContext::find_sharding_function(ShardingID sid) + //-------------------------------------------------------------------------- + { + return shard_manager->find_sharding_function(sid); + } + + //-------------------------------------------------------------------------- + InstanceView* ReplicateContext::create_instance_top_view( + PhysicalManager *manager, AddressSpaceID source) + //-------------------------------------------------------------------------- + { + // First do a little check to see if we already have it and if not record + // that we're the first ones requesting from this replicate context + RtEvent wait_on; + bool send_request = false; + { + AutoLock inst_lock(instance_view_lock); + std::map::const_iterator finder = + instance_top_views.find(manager); + if (finder != instance_top_views.end()) + return finder->second; + // Didn't find it, see if we need to request it or whether + // someone else is already doing that + std::map::const_iterator wait_finder = + pending_request_views.find(manager); + if (wait_finder == pending_request_views.end()) + { + RtUserEvent wait_for = Runtime::create_rt_user_event(); + pending_request_views[manager] = wait_for; + wait_on = wait_for; + send_request = true; + } + else + wait_on = wait_finder->second; + } + // Send the request if we're first + // Since we are a control replicated context we have to bounce this + // off the shard manager to find the right context to make the view + if (send_request) + shard_manager->create_instance_top_view(manager, source, this, + runtime->address_space); + // Wait for the result to be ready + wait_on.wait(); + // Retake the lock and retrieve the result + AutoLock inst_lock(instance_view_lock,1,false/*exclusive*/); +#ifdef DEBUG_LEGION + assert(instance_top_views.find(manager) != instance_top_views.end()); +#endif + return instance_top_views[manager]; + } + + //-------------------------------------------------------------------------- + InstanceView* ReplicateContext::create_replicate_instance_top_view( + PhysicalManager *manager, AddressSpaceID source) + //-------------------------------------------------------------------------- + { + // If we got picked then we can just do the base inner version + return InnerContext::create_instance_top_view(manager, source); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::record_replicate_instance_top_view( + PhysicalManager *manager, InstanceView *result) + //-------------------------------------------------------------------------- + { + // Always add the reference, we'll remove duplicates if necessary + result->add_base_resource_ref(CONTEXT_REF); + bool remove_duplicate_reference = false; + RtUserEvent to_trigger; + { + AutoLock inst_lock(instance_view_lock); + std::map::const_iterator finder = + instance_top_views.find(manager); + if (finder != instance_top_views.end()) + { +#ifdef DEBUG_LEGION + assert(finder->second == result); +#endif + remove_duplicate_reference = true; + } + else + instance_top_views[manager] = result; + // Now we find the event to trigger + std::map::iterator pending_finder = + pending_request_views.find(manager); +#ifdef DEBUG_LEGION + assert(pending_finder != pending_request_views.end()); +#endif + to_trigger = pending_finder->second; + pending_request_views.erase(pending_finder); + } + Runtime::trigger_event(to_trigger); + if (remove_duplicate_reference && + result->remove_base_resource_ref(CONTEXT_REF)) + delete result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::exchange_common_resources(void) + //-------------------------------------------------------------------------- + { + size_t num_barriers = LEGION_CONTROL_REPLICATION_COMMUNICATION_BARRIERS; + if (shard_manager->total_shards > num_barriers) + num_barriers = shard_manager->total_shards; + // Exchange close map barriers across all the shards + BarrierExchangeCollective mapped_collective(this, + num_barriers, close_mapped_barriers, COLLECTIVE_LOC_50); + mapped_collective.exchange_barriers_async(); + BarrierExchangeCollective indirect_collective(this, + num_barriers, indirection_barriers, COLLECTIVE_LOC_79); + indirect_collective.exchange_barriers_async(); + BarrierExchangeCollective future_map_collective(this, + num_barriers, future_map_barriers, COLLECTIVE_LOC_90); + future_map_collective.exchange_barriers_async(); + // Wait for everything to be done + mapped_collective.wait_for_barrier_exchange(); + indirect_collective.wait_for_barrier_exchange(); + future_map_collective.wait_for_barrier_exchange(); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::handle_collective_message(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardCollective *collective = find_or_buffer_collective(derez); + if (collective != NULL) + collective->handle_collective_message(derez); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::handle_future_map_request(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ReplFutureMapImpl *impl = find_or_buffer_future_map_request(derez); + // If impl is NULL then the request was buffered + if (impl == NULL) + return; + impl->handle_future_map_request(derez); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::handle_equivalence_set_request(Deserializer &derez) + //-------------------------------------------------------------------------- + { + RegionTreeID tree_id; + derez.deserialize(tree_id); + ReplicateContext *requester; + derez.deserialize(requester); + AddressSpaceID source; + derez.deserialize(source); + EquivalenceSet *result = find_or_create_top_equivalence_set(tree_id); + if (source != runtime->address_space) + { + Serializer rez; + { + RezCheck z(rez); + rez.serialize(result->did); + rez.serialize(tree_id); + rez.serialize(requester); + } + runtime->send_control_replicate_equivalence_set_response(source, rez); + } + else + requester->handle_equivalence_set_response(tree_id, result); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::handle_equivalence_set_response(RegionTreeID tree_id, + EquivalenceSet *result) + //-------------------------------------------------------------------------- + { + RtUserEvent to_trigger; + result->add_base_resource_ref(CONTEXT_REF); + { + AutoLock tree_lock(tree_set_lock); +#ifdef DEBUG_LEGION + assert(tree_equivalence_sets.find(tree_id) == + tree_equivalence_sets.end()); +#endif + tree_equivalence_sets[tree_id] = result; + std::map::iterator finder = + pending_tree_requests.find(tree_id); +#ifdef DEBUG_LEGION + assert(finder != pending_tree_requests.end()); +#endif + to_trigger = finder->second; + pending_tree_requests.erase(finder); + } + Runtime::trigger_event(to_trigger); + } + + //-------------------------------------------------------------------------- + /*static*/ void ReplicateContext::handle_eq_response(Deserializer &derez, + Runtime *runtime) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + DistributedID did; + derez.deserialize(did); + RtEvent ready; + EquivalenceSet *set = runtime->find_or_request_equivalence_set(did,ready); + RegionTreeID tree_id; + derez.deserialize(tree_id); + ReplicateContext *context; + derez.deserialize(context); + + if (ready.exists() && !ready.has_triggered()) + ready.wait(); + context->handle_equivalence_set_response(tree_id, set); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::handle_resource_update(Deserializer &derez, + std::set &applied) + //-------------------------------------------------------------------------- + { + size_t return_index; + derez.deserialize(return_index); + RtBarrier ready_barrier, mapped_barrier, execution_barrier; + derez.deserialize(ready_barrier); + derez.deserialize(mapped_barrier); + derez.deserialize(execution_barrier); + size_t num_created_regions; + derez.deserialize(num_created_regions); + std::map created_regs; + for (unsigned idx = 0; idx < num_created_regions; idx++) + { + LogicalRegion reg; + derez.deserialize(reg); + derez.deserialize(created_regs[reg]); + } + size_t num_deleted_regions; + derez.deserialize(num_deleted_regions); + std::vector deleted_regs(num_deleted_regions); + for (unsigned idx = 0; idx < num_deleted_regions; idx++) + derez.deserialize(deleted_regs[idx]); + size_t num_created_fields; + derez.deserialize(num_created_fields); + std::set > created_fids; + for (unsigned idx = 0; idx < num_created_fields; idx++) + { + std::pair key; + derez.deserialize(key.first); + derez.deserialize(key.second); + created_fids.insert(key); + } + size_t num_deleted_fields; + derez.deserialize(num_deleted_fields); + std::vector > + deleted_fids(num_deleted_fields); + for (unsigned idx = 0; idx < num_deleted_fields; idx++) + { + derez.deserialize(deleted_fids[idx].first); + derez.deserialize(deleted_fids[idx].second); + } + size_t num_created_field_spaces; + derez.deserialize(num_created_field_spaces); + std::map created_fs; + for (unsigned idx = 0; idx < num_created_field_spaces; idx++) + { + FieldSpace sp; + derez.deserialize(sp); + derez.deserialize(created_fs[sp]); + } + size_t num_latent_field_spaces; + derez.deserialize(num_latent_field_spaces); + std::map > latent_fs; + for (unsigned idx = 0; idx < num_latent_field_spaces; idx++) + { + FieldSpace sp; + derez.deserialize(sp); + std::set ®ions = latent_fs[sp]; + size_t num_regions; + derez.deserialize(num_regions); + for (unsigned idx2 = 0; idx2 < num_regions; idx2++) + { + LogicalRegion region; + derez.deserialize(region); + regions.insert(region); + } + } + size_t num_deleted_field_spaces; + derez.deserialize(num_deleted_field_spaces); + std::vector deleted_fs(num_deleted_field_spaces); + for (unsigned idx = 0; idx < num_deleted_field_spaces; idx++) + derez.deserialize(deleted_fs[idx]); + size_t num_created_index_spaces; + derez.deserialize(num_created_index_spaces); + std::map created_is; + for (unsigned idx = 0; idx < num_created_index_spaces; idx++) + { + IndexSpace sp; + derez.deserialize(sp); + derez.deserialize(created_is[sp]); + } + size_t num_deleted_index_spaces; + derez.deserialize(num_deleted_index_spaces); + std::vector > + deleted_is(num_deleted_index_spaces); + for (unsigned idx = 0; idx < num_deleted_index_spaces; idx++) + derez.deserialize(deleted_is[idx]); + size_t num_created_index_partitions; + derez.deserialize(num_created_index_partitions); + std::map created_partitions; + for (unsigned idx = 0; idx < num_created_index_partitions; idx++) + { + IndexPartition ip; + derez.deserialize(ip); + derez.deserialize(created_partitions[ip]); + } + size_t num_deleted_index_partitions; + derez.deserialize(num_deleted_index_partitions); + std::vector > + deleted_partitions(num_deleted_index_partitions); + for (unsigned idx = 0; idx < num_deleted_index_partitions; idx++) + derez.deserialize(deleted_partitions[idx]); + // Send this down to the base class to avoid re-broadcasting + receive_replicate_resources(return_index, created_regs, deleted_regs, + created_fids, deleted_fids, created_fs, latent_fs, deleted_fs, + created_is, deleted_is, created_partitions, deleted_partitions, + applied, ready_barrier, mapped_barrier, execution_barrier); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::handle_trace_update(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + ShardedPhysicalTemplate *tpl = find_or_buffer_trace_update(derez, source); + // If the template is NULL then the request was buffered + if (tpl == NULL) + return; + tpl->handle_trace_update(derez, source); + } + + //-------------------------------------------------------------------------- + ApBarrier ReplicateContext::handle_find_trace_shard_event( + size_t template_index, ApEvent event, ShardID remote_shard) + //-------------------------------------------------------------------------- + { + ShardedPhysicalTemplate *physical_template = NULL; + { + AutoLock r_lock(replication_lock); + std::map::const_iterator finder = + physical_templates.find(template_index); + // If we can't find the template index that means it hasn't been + // started here so it can't have produced the event we're looking for + // Note it also can't have been reclaimed yet as all the shard + // templates need to come to the same decision on whether they + // are replayable before any of them can be deleted and so if one + // is still tracing then they all are + if (finder == physical_templates.end()) + return ApBarrier::NO_AP_BARRIER; + physical_template = finder->second; + } + return physical_template->find_trace_shard_event(event, remote_shard); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::record_intra_space_dependence(size_t context_index, + const DomainPoint &point, RtEvent point_mapped, ShardID next_shard) + //-------------------------------------------------------------------------- + { + const std::pair key(context_index,point); + AutoLock r_lock(replication_lock); + IntraSpaceDeps &deps = intra_space_deps[key]; + // Check to see if someone has already registered this + std::map::iterator finder = + deps.pending_deps.find(next_shard); + if (finder != deps.pending_deps.end()) + { + Runtime::trigger_event(finder->second, point_mapped); + deps.pending_deps.erase(finder); + if (deps.pending_deps.empty() && deps.ready_deps.empty()) + intra_space_deps.erase(key); + } + else + { + // Not seen yet so just record our entry for this shard +#ifdef DEBUG_LEGION + assert(deps.ready_deps.find(next_shard) == deps.ready_deps.end()); +#endif + deps.ready_deps[next_shard] = point_mapped; + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::handle_intra_space_dependence(Deserializer &derez) + //-------------------------------------------------------------------------- + { + std::pair key; + derez.deserialize(key.first); + derez.deserialize(key.second); + RtUserEvent pending_event; + derez.deserialize(pending_event); + ShardID requesting_shard; + derez.deserialize(requesting_shard); + + AutoLock r_lock(replication_lock); + IntraSpaceDeps &deps = intra_space_deps[key]; + // Check to see if someone has already registered this shard + std::map::iterator finder = + deps.ready_deps.find(requesting_shard); + if (finder != deps.ready_deps.end()) + { + Runtime::trigger_event(pending_event, finder->second); + deps.ready_deps.erase(finder); + if (deps.ready_deps.empty() && deps.pending_deps.empty()) + intra_space_deps.erase(key); + } + else + { + // Not seen yet so just record our entry for this shard +#ifdef DEBUG_LEGION + assert(deps.pending_deps.find(requesting_shard) == + deps.pending_deps.end()); +#endif + deps.pending_deps[requesting_shard] = pending_event; + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::receive_resources(size_t return_index, + std::map &created_regs, + std::vector &deleted_regs, + std::set > &created_fids, + std::vector > &deleted_fids, + std::map &created_fs, + std::map > &latent_fs, + std::vector &deleted_fs, + std::map &created_is, + std::vector > &deleted_is, + std::map &created_partitions, + std::vector > &deleted_partitions, + std::set &preconditions) + //-------------------------------------------------------------------------- + { + // We need to broadcast these updates out to other shards + Serializer rez; + // If we have any deletions make barriers for use with + // the deletion operations we may need to perform + if (!deleted_regs.empty() || !deleted_fids.empty() || + !deleted_fs.empty() || !deleted_is.empty() || + !deleted_partitions.empty()) + { + if (!returned_resource_ready_barrier.exists()) + returned_resource_ready_barrier = RtBarrier( + Realm::Barrier::create_barrier(shard_manager->total_shards)); + if (!returned_resource_mapped_barrier.exists()) + returned_resource_mapped_barrier = RtBarrier( + Realm::Barrier::create_barrier(shard_manager->total_shards)); + if (!returned_resource_execution_barrier.exists()) + returned_resource_execution_barrier = RtBarrier( + Realm::Barrier::create_barrier(shard_manager->total_shards)); + } + rez.serialize(return_index); + rez.serialize(returned_resource_ready_barrier); + rez.serialize(returned_resource_mapped_barrier); + rez.serialize(returned_resource_execution_barrier); + rez.serialize(created_regs.size()); + if (!created_regs.empty()) + { + for (std::map::const_iterator it = + created_regs.begin(); it != created_regs.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + rez.serialize(deleted_regs.size()); + if (!deleted_regs.empty()) + { + for (std::vector::const_iterator it = + deleted_regs.begin(); it != deleted_regs.end(); it++) + rez.serialize(*it); + } + rez.serialize(created_fids.size()); + if (!created_fids.empty()) + { + for (std::set >::const_iterator + it = created_fids.begin(); it != created_fids.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + rez.serialize(deleted_fids.size()); + if (!deleted_fids.empty()) + { + for (std::vector >::const_iterator it = + deleted_fids.begin(); it != deleted_fids.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + rez.serialize(created_fs.size()); + if (!created_fs.empty()) + { + for (std::map::const_iterator it = + created_fs.begin(); it != created_fs.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + rez.serialize(latent_fs.size()); + if (!latent_fs.empty()) + { + for (std::map >::const_iterator it = + latent_fs.begin(); it != latent_fs.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second.size()); + for (std::set::const_iterator it2 = + it->second.begin(); it2 != it->second.end(); it2++) + rez.serialize(*it2); + } + } + rez.serialize(deleted_fs.size()); + if (!deleted_fs.empty()) + { + for (std::vector::const_iterator it = + deleted_fs.begin(); it != deleted_fs.end(); it++) + rez.serialize(*it); + } + rez.serialize(created_is.size()); + if (!created_is.empty()) + { + for (std::map::const_iterator it = + created_is.begin(); it != created_is.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + rez.serialize(deleted_is.size()); + if (!deleted_is.empty()) + { + for (std::vector >::const_iterator it = + deleted_is.begin(); it != deleted_is.end(); it++) + rez.serialize(*it); + } + rez.serialize(created_partitions.size()); + if (!created_partitions.empty()) + { + for (std::map::const_iterator it = + created_partitions.begin(); it != + created_partitions.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + rez.serialize(deleted_partitions.size()); + if (!deleted_partitions.empty()) + { + for (std::vector >::const_iterator it = + deleted_partitions.begin(); it != deleted_partitions.end(); it++) + rez.serialize(*it); + } + shard_manager->broadcast_resource_update(owner_shard, rez, preconditions); + // Now we can handle this for ourselves + receive_replicate_resources(return_index, created_regs, deleted_regs, + created_fids, deleted_fids, created_fs, latent_fs, deleted_fs, + created_is, deleted_is, created_partitions, deleted_partitions, + preconditions, returned_resource_ready_barrier, + returned_resource_mapped_barrier,returned_resource_execution_barrier); + } + + //-------------------------------------------------------------------------- + void ReplicateContext::receive_replicate_resources(size_t return_index, + std::map &created_regs, + std::vector &deleted_regs, + std::set > &created_fids, + std::vector > &deleted_fids, + std::map &created_fs, + std::map > &latent_fs, + std::vector &deleted_fs, + std::map &created_is, + std::vector > &deleted_is, + std::map &created_partitions, + std::vector > &deleted_partitions, + std::set &preconditions, RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, RtBarrier &execution_barrier) + //-------------------------------------------------------------------------- + { + bool need_deletion_dependences = true; + ApEvent precondition; + std::map dependences; + if (!created_regs.empty()) + register_region_creations(created_regs); + if (!deleted_regs.empty()) + { + precondition = + compute_return_deletion_dependences(return_index, dependences); + need_deletion_dependences = false; + register_region_deletions(precondition, dependences, + deleted_regs, preconditions, ready_barrier, + mapped_barrier, execution_barrier); + } + if (!created_fids.empty()) + register_field_creations(created_fids); + if (!deleted_fids.empty()) + { + if (need_deletion_dependences) + { + precondition = + compute_return_deletion_dependences(return_index, dependences); + need_deletion_dependences = false; + } + register_field_deletions(precondition, dependences, + deleted_fids, preconditions, ready_barrier, + mapped_barrier, execution_barrier); + } + if (!created_fs.empty()) + register_field_space_creations(created_fs); + if (!latent_fs.empty()) + register_latent_field_spaces(latent_fs); + if (!deleted_fs.empty()) + { + if (need_deletion_dependences) + { + precondition = + compute_return_deletion_dependences(return_index, dependences); + need_deletion_dependences = false; + } + register_field_space_deletions(precondition, dependences, + deleted_fs, preconditions, ready_barrier, + mapped_barrier, execution_barrier); + } + if (!created_is.empty()) + register_index_space_creations(created_is); + if (!deleted_is.empty()) + { + if (need_deletion_dependences) + { + precondition = + compute_return_deletion_dependences(return_index, dependences); + need_deletion_dependences = false; + } + register_index_space_deletions(precondition, dependences, + deleted_is, preconditions, ready_barrier, + mapped_barrier, execution_barrier); + } + if (!created_partitions.empty()) + register_index_partition_creations(created_partitions); + if (!deleted_partitions.empty()) + { + if (need_deletion_dependences) + { + precondition = + compute_return_deletion_dependences(return_index, dependences); + need_deletion_dependences = false; + } + register_index_partition_deletions(precondition, dependences, + deleted_partitions, preconditions, + ready_barrier, mapped_barrier, + execution_barrier); + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::register_region_deletions(ApEvent precondition, + const std::map &dependences, + std::vector ®ions, + std::set &preconditions, + RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, + RtBarrier &execution_barrier) + //-------------------------------------------------------------------------- + { + std::vector delete_now; + { + AutoLock priv_lock(privilege_lock); + for (std::vector::const_iterator rit = + regions.begin(); rit != regions.end(); rit++) + { + std::map::iterator region_finder = + created_regions.find(*rit); + if (region_finder == created_regions.end()) + { + if (local_regions.find(*rit) != local_regions.end()) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_RESOURCE_DESTRUCTION, + "Local logical region (%x,%x,%x) in task %s (UID %lld) was " + "not deleted by this task. Local regions can only be deleted " + "by the task that made them.", rit->index_space.id, + rit->field_space.id, rit->tree_id, + get_task_name(), get_unique_id()) + // Deletion keeps going up + deleted_regions.push_back(*rit); + } + else + { + // One of ours to delete +#ifdef DEBUG_LEGION + assert(region_finder->second > 0); +#endif + if (--region_finder->second == 0) + { + created_regions.erase(region_finder); + delete_now.push_back(*rit); + // Check to see if we have any latent field spaces to clean up + if (!latent_field_spaces.empty()) + { + std::map >::iterator finder = + latent_field_spaces.find(rit->get_field_space()); + if (finder != latent_field_spaces.end()) + { + std::set::iterator latent_finder = + finder->second.find(*rit); +#ifdef DEBUG_LEGION + assert(latent_finder != finder->second.end()); +#endif + finder->second.erase(latent_finder); + if (finder->second.empty()) + { + // Now that all the regions using this field space have + // been deleted we can clean up all the created_fields + for (std::set >::iterator it = + created_fields.begin(); it != + created_fields.end(); /*nothing*/) + { + if (it->first == finder->first) + { + std::set >::iterator + to_delete = it++; + created_fields.erase(to_delete); + } + else + it++; + } + latent_field_spaces.erase(finder); + } + } + } + } + } + } + } + if (!delete_now.empty()) + { + for (std::vector::const_iterator it = + delete_now.begin(); it != delete_now.end(); it++) + { + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_logical_region_deletion(this, *it, true/*unordered*/); + op->initialize_replication(this, ready_barrier, mapped_barrier, + execution_barrier, shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + op->set_execution_precondition(precondition); + preconditions.insert( + Runtime::protect_event(op->get_completion_event())); + op->begin_dependence_analysis(); + for (std::map::const_iterator dit = + dependences.begin(); dit != dependences.end(); dit++) + op->register_dependence(dit->first, dit->second); + op->end_dependence_analysis(); + } + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::register_field_deletions(ApEvent precondition, + const std::map &dependences, + std::vector > &fields, + std::set &preconditions, + RtBarrier &ready_barrier, RtBarrier &mapped_barrier, + RtBarrier &execution_barrier) + //-------------------------------------------------------------------------- + { + std::map > delete_now; + { + AutoLock priv_lock(privilege_lock); + for (std::vector >::const_iterator fit = + fields.begin(); fit != fields.end(); fit++) + { + std::set >::const_iterator + field_finder = created_fields.find(*fit); + if (field_finder == created_fields.end()) + { + std::map,bool>::iterator + local_finder = local_fields.find(*fit); + if (local_finder != local_fields.end()) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_RESOURCE_DESTRUCTION, + "Local field %d in field space %x in task %s (UID %lld) was " + "not deleted by this task. Local fields can only be deleted " + "by the task that made them.", fit->second, fit->first.id, + get_task_name(), get_unique_id()) + deleted_fields.push_back(*fit); + } + else + { + // One of ours to delete + delete_now[fit->first].insert(fit->second); + created_fields.erase(field_finder); + } + } + } + if (!delete_now.empty()) + { + for (std::map >::const_iterator it = + delete_now.begin(); it != delete_now.end(); it++) + { + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + FieldAllocatorImpl *allocator = + create_field_allocator(it->first, true/*unordered*/); + op->initialize_field_deletions(this, it->first, it->second, + true/*unordered*/, allocator); + op->initialize_replication(this, ready_barrier, mapped_barrier, + execution_barrier, shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + op->set_execution_precondition(precondition); + preconditions.insert( + Runtime::protect_event(op->get_completion_event())); + op->begin_dependence_analysis(); + for (std::map::const_iterator dit = + dependences.begin(); dit != dependences.end(); dit++) + op->register_dependence(dit->first, dit->second); + op->end_dependence_analysis(); + } + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::register_field_space_deletions(ApEvent precondition, + const std::map &dependences, + std::vector &spaces, + std::set &preconditions, + RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, + RtBarrier &execution_barrier) + //-------------------------------------------------------------------------- + { + std::vector delete_now; + { + AutoLock priv_lock(privilege_lock); + for (std::vector::const_iterator fit = + spaces.begin(); fit != spaces.end(); fit++) + { + std::map::iterator finder = + created_field_spaces.find(*fit); + if (finder != created_field_spaces.end()) + { +#ifdef DEBUG_LEGION + assert(finder->second > 0); +#endif + if (--finder->second == 0) + { + delete_now.push_back(*fit); + created_field_spaces.erase(finder); + // Count how many regions are still using this field space + // that still need to be deleted before we can remove the + // list of created fields + std::set remaining_regions; + for (std::map::const_iterator it = + created_regions.begin(); it != created_regions.end(); it++) + if (it->first.get_field_space() == *fit) + remaining_regions.insert(it->first); + for (std::map::const_iterator it = + local_regions.begin(); it != local_regions.end(); it++) + if (it->first.get_field_space() == *fit) + remaining_regions.insert(it->first); + if (remaining_regions.empty()) + { + // No remaining regions so we can remove any created fields now + for (std::set >::iterator it = + created_fields.begin(); it != + created_fields.end(); /*nothing*/) + { + if (it->first == *fit) + { + std::set >::iterator + to_delete = it++; + created_fields.erase(to_delete); + } + else + it++; + } + } + else + latent_field_spaces[*fit] = remaining_regions; + } + } + else + // If we didn't make this field space, record the deletion + // and keep going. It will be handled by the context that + // made the field space + deleted_field_spaces.push_back(*fit); + } + } + if (!delete_now.empty()) + { + for (std::vector::const_iterator it = + delete_now.begin(); it != delete_now.end(); it++) + { + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_field_space_deletion(this, *it, true/*unordered*/); + op->initialize_replication(this, ready_barrier, mapped_barrier, + execution_barrier, shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + op->set_execution_precondition(precondition); + preconditions.insert( + Runtime::protect_event(op->get_completion_event())); + op->begin_dependence_analysis(); + for (std::map::const_iterator dit = + dependences.begin(); dit != dependences.end(); dit++) + op->register_dependence(dit->first, dit->second); + op->end_dependence_analysis(); + } + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::register_index_space_deletions(ApEvent precondition, + const std::map &dependences, + std::vector > &spaces, + std::set &preconditions, + RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, + RtBarrier &execution_barrier) + //-------------------------------------------------------------------------- + { + std::vector delete_now; + std::vector > sub_partitions; + { + AutoLock priv_lock(privilege_lock); + for (std::vector >::const_iterator sit = + spaces.begin(); sit != spaces.end(); sit++) + { + std::map::iterator finder = + created_index_spaces.find(sit->first); + if (finder != created_index_spaces.end()) + { +#ifdef DEBUG_LEGION + assert(finder->second > 0); +#endif + if (--finder->second == 0) + { + delete_now.push_back(sit->first); + sub_partitions.resize(sub_partitions.size() + 1); + created_index_spaces.erase(finder); + if (sit->second) + { + std::vector &subs = sub_partitions.back(); + // Also remove any index partitions for this index space tree + for (std::map::iterator it = + created_index_partitions.begin(); it != + created_index_partitions.end(); /*nothing*/) + { + if (it->first.get_tree_id() == sit->first.get_tree_id()) + { +#ifdef DEBUG_LEGION + assert(it->second > 0); +#endif + if (--it->second == 0) + { + subs.push_back(it->first); + std::map::iterator + to_delete = it++; + created_index_partitions.erase(to_delete); + } + else + it++; + } + else + it++; + } + } + } + } + else + // If we didn't make the index space in this context, just + // record it and keep going, it will get handled later + deleted_index_spaces.push_back(*sit); + } + } + if (!delete_now.empty()) + { +#ifdef DEBUG_LEGION + assert(delete_now.size() == sub_partitions.size()); +#endif + for (unsigned idx = 0; idx < delete_now.size(); idx++) + { + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_index_space_deletion(this, delete_now[idx], + sub_partitions[idx], true/*unordered*/); + op->initialize_replication(this, ready_barrier, mapped_barrier, + execution_barrier, shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + op->set_execution_precondition(precondition); + preconditions.insert( + Runtime::protect_event(op->get_completion_event())); + op->begin_dependence_analysis(); + for (std::map::const_iterator dit = + dependences.begin(); dit != dependences.end(); dit++) + op->register_dependence(dit->first, dit->second); + op->end_dependence_analysis(); + } + } + } + + //-------------------------------------------------------------------------- + void ReplicateContext::register_index_partition_deletions(ApEvent precond, + const std::map &dependences, + std::vector > &parts, + std::set &preconditions, + RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, + RtBarrier &execution_barrier) + //-------------------------------------------------------------------------- + { + std::vector delete_now; + std::vector > sub_partitions; + { + AutoLock priv_lock(privilege_lock); + for (std::vector >::const_iterator pit = + parts.begin(); pit != parts.end(); pit++) + { + std::map::iterator finder = + created_index_partitions.find(pit->first); + if (finder != created_index_partitions.end()) + { +#ifdef DEBUG_LEGION + assert(finder->second > 0); +#endif + if (--finder->second == 0) + { + delete_now.push_back(pit->first); + sub_partitions.resize(sub_partitions.size() + 1); + created_index_partitions.erase(finder); + if (pit->second) + { + std::vector &subs = sub_partitions.back(); + // Remove any other partitions that this partition dominates + for (std::map::iterator it = + created_index_partitions.begin(); it != + created_index_partitions.end(); /*nothing*/) + { + if ((pit->first.get_tree_id() == it->first.get_tree_id()) && + runtime->forest->is_dominated_tree_only(it->first, + pit->first)) + { +#ifdef DEBUG_LEGION + assert(it->second > 0); +#endif + if (--it->second == 0) + { + subs.push_back(it->first); + std::map::iterator + to_delete = it++; + created_index_partitions.erase(to_delete); + } + else + it++; + } + else + it++; + } + } + } + } + else + // If we didn't make the partition, record it and keep going + deleted_index_partitions.push_back(*pit); + } + } + if (!delete_now.empty()) + { +#ifdef DEBUG_LEGION + assert(delete_now.size() == sub_partitions.size()); +#endif + for (unsigned idx = 0; idx < delete_now.size(); idx++) + { + ReplDeletionOp *op = runtime->get_available_repl_deletion_op(); + op->initialize_index_part_deletion(this, delete_now[idx], + sub_partitions[idx], true/*unordered*/); + op->initialize_replication(this, ready_barrier, mapped_barrier, + execution_barrier, shard_manager->is_total_sharding(), + shard_manager->is_first_local_shard(owner_shard)); + op->set_execution_precondition(precond); + preconditions.insert( + Runtime::protect_event(op->get_completion_event())); + op->begin_dependence_analysis(); + for (std::map::const_iterator dit = + dependences.begin(); dit != dependences.end(); dit++) + op->register_dependence(dit->first, dit->second); + op->end_dependence_analysis(); + } + } + } + + //-------------------------------------------------------------------------- + CollectiveID ReplicateContext::get_next_collective_index( + CollectiveIndexLocation loc) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION_COLLECTIVES + CollectiveCheckReduction::RHS location = loc; + Runtime::phase_barrier_arrive(collective_check_barrier, 1/*count*/, + RtEvent::NO_RT_EVENT, &location, sizeof(location)); + collective_check_barrier.wait(); + CollectiveCheckReduction::RHS actual_location; + bool ready = Runtime::get_barrier_result(collective_check_barrier, + &actual_location, sizeof(actual_location)); + assert(ready); + assert(location == actual_location); + advance_replicate_barrier(collective_check_barrier, total_shards); +#endif + // No need for a lock, should only be coming from the creation + // of operations directly from the application and therefore + // should be deterministic + return next_available_collective_index++; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::register_collective(ShardCollective *collective) + //-------------------------------------------------------------------------- + { + std::vector > to_apply; + { + AutoLock repl_lock(replication_lock); +#ifdef DEBUG_LEGION + assert(collectives.find(collective->collective_index) == + collectives.end()); + assert(shard_manager != NULL); +#endif + // If the collectives are empty then we add a reference to the + // shard manager to prevent it being collected before we're + // done handling all the collectives + if (collectives.empty()) + shard_manager->add_reference(); + collectives[collective->collective_index] = collective; + std::map > >:: + iterator finder = pending_collective_updates.find( + collective->collective_index); + if (finder != pending_collective_updates.end()) + { + to_apply.swap(finder->second); + pending_collective_updates.erase(finder); + } + } + if (!to_apply.empty()) + { + for (std::vector >::const_iterator it = + to_apply.begin(); it != to_apply.end(); it++) + { + Deserializer derez(it->first, it->second); + collective->handle_collective_message(derez); + free(it->first); + } + } + } + + //-------------------------------------------------------------------------- + ShardCollective* ReplicateContext::find_or_buffer_collective( + Deserializer &derez) + //-------------------------------------------------------------------------- + { + CollectiveID collective_index; + derez.deserialize(collective_index); + AutoLock repl_lock(replication_lock); + // See if we already have the collective in which case we can just + // return it, otherwise we need to buffer the deserializer + std::map::const_iterator finder = + collectives.find(collective_index); + if (finder != collectives.end()) + return finder->second; + // If we couldn't find it then we have to buffer it for the future + const size_t remaining_bytes = derez.get_remaining_bytes(); + void *buffer = malloc(remaining_bytes); + memcpy(buffer, derez.get_current_pointer(), remaining_bytes); + derez.advance_pointer(remaining_bytes); + pending_collective_updates[collective_index].push_back( + std::pair(buffer, remaining_bytes)); + return NULL; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::unregister_collective(ShardCollective *collective) + //-------------------------------------------------------------------------- + { + bool remove_reference = false; + { + AutoLock repl_lock(replication_lock); + std::map::iterator finder = + collectives.find(collective->collective_index); + // Sometimes collectives are not used + if (finder != collectives.end()) + { + collectives.erase(finder); + // Once we've done all our collectives then we can remove the + // reference that we added on the shard manager + remove_reference = collectives.empty(); + } + } + if (remove_reference && shard_manager->remove_reference()) + delete shard_manager; + } + + //-------------------------------------------------------------------------- + unsigned ReplicateContext::peek_next_future_map_barrier_index(void) const + //-------------------------------------------------------------------------- + { + return next_future_map_bar_index; + } + + //-------------------------------------------------------------------------- + RtBarrier ReplicateContext::get_next_future_map_barrier(void) + //-------------------------------------------------------------------------- + { + RtBarrier &next = future_map_barriers[next_future_map_bar_index++]; + if (next_future_map_bar_index == future_map_barriers.size()) + next_future_map_bar_index = 0; + RtBarrier result = next; + advance_replicate_barrier(next, total_shards); + return result; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::register_future_map(ReplFutureMapImpl *map) + //-------------------------------------------------------------------------- + { + map->add_base_resource_ref(REPLICATION_REF); + std::vector > to_apply; + { + AutoLock repl_lock(replication_lock); +#ifdef DEBUG_LEGION + assert(future_maps.find(map->future_map_barrier) == future_maps.end()); +#endif + future_maps[map->future_map_barrier] = map; + // Check to see if we have any pending requests to perform + std::map > >::iterator + finder = pending_future_map_requests.find(map->future_map_barrier); + if (finder != pending_future_map_requests.end()) + { + to_apply.swap(finder->second); + pending_future_map_requests.erase(finder); + } + } + if (!to_apply.empty()) + { + for (std::vector >::const_iterator it = + to_apply.begin(); it != to_apply.end(); it++) + { + Deserializer derez(it->first, it->second); + map->handle_future_map_request(derez); + free(it->first); + } + } + } + + //-------------------------------------------------------------------------- + ReplFutureMapImpl* ReplicateContext::find_or_buffer_future_map_request( + Deserializer &derez) + //-------------------------------------------------------------------------- + { + RtEvent future_map_event; + derez.deserialize(future_map_event); + AutoLock repl_lock(replication_lock); + // See if we already have the future map in which case we can just + // return it, otherwise we need to buffer the deserializer + std::map::const_iterator finder = + future_maps.find(future_map_event); + if (finder != future_maps.end()) + return finder->second; + // If we couldn't find it then we have to buffer it for the future + const size_t remaining_bytes = derez.get_remaining_bytes(); + void *buffer = malloc(remaining_bytes); + memcpy(buffer, derez.get_current_pointer(), remaining_bytes); + derez.advance_pointer(remaining_bytes); + pending_future_map_requests[future_map_event].push_back( + std::pair(buffer, remaining_bytes)); + return NULL; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::unregister_future_map(ReplFutureMapImpl *map) + //-------------------------------------------------------------------------- + { + { + AutoLock repl_lock(replication_lock); + std::map::iterator finder = + future_maps.find(map->future_map_barrier); +#ifdef DEBUG_LEGION + assert(finder != future_maps.end()); +#endif + future_maps.erase(finder); + } + if (map->remove_base_resource_ref(REPLICATION_REF)) + delete map; + } + + //-------------------------------------------------------------------------- + size_t ReplicateContext::register_trace_template( + ShardedPhysicalTemplate *physical_template) + //-------------------------------------------------------------------------- + { + size_t index; + std::vector to_apply; + { + AutoLock r_lock(replication_lock); + index = next_physical_template_index++; +#ifdef DEBUG_LEGION + assert(physical_templates.find(index) == physical_templates.end()); +#endif + physical_templates[index] = physical_template; + // Check to see if we have any pending updates to perform + std::map >::iterator + finder = pending_template_updates.find(index); + if (finder != pending_template_updates.end()) + { + to_apply.swap(finder->second); + pending_template_updates.erase(finder); + } + } + if (!to_apply.empty()) + { + for (std::vector::const_iterator it = + to_apply.begin(); it != to_apply.end(); it++) + { + Deserializer derez(it->ptr, it->size); + physical_template->handle_trace_update(derez, it->source); + free(it->ptr); + } + } + return index; + } + + //-------------------------------------------------------------------------- + ShardedPhysicalTemplate* ReplicateContext::find_or_buffer_trace_update( + Deserializer &derez, AddressSpaceID source) + //-------------------------------------------------------------------------- + { + size_t trace_index; + derez.deserialize(trace_index); + AutoLock r_lock(replication_lock); + std::map::const_iterator finder = + physical_templates.find(trace_index); + if (finder != physical_templates.end()) + return finder->second; +#ifdef DEBUG_LEGION + assert(next_physical_template_index <= trace_index); +#endif + // If we couldn't find it then we have to buffer it for the future + const size_t remaining_bytes = derez.get_remaining_bytes(); + void *buffer = malloc(remaining_bytes); + memcpy(buffer, derez.get_current_pointer(), remaining_bytes); + derez.advance_pointer(remaining_bytes); + pending_template_updates[trace_index].push_back( + PendingTemplateUpdate(buffer, remaining_bytes, source)); + return NULL; + } + + //-------------------------------------------------------------------------- + void ReplicateContext::unregister_trace_template(size_t index) + //-------------------------------------------------------------------------- + { + AutoLock r_lock(replication_lock); +#ifdef DEBUG_LEGION + std::map::iterator finder = + physical_templates.find(index); + assert(finder != physical_templates.end()); + physical_templates.erase(finder); +#else + physical_templates.erase(index); +#endif + } + + //-------------------------------------------------------------------------- + RtBarrier ReplicateContext::get_next_mapping_fence_barrier(void) + //-------------------------------------------------------------------------- + { + RtBarrier result = mapping_fence_barrier; + advance_replicate_barrier(mapping_fence_barrier, total_shards); + return result; + } + + //-------------------------------------------------------------------------- + ApBarrier ReplicateContext::get_next_execution_fence_barrier(void) + //-------------------------------------------------------------------------- + { + ApBarrier result = execution_fence_barrier; + advance_replicate_barrier(execution_fence_barrier, total_shards); + return result; + } + + //-------------------------------------------------------------------------- + RtBarrier ReplicateContext::get_next_trace_recording_barrier(void) + //-------------------------------------------------------------------------- + { + const RtBarrier result = trace_recording_barrier; + Runtime::advance_barrier(trace_recording_barrier); + if (trace_recording_barrier.exists()) + return result; + // If it doesn't exist then we have to make a new one + // We can't make a collective ID here because we're in the dependence + // analysis stage of the pipeline +#ifdef DEBUG_LEGION + // There better be one of these here + assert(trace_recording_collective_id > 0); +#endif + ValueBroadcast + collective(trace_recording_collective_id, this, next_trace_bar_index); + if (owner_shard->shard_id == next_trace_bar_index++) + { + trace_recording_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + collective.broadcast(trace_recording_barrier); + } + else + trace_recording_barrier = collective.get_value(); + // Check to see if we need to reset th next_trace_bar_index + if (next_trace_bar_index == total_shards) + next_trace_bar_index = 0; + // Set this back to zero so we can re-initialize it in the application + // at some point in the future. This is safe as long as we know there + // are more barrier generations than possible oustanding replays which + // should always be true + trace_recording_collective_id = 0; + return result; + } + + //-------------------------------------------------------------------------- + RtBarrier ReplicateContext::get_next_summary_fence_barrier(void) + //-------------------------------------------------------------------------- + { + const RtBarrier result = summary_fence_barrier; + Runtime::advance_barrier(summary_fence_barrier); + if (summary_fence_barrier.exists()) + return result; + // If it doesn't exist then we have to make a new one + // We can't make a collective ID here because we're in the dependence + // analysis stage of the pipeline +#ifdef DEBUG_LEGION + // There better be one of these here + assert(summary_collective_id > 0); +#endif + ValueBroadcast + collective(summary_collective_id, this, next_summary_bar_index); + if (owner_shard->shard_id == next_summary_bar_index++) + { + summary_fence_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + collective.broadcast(summary_fence_barrier); } - // Send it to the owner space - runtime->send_compute_equivalence_sets_request(owner_space, rez); - return ready_event; + else + summary_fence_barrier = collective.get_value(); + // Check to see if we need to reset th next_summary_bar_index + if (next_summary_bar_index == total_shards) + next_summary_bar_index = 0; + // Set this back to zero so we can re-initialize it in the application + // at some point in the future. This is safe as long as we know there + // are more barrier generations than possible oustanding replays which + // should always be true + summary_collective_id = 0; + return result; } //-------------------------------------------------------------------------- - InnerContext* TopLevelContext::find_outermost_local_context( - InnerContext *previous) + void ReplicateContext::create_new_replicate_barrier(RtBarrier &bar, + size_t arrivals) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert(previous != NULL); + assert(!bar.exists()); + assert(next_replicate_bar_index < total_shards); #endif - return previous; + ValueBroadcast + collective(this, next_replicate_bar_index, COLLECTIVE_LOC_83); + if (owner_shard->shard_id == next_replicate_bar_index++) + { + bar = RtBarrier(Realm::Barrier::create_barrier(arrivals)); + collective.broadcast(bar); + } + else + bar = collective.get_value(); + // Check to see if we need to reset the next_replicate_bar_index + if (next_replicate_bar_index == total_shards) + next_replicate_bar_index = 0; } //-------------------------------------------------------------------------- - InnerContext* TopLevelContext::find_top_context(void) + void ReplicateContext::create_new_replicate_barrier(ApBarrier &bar, + size_t arrivals) //-------------------------------------------------------------------------- { - return this; +#ifdef DEBUG_LEGION + assert(!bar.exists()); + assert(next_replicate_bar_index < total_shards); +#endif + ValueBroadcast + collective(this, next_replicate_bar_index, COLLECTIVE_LOC_84); + if (owner_shard->shard_id == next_replicate_bar_index++) + { + bar = ApBarrier(Realm::Barrier::create_barrier(arrivals)); + collective.broadcast(bar); + } + else + bar = collective.get_value(); + // Check to see if we need to reset the next_replicate_bar_index + if (next_replicate_bar_index == total_shards) + next_replicate_bar_index = 0; } ///////////////////////////////////////////////////////////// @@ -9408,13 +17160,6 @@ namespace Legion { return *this; } - //-------------------------------------------------------------------------- - int RemoteTask::get_depth(void) const - //-------------------------------------------------------------------------- - { - return owner->get_depth(); - } - //-------------------------------------------------------------------------- UniqueID RemoteTask::get_unique_id(void) const //-------------------------------------------------------------------------- @@ -9436,6 +17181,13 @@ namespace Legion { context_index = index; } + //-------------------------------------------------------------------------- + int RemoteTask::get_depth(void) const + //-------------------------------------------------------------------------- + { + return owner->get_depth(); + } + //-------------------------------------------------------------------------- const char* RemoteTask::get_task_name(void) const //-------------------------------------------------------------------------- @@ -9460,8 +17212,8 @@ namespace Legion { : InnerContext(rt, NULL, -1, false/*full inner*/, remote_task.regions, local_parent_req_indexes, local_virtual_mapped, context_uid, ApEvent::NO_AP_EVENT, true/*remote*/), - parent_ctx(NULL), top_level_context(false), - remote_task(RemoteTask(this)) + parent_ctx(NULL), shard_manager(NULL), + top_level_context(false), remote_task(RemoteTask(this)), repl_id(0) //-------------------------------------------------------------------------- { } @@ -9675,6 +17427,48 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + void RemoteContext::record_using_physical_context(LogicalRegion handle) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(handle.exists()); +#endif + AutoLock rem_lock(remote_lock); + local_physical_contexts.insert(handle); + } + + //-------------------------------------------------------------------------- + InstanceView* RemoteContext::create_instance_top_view( + PhysicalManager* manager, AddressSpaceID source) + //-------------------------------------------------------------------------- + { + // Check to see if we are part of a replicate context, if we do + // then we need to send this request back to our owner node + if (repl_id > 0) + { + InstanceView *volatile result = NULL; + RtUserEvent wait_on = Runtime::create_rt_user_event(); + Serializer rez; + { + RezCheck z(rez); + rez.serialize(context_uid); + rez.serialize(manager->did); + rez.serialize(const_cast(&result)); + rez.serialize(wait_on); + } + const AddressSpaceID target = runtime->get_runtime_owner(context_uid); + runtime->send_create_top_view_request(target, rez); + wait_on.wait(); +#ifdef DEBUG_LEGION + assert(result != NULL); +#endif + return result; + } + else + return InnerContext::create_instance_top_view(manager, source); + } + //-------------------------------------------------------------------------- void RemoteContext::invalidate_region_tree_contexts(void) //-------------------------------------------------------------------------- @@ -9724,6 +17518,36 @@ namespace Legion { invalidate_region_tree_contexts(); } + //-------------------------------------------------------------------------- + ShardingFunction* RemoteContext::find_sharding_function(ShardingID sid) + //-------------------------------------------------------------------------- + { + if (shard_manager != NULL) + return shard_manager->find_sharding_function(sid); + // Check to see if it is in the cache + { + AutoLock rem_lock(remote_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + sharding_functions.find(sid); + if (finder != sharding_functions.end()) + return finder->second; + } + // Get the functor from the runtime + ShardingFunctor *functor = runtime->find_sharding_functor(sid); + // Retake the lock + AutoLock rem_lock(remote_lock); + // See if we lost the race + std::map::const_iterator finder = + sharding_functions.find(sid); + if (finder != sharding_functions.end()) + return finder->second; + ShardingFunction *result = + new ShardingFunction(functor, runtime->forest, sid, total_shards); + // Save the result for the future + sharding_functions[sid] = result; + return result; + } + //-------------------------------------------------------------------------- void RemoteContext::unpack_remote_context(Deserializer &derez, std::set &preconditions) @@ -9753,7 +17577,15 @@ namespace Legion { derez.deserialize(parent_context_uid); // Unpack any local fields that we have unpack_local_field_update(derez); - + bool replicate; + derez.deserialize(replicate); + if (replicate) + { + derez.deserialize(total_shards); + derez.deserialize(repl_id); + // See if we have a local shard manager + shard_manager = runtime->find_shard_manager(repl_id, true/*can fail*/); + } // See if we can find our parent task, if not don't worry about it // DO NOT CHANGE THIS UNLESS YOU THINK REALLY HARD ABOUT VIRTUAL // CHANNELS AND HOW CONTEXT META-DATA IS MOVED! @@ -9994,7 +17826,8 @@ namespace Legion { } //-------------------------------------------------------------------------- - void LeafContext::pack_remote_context(Serializer &rez,AddressSpaceID target) + void LeafContext::pack_remote_context(Serializer &rez, + AddressSpaceID target, bool replicate) //-------------------------------------------------------------------------- { assert(false); @@ -10089,6 +17922,8 @@ namespace Legion { void LeafContext::handle_registration_callback_effects(RtEvent effects) //-------------------------------------------------------------------------- { + if (effects.has_triggered()) + return; AutoLock l_lock(leaf_lock); execution_events.insert(effects); } @@ -10368,6 +18203,22 @@ namespace Legion { return IndexPartition::NO_PART; } + //-------------------------------------------------------------------------- + IndexPartition LeafContext::create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_ILLEGAL_PARTITION_BY_DOMAIN, + "Illegal create partition by domain performed in leaf " + "task %s (UID %lld)", get_task_name(), get_unique_id()) + return IndexPartition::NO_PART; + } + //-------------------------------------------------------------------------- IndexPartition LeafContext::create_partition_by_domain( IndexSpace parent, @@ -10490,6 +18341,7 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace LeafContext::create_index_space_union(IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles) //-------------------------------------------------------------------------- @@ -10503,6 +18355,7 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpace LeafContext::create_index_space_union(IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) //-------------------------------------------------------------------------- @@ -10517,6 +18370,7 @@ namespace Legion { IndexSpace LeafContext::create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles) //-------------------------------------------------------------------------- @@ -10531,6 +18385,7 @@ namespace Legion { IndexSpace LeafContext::create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) //-------------------------------------------------------------------------- @@ -10545,6 +18400,7 @@ namespace Legion { IndexSpace LeafContext::create_index_space_difference( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexSpace initial, const std::vector &handles) @@ -10922,7 +18778,8 @@ namespace Legion { //-------------------------------------------------------------------------- FutureMap LeafContext::construct_future_map(const Domain &domain, - const std::map &futures, bool internal) + const std::map &futures, + RtUserEvent domain_deletion, bool internal) //-------------------------------------------------------------------------- { REPORT_LEGION_ERROR(ERROR_ILLEGAL_EXECUTE_INDEX_SPACE, @@ -11194,6 +19051,85 @@ namespace Legion { return Future(); } + //-------------------------------------------------------------------------- + ApBarrier LeafContext::create_phase_barrier(unsigned arrivals, + ReductionOpID redop, + const void *init_value, + size_t init_size) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + log_run.debug("Creating application barrier in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + return ApBarrier(Realm::Barrier::create_barrier(arrivals, redop, + init_value, init_size)); + } + + //-------------------------------------------------------------------------- + void LeafContext::destroy_phase_barrier(ApBarrier bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + log_run.debug("Destroying phase barrier in task %s (ID %lld)", + get_task_name(), get_unique_id()); +#endif + destroy_user_barrier(bar); + } + + //-------------------------------------------------------------------------- + PhaseBarrier LeafContext::advance_phase_barrier(PhaseBarrier bar) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_LEAF_TASK_VIOLATION, + "Illegal advance phase barrier call performed in leaf task %s " + "(UID %lld)", get_task_name(), get_unique_id()); + return bar; + } + + //-------------------------------------------------------------------------- + void LeafContext::arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, unsigned count) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_LEAF_TASK_VIOLATION, + "Illegal arrive dynamic collective call performed in leaf task %s " + "(UID %lld)", get_task_name(), get_unique_id()); + } + + //-------------------------------------------------------------------------- + void LeafContext::defer_dynamic_collective_arrival(DynamicCollective dc, + const Future &f, + unsigned count) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_LEAF_TASK_VIOLATION, + "Illegal defer dynamic collective call performed in leaf task %s " + "(UID %lld)", get_task_name(), get_unique_id()); + } + + //-------------------------------------------------------------------------- + Future LeafContext::get_dynamic_collective_result(DynamicCollective dc) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_LEAF_TASK_VIOLATION, + "Illegal get dynamic collective result call performed in leaf task %s" + " (UID %lld)", get_task_name(), get_unique_id()); + return Future(); + } + + //-------------------------------------------------------------------------- + DynamicCollective LeafContext::advance_dynamic_collective( + DynamicCollective dc) + //-------------------------------------------------------------------------- + { + REPORT_LEGION_ERROR(ERROR_LEAF_TASK_VIOLATION, + "Illegal advance dynamic collective call performed in leaf task %s " + "(UID %lld)", get_task_name(), get_unique_id()); + return dc; + } + //-------------------------------------------------------------------------- size_t LeafContext::register_new_child_operation(Operation *op, const std::vector *dependences) @@ -11203,6 +19139,13 @@ namespace Legion { return 0; } + //-------------------------------------------------------------------------- + void LeafContext::register_new_internal_operation(InternalOp *op) + //-------------------------------------------------------------------------- + { + assert(false); + } + //-------------------------------------------------------------------------- size_t LeafContext::register_new_close_operation(CloseOp *op) //-------------------------------------------------------------------------- @@ -11220,10 +19163,12 @@ namespace Legion { } //-------------------------------------------------------------------------- - void LeafContext::add_to_dependence_queue(Operation *op, bool unordered) + ApEvent LeafContext::add_to_dependence_queue(Operation *op, + bool unordered, bool block) //-------------------------------------------------------------------------- { assert(false); + return ApEvent::NO_AP_EVENT; } //-------------------------------------------------------------------------- @@ -11439,6 +19384,19 @@ namespace Legion { assert(false); } + //-------------------------------------------------------------------------- +#ifdef DEBUG_LEGION_COLLECTIVES + MergeCloseOp* LeafContext::get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node) +#else + MergeCloseOp* LeafContext::get_merge_close_op(void) +#endif + //-------------------------------------------------------------------------- + { + assert(false); + return NULL; + } + //-------------------------------------------------------------------------- InnerContext* LeafContext::find_parent_logical_context(unsigned index) //-------------------------------------------------------------------------- @@ -11498,7 +19456,7 @@ namespace Legion { //-------------------------------------------------------------------------- InstanceView* LeafContext::create_instance_top_view( - PhysicalManager *manager, AddressSpaceID source, RtEvent *ready) + PhysicalManager *manager, AddressSpaceID source) //-------------------------------------------------------------------------- { assert(false); @@ -11631,14 +19589,6 @@ namespace Legion { assert(false); } - //-------------------------------------------------------------------------- - Future LeafContext::get_dynamic_collective_result(DynamicCollective dc) - //-------------------------------------------------------------------------- - { - assert(false); - return Future(); - } - //-------------------------------------------------------------------------- TaskPriority LeafContext::get_current_priority(void) const //-------------------------------------------------------------------------- @@ -11759,7 +19709,7 @@ namespace Legion { //-------------------------------------------------------------------------- void InlineContext::pack_remote_context(Serializer &rez, - AddressSpaceID target) + AddressSpaceID target, bool replicate) //-------------------------------------------------------------------------- { assert(false); @@ -11987,6 +19937,20 @@ namespace Legion { part_kind, color); } + //-------------------------------------------------------------------------- + IndexPartition InlineContext::create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color) + //-------------------------------------------------------------------------- + { + return enclosing->create_partition_by_domain(parent, domains, + color_space, perform_intersections, part_kind, color); + } + //-------------------------------------------------------------------------- IndexPartition InlineContext::create_partition_by_domain( IndexSpace parent, @@ -12101,61 +20065,66 @@ namespace Legion { IndexSpace InlineContext::create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles) //-------------------------------------------------------------------------- { return enclosing->create_index_space_union(parent, realm_color, - type_tag, handles); + color_size, type_tag, handles); } //-------------------------------------------------------------------------- IndexSpace InlineContext::create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) //-------------------------------------------------------------------------- { return enclosing->create_index_space_union(parent, realm_color, - type_tag, handle); + color_size, type_tag, handle); } //-------------------------------------------------------------------------- IndexSpace InlineContext::create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles) //-------------------------------------------------------------------------- { return enclosing->create_index_space_intersection(parent, realm_color, - type_tag, handles); + color_size, type_tag, handles); } //-------------------------------------------------------------------------- IndexSpace InlineContext::create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) //-------------------------------------------------------------------------- { return enclosing->create_index_space_intersection(parent, realm_color, - type_tag, handle); + color_size, type_tag, handle); } //-------------------------------------------------------------------------- IndexSpace InlineContext::create_index_space_difference( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexSpace initial, const std::vector &handles) //-------------------------------------------------------------------------- { return enclosing->create_index_space_difference(parent, realm_color, - type_tag, initial, handles); + color_size, type_tag, initial, handles); } //-------------------------------------------------------------------------- @@ -12307,10 +20276,11 @@ namespace Legion { } //-------------------------------------------------------------------------- - FieldAllocatorImpl* InlineContext::create_field_allocator(FieldSpace handle) + FieldAllocatorImpl* InlineContext::create_field_allocator(FieldSpace handle, + bool unordered) //-------------------------------------------------------------------------- { - return enclosing->create_field_allocator(handle); + return enclosing->create_field_allocator(handle, unordered); } //-------------------------------------------------------------------------- @@ -12371,10 +20341,12 @@ namespace Legion { //-------------------------------------------------------------------------- FutureMap InlineContext::construct_future_map(const Domain &domain, - const std::map &futures, bool internal) + const std::map &futures, + RtUserEvent domain_deletion, bool internal) //-------------------------------------------------------------------------- { - return enclosing->construct_future_map(domain, futures, internal); + return enclosing->construct_future_map(domain, futures, + domain_deletion, internal); } //-------------------------------------------------------------------------- @@ -12528,6 +20500,64 @@ namespace Legion { return enclosing->get_predicate_future(p); } + //-------------------------------------------------------------------------- + ApBarrier InlineContext::create_phase_barrier(unsigned arrivals, + ReductionOpID redop, + const void *init_value, + size_t init_size) + //-------------------------------------------------------------------------- + { + return enclosing->create_phase_barrier(arrivals, redop, + init_value, init_size); + } + + //-------------------------------------------------------------------------- + void InlineContext::destroy_phase_barrier(ApBarrier bar) + //-------------------------------------------------------------------------- + { + enclosing->destroy_phase_barrier(bar); + } + + //-------------------------------------------------------------------------- + PhaseBarrier InlineContext::advance_phase_barrier(PhaseBarrier bar) + //-------------------------------------------------------------------------- + { + return enclosing->advance_phase_barrier(bar); + } + + //-------------------------------------------------------------------------- + void InlineContext::arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, unsigned count) + //-------------------------------------------------------------------------- + { + enclosing->arrive_dynamic_collective(dc, buffer, size, count); + } + + //-------------------------------------------------------------------------- + void InlineContext::defer_dynamic_collective_arrival(DynamicCollective dc, + const Future &f, + unsigned count) + //-------------------------------------------------------------------------- + { + enclosing->defer_dynamic_collective_arrival(dc, f, count); + } + + //-------------------------------------------------------------------------- + Future InlineContext::get_dynamic_collective_result(DynamicCollective dc) + //-------------------------------------------------------------------------- + { + return enclosing->get_dynamic_collective_result(dc); + } + + //-------------------------------------------------------------------------- + DynamicCollective InlineContext::advance_dynamic_collective( + DynamicCollective dc) + //-------------------------------------------------------------------------- + { + return enclosing->advance_dynamic_collective(dc); + } + //-------------------------------------------------------------------------- size_t InlineContext::register_new_child_operation(Operation *op, const std::vector *dependences) @@ -12536,6 +20566,13 @@ namespace Legion { return enclosing->register_new_child_operation(op, dependences); } + //-------------------------------------------------------------------------- + void InlineContext::register_new_internal_operation(InternalOp *op) + //-------------------------------------------------------------------------- + { + enclosing->register_new_internal_operation(op); + } + //-------------------------------------------------------------------------- size_t InlineContext::register_new_close_operation(CloseOp *op) //-------------------------------------------------------------------------- @@ -12551,10 +20588,11 @@ namespace Legion { } //-------------------------------------------------------------------------- - void InlineContext::add_to_dependence_queue(Operation *op, bool unordered) + ApEvent InlineContext::add_to_dependence_queue(Operation *op, + bool unordered, bool outermost) //-------------------------------------------------------------------------- { - enclosing->add_to_dependence_queue(op, unordered); + return enclosing->add_to_dependence_queue(op, unordered, outermost); } //-------------------------------------------------------------------------- @@ -12761,6 +20799,22 @@ namespace Legion { enclosing->decrement_frame(); } + //-------------------------------------------------------------------------- +#ifdef DEBUG_LEGION_COLLECTIVES + MergeCloseOp* InlineContext::get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node) +#else + MergeCloseOp* InlineContext::get_merge_close_op(void) +#endif + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION_COLLECTIVES + return enclosing->get_merge_close_op(user, node); +#else + return enclosing->get_merge_close_op(); +#endif + } + //-------------------------------------------------------------------------- InnerContext* InlineContext::find_parent_logical_context(unsigned index) //-------------------------------------------------------------------------- @@ -12823,7 +20877,7 @@ namespace Legion { //-------------------------------------------------------------------------- InstanceView* InlineContext::create_instance_top_view( - PhysicalManager *manager, AddressSpaceID source, RtEvent *ready) + PhysicalManager *manager, AddressSpaceID source) //-------------------------------------------------------------------------- { assert(false); @@ -12891,13 +20945,6 @@ namespace Legion { enclosing->find_collective_contributions(dc, contributions); } - //-------------------------------------------------------------------------- - Future InlineContext::get_dynamic_collective_result(DynamicCollective dc) - //-------------------------------------------------------------------------- - { - return enclosing->get_dynamic_collective_result(dc); - } - //-------------------------------------------------------------------------- TaskPriority InlineContext::get_current_priority(void) const //-------------------------------------------------------------------------- diff --git a/runtime/legion/legion_context.h b/runtime/legion/legion_context.h index c4f73633f7..febb0db5ea 100644 --- a/runtime/legion/legion_context.h +++ b/runtime/legion/legion_context.h @@ -81,7 +81,8 @@ namespace Legion { virtual Task* get_task(void); virtual TaskContext* find_parent_context(void); virtual void pack_remote_context(Serializer &rez, - AddressSpaceID target) = 0; + AddressSpaceID target, + bool replicate = false) = 0; virtual bool attempt_children_complete(void) = 0; virtual bool attempt_children_commit(void) = 0; virtual void inline_child_task(TaskOp *child) = 0; @@ -92,6 +93,12 @@ namespace Legion { Realm::DSOReferenceImplementation *dso, RtEvent local_done, RtEvent global_done, std::set &preconditions); virtual void handle_registration_callback_effects(RtEvent effects) = 0; + virtual void print_once(FILE *f, const char *message) const; + virtual void log_once(Realm::LoggerMessage &message) const; + virtual ShardID get_shard_id(void) const; + virtual size_t get_num_shards(void) const; + virtual Future consensus_match(const void *input, void *output, + size_t num_elements, size_t element_size); public: virtual VariantID register_variant(const TaskVariantRegistrar ®istrar, const void *user_data, size_t user_data_size, @@ -100,6 +107,7 @@ namespace Legion { virtual TraceID generate_dynamic_trace_id(void); virtual MapperID generate_dynamic_mapper_id(void); virtual ProjectionID generate_dynamic_projection_id(void); + virtual ShardingID generate_dynamic_sharding_id(void); virtual TaskID generate_dynamic_task_id(void); virtual ReductionOpID generate_dynamic_reduction_id(void); virtual CustomSerdezID generate_dynamic_serdez_id(void); @@ -182,6 +190,13 @@ namespace Legion { size_t extent_size, PartitionKind part_kind, Color color) = 0; + virtual IndexPartition create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color) = 0; virtual IndexPartition create_partition_by_domain( IndexSpace parent, const FutureMap &domains, @@ -241,26 +256,31 @@ namespace Legion { virtual IndexSpace create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles) = 0; virtual IndexSpace create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) = 0; virtual IndexSpace create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles) = 0; virtual IndexSpace create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle) = 0; virtual IndexSpace create_index_space_difference( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexSpace initial, const std::vector &handles) = 0; @@ -309,7 +329,8 @@ namespace Legion { virtual void create_shared_ownership(LogicalRegion handle); virtual void destroy_logical_region(LogicalRegion handle, const bool unordered) = 0; - virtual FieldAllocatorImpl* create_field_allocator(FieldSpace handle); + virtual FieldAllocatorImpl* create_field_allocator(FieldSpace handle, + bool unordered); virtual void destroy_field_allocator(FieldSpace handle); virtual void get_local_field_set(const FieldSpace handle, const std::set &indexes, @@ -326,7 +347,8 @@ namespace Legion { virtual Future reduce_future_map(const FutureMap &future_map, ReductionOpID redop, bool deterministic) = 0; virtual FutureMap construct_future_map(const Domain &domain, - const std::map &futures, + const std::map &futures, + RtUserEvent domain_deletion = RtUserEvent::NO_RT_USER_EVENT, bool internal = false) = 0; virtual PhysicalRegion map_region(const InlineLauncher &launcher) = 0; virtual ApEvent remap_region(PhysicalRegion region) = 0; @@ -353,6 +375,24 @@ namespace Legion { virtual Predicate predicate_not(const Predicate &p) = 0; virtual Predicate create_predicate(const PredicateLauncher &launcher) = 0; virtual Future get_predicate_future(const Predicate &p) = 0; + public: + // Calls for barriers and dynamic collectives + virtual ApBarrier create_phase_barrier(unsigned arrivals, + ReductionOpID redop = 0, + const void *init_value = NULL, + size_t init_size = 0) = 0; + virtual void destroy_phase_barrier(ApBarrier bar) = 0; + virtual PhaseBarrier advance_phase_barrier(PhaseBarrier bar) = 0; + virtual void arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, + unsigned count) = 0; + virtual void defer_dynamic_collective_arrival(DynamicCollective dc, + const Future &f, + unsigned count) = 0; + virtual Future get_dynamic_collective_result(DynamicCollective dc) = 0; + virtual DynamicCollective advance_dynamic_collective( + DynamicCollective dc) = 0; public: // The following set of operations correspond directly // to the complete_mapping, complete_operation, and @@ -361,10 +401,12 @@ namespace Legion { // these calls to notify the parent context. virtual size_t register_new_child_operation(Operation *op, const std::vector *dependences) = 0; + virtual void register_new_internal_operation(InternalOp *op) = 0; virtual size_t register_new_close_operation(CloseOp *op) = 0; virtual size_t register_new_summary_operation(TraceSummaryOp *op) = 0; - virtual void add_to_dependence_queue(Operation *op, - bool unordered = false) = 0; + virtual ApEvent add_to_dependence_queue(Operation *op, + bool unordered = false, + bool outermost = true) = 0; virtual void add_to_post_task_queue(TaskContext *ctx, RtEvent wait_on, const void *result, size_t size, #ifdef LEGION_MALLOC_INSTANCES @@ -412,6 +454,13 @@ namespace Legion { virtual void increment_frame(void) = 0; virtual void decrement_frame(void) = 0; public: +#ifdef DEBUG_LEGION_COLLECTIVES + virtual MergeCloseOp* get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node) = 0; +#else + virtual MergeCloseOp* get_merge_close_op(void) = 0; +#endif + public: virtual InnerContext* find_parent_logical_context(unsigned index) = 0; virtual InnerContext* find_parent_physical_context(unsigned index, LogicalRegion parent) = 0; @@ -428,7 +477,7 @@ namespace Legion { virtual void send_back_created_state(AddressSpaceID target) = 0; public: virtual InstanceView* create_instance_top_view(PhysicalManager *manager, - AddressSpaceID source, RtEvent *ready = NULL) = 0; + AddressSpaceID source) = 0; public: virtual const std::vector& begin_task( Legion::Runtime *&runtime); @@ -446,7 +495,6 @@ namespace Legion { const Future &f) = 0; virtual void find_collective_contributions(DynamicCollective dc, std::vector &futures) = 0; - virtual Future get_dynamic_collective_result(DynamicCollective dc) = 0; public: virtual TaskPriority get_current_priority(void) const = 0; virtual void set_current_priority(TaskPriority priority) = 0; @@ -456,7 +504,7 @@ namespace Legion { public: void add_created_region(LogicalRegion handle, bool task_local); // for logging created region requirements - void log_created_requirements(void); + void log_created_requirements(void); public: void register_region_creation(LogicalRegion handle, bool task_local); public: @@ -791,16 +839,15 @@ namespace Legion { std::vector > &deleted_partitions, std::set &preconditions); protected: - // Deletions are virtual so they can be overridden for control replication void register_region_creations( std::map ®ions); - virtual void register_region_deletions(ApEvent precondition, + void register_region_deletions(ApEvent precondition, const std::map &dependences, std::vector ®ions, std::set &preconditions); void register_field_creations( std::set > &fields); - virtual void register_field_deletions(ApEvent precondition, + void register_field_deletions(ApEvent precondition, const std::map &dependences, std::vector > &fields, std::set &preconditions); @@ -808,19 +855,19 @@ namespace Legion { std::map &spaces); void register_latent_field_spaces( std::map > &spaces); - virtual void register_field_space_deletions(ApEvent precondition, + void register_field_space_deletions(ApEvent precondition, const std::map &dependences, std::vector &spaces, std::set &preconditions); void register_index_space_creations( std::map &spaces); - virtual void register_index_space_deletions(ApEvent precondition, + void register_index_space_deletions(ApEvent precondition, const std::map &dependences, std::vector > &spaces, std::set &preconditions); void register_index_partition_creations( std::map &parts); - virtual void register_index_partition_deletions(ApEvent precondition, + void register_index_partition_deletions(ApEvent precondition, const std::map &dependences, std::vector > &parts, std::set &preconditions); @@ -835,7 +882,8 @@ namespace Legion { virtual ContextID get_context_id(void) const; virtual UniqueID get_context_uid(void) const; virtual bool is_inner_context(void) const; - virtual void pack_remote_context(Serializer &rez, AddressSpaceID target); + virtual void pack_remote_context(Serializer &rez, + AddressSpaceID target, bool replicate = false); virtual void unpack_remote_context(Deserializer &derez, std::set &preconditions); virtual RtEvent compute_equivalence_sets(VersionManager *manager, @@ -917,6 +965,13 @@ namespace Legion { size_t extent_size, PartitionKind part_kind, Color color); + virtual IndexPartition create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color); virtual IndexPartition create_partition_by_domain( IndexSpace parent, const FutureMap &domains, @@ -976,32 +1031,41 @@ namespace Legion { virtual IndexSpace create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles); virtual IndexSpace create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle); virtual IndexSpace create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles); virtual IndexSpace create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle); virtual IndexSpace create_index_space_difference( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexSpace initial, const std::vector &handles); virtual void verify_partition(IndexPartition pid, PartitionKind kind, const char *function_name); static void handle_partition_verification(const void *args); + virtual FieldSpace create_field_space(void); + virtual FieldSpace create_field_space(const std::vector &sizes, + std::vector &resulting_fields, + CustomSerdezID serdez_id); virtual FieldSpace create_field_space(const std::vector &sizes, std::vector &resulting_fields, CustomSerdezID serdez_id); @@ -1035,6 +1099,7 @@ namespace Legion { const std::set &indexes, std::vector &to_set) const; public: + // Find an index space name for a concrete launch domain virtual Future execute_task(const TaskLauncher &launcher); virtual FutureMap execute_index_space(const IndexTaskLauncher &launcher); virtual Future execute_index_space(const IndexTaskLauncher &launcher, @@ -1042,7 +1107,8 @@ namespace Legion { virtual Future reduce_future_map(const FutureMap &future_map, ReductionOpID redop, bool deterministic); virtual FutureMap construct_future_map(const Domain &domain, - const std::map &futures, + const std::map &futures, + RtUserEvent domain_deletion = RtUserEvent::NO_RT_USER_EVENT, bool internal = false); virtual PhysicalRegion map_region(const InlineLauncher &launcher); virtual ApEvent remap_region(PhysicalRegion region); @@ -1066,6 +1132,24 @@ namespace Legion { virtual Predicate predicate_not(const Predicate &p); virtual Predicate create_predicate(const PredicateLauncher &launcher); virtual Future get_predicate_future(const Predicate &p); + public: + // Calls for barriers and dynamic collectives + virtual ApBarrier create_phase_barrier(unsigned arrivals, + ReductionOpID redop = 0, + const void *init_value = NULL, + size_t init_size = 0); + virtual void destroy_phase_barrier(ApBarrier bar); + virtual PhaseBarrier advance_phase_barrier(PhaseBarrier bar); + virtual void arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, + unsigned count); + virtual void defer_dynamic_collective_arrival(DynamicCollective dc, + const Future &f, + unsigned count); + virtual Future get_dynamic_collective_result(DynamicCollective dc); + virtual DynamicCollective advance_dynamic_collective( + DynamicCollective dc); public: // The following set of operations correspond directly // to the complete_mapping, complete_operation, and @@ -1074,12 +1158,17 @@ namespace Legion { // these calls to notify the parent context. virtual size_t register_new_child_operation(Operation *op, const std::vector *dependences); + virtual void register_new_internal_operation(InternalOp *op); + // Must be called while holding the dependence lock + virtual void insert_unordered_ops(AutoLock &d_lock, const bool end_task, + const bool progress); virtual size_t register_new_close_operation(CloseOp *op); virtual size_t register_new_summary_operation(TraceSummaryOp *op); void add_to_prepipeline_queue(Operation *op); bool process_prepipeline_stage(void); - virtual void add_to_dependence_queue(Operation *op, - bool unordered = false); + virtual ApEvent add_to_dependence_queue(Operation *op, + bool unordered = false, + bool outermost = true); void process_dependence_stage(void); virtual void add_to_post_task_queue(TaskContext *ctx, RtEvent wait_on, const void *result, size_t size, @@ -1125,6 +1214,13 @@ namespace Legion { virtual void increment_frame(void); virtual void decrement_frame(void); public: +#ifdef DEBUG_LEGION_COLLECTIVES + virtual MergeCloseOp* get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node); +#else + virtual MergeCloseOp* get_merge_close_op(void); +#endif + public: virtual InnerContext* find_parent_logical_context(unsigned index); virtual InnerContext* find_parent_physical_context(unsigned index, LogicalRegion parent); @@ -1140,11 +1236,12 @@ namespace Legion { const std::vector &unmap_events, std::set &applied_events); virtual void invalidate_region_tree_contexts(void); + void invalidate_created_requirement_contexts(void); virtual void invalidate_remote_tree_contexts(Deserializer &derez); virtual void send_back_created_state(AddressSpaceID target); public: virtual InstanceView* create_instance_top_view(PhysicalManager *manager, - AddressSpaceID source, RtEvent *ready = NULL); + AddressSpaceID source); virtual FillView* find_or_create_fill_view(FillOp *op, std::set &map_applied_events, const void *value, const size_t value_size); @@ -1168,7 +1265,8 @@ namespace Legion { const Future &f); virtual void find_collective_contributions(DynamicCollective dc, std::vector &contributions); - virtual Future get_dynamic_collective_result(DynamicCollective dc); + public: + virtual ShardingFunction* find_sharding_function(ShardingID sid); public: virtual TaskPriority get_current_priority(void) const; virtual void set_current_priority(TaskPriority priority); @@ -1176,9 +1274,8 @@ namespace Legion { static void handle_compute_equivalence_sets_request(Deserializer &derez, Runtime *runtime, AddressSpaceID source); public: - static void handle_prepipeline_stage(const void *args); - static void handle_dependence_stage(const void *args); - static void handle_post_end_task(const void *args); + void invalidate_remote_contexts(void); + void clear_instance_top_views(void); public: void free_remote_contexts(void); void send_remote_context(AddressSpaceID remote_instance, @@ -1193,8 +1290,7 @@ namespace Legion { void execute_task_launch(TaskOp *task, bool index, LegionTrace *current_trace, bool silence_warnings, bool inlining_enabled); - // Must be called while holding the dependence lock - void insert_unordered_ops(AutoLock &d_lock); + EquivalenceSet* find_or_create_top_equivalence_set(RegionTreeID tree_id); public: void clone_local_fields( std::map > &child_local) const; @@ -1204,6 +1300,10 @@ namespace Legion { // which is especially useful when debugging scheduler hangs Operation* get_earliest(void) const; #endif + public: + static void handle_prepipeline_stage(const void *args); + static void handle_dependence_stage(const void *args); + static void handle_post_end_task(const void *args); public: const RegionTreeContext tree_context; const UniqueID context_uid; @@ -1230,7 +1330,7 @@ namespace Legion { // For tracking any operations that come from outside the // task like a garbage collector that need to be inserted // into the stream of operations from the task - std::vector unordered_ops; + std::list unordered_ops; #ifdef DEBUG_LEGION // In debug mode also keep track of them in context order so // we can see what the longest outstanding operation is which @@ -1341,7 +1441,8 @@ namespace Legion { public: TopLevelContext& operator=(const TopLevelContext &rhs); public: - virtual void pack_remote_context(Serializer &rez, AddressSpaceID target); + virtual void pack_remote_context(Serializer &rez, + AddressSpaceID target, bool replicate = false); virtual TaskContext* find_parent_context(void); public: virtual InnerContext* find_outermost_local_context( @@ -1358,6 +1459,726 @@ namespace Legion { std::vector dummy_mapped; }; + /** + * \class ReplicateContext + * A replicate context is a special kind of inner context for + * executing control-replicated tasks. + */ + class ReplicateContext : public InnerContext { + public: + struct ISBroadcast { + public: + ISBroadcast(void) : expr_id(0), did(0), double_buffer(false) { } + ISBroadcast(IndexSpaceID i, IndexTreeID t, IndexSpaceExprID e, + DistributedID d, bool db) + : space_id(i), tid(t), expr_id(e), did(d), double_buffer(db) { } + public: + IndexSpaceID space_id; + IndexTreeID tid; + IndexSpaceExprID expr_id; + DistributedID did; + bool double_buffer; + }; + struct IPBroadcast { + public: + IPBroadcast(void) : did(0), double_buffer(false) { } + IPBroadcast(IndexPartitionID p, DistributedID d, bool db) + : pid(p), did(d), double_buffer(db) { } + public: + IndexPartitionID pid; + DistributedID did; + bool double_buffer; + }; + struct FSBroadcast { + public: + FSBroadcast(void) : did(0), double_buffer(false) { } + FSBroadcast(FieldSpaceID i, DistributedID d, bool db) + : space_id(i), did(d), double_buffer(db) { } + public: + FieldSpaceID space_id; + DistributedID did; + bool double_buffer; + }; + struct FIDBroadcast { + public: + FIDBroadcast(void) : field_id(0), double_buffer(false) { } + FIDBroadcast(FieldID fid, bool db) + : field_id(fid), double_buffer(db) { } + public: + FieldID field_id; + bool double_buffer; + }; + struct LRBroadcast { + public: + LRBroadcast(void) : tid(0), double_buffer(0) { } + LRBroadcast(RegionTreeID t, bool db) : + tid(t), double_buffer(db) { } + public: + RegionTreeID tid; + bool double_buffer; + }; + struct IntraSpaceDeps { + public: + std::map ready_deps; + std::map pending_deps; + }; + enum ReplicateAPICall { + REPLICATE_PERFORM_REGISTRATION_CALLBACK, + REPLICATE_CONSENSUS_MATCH, + REPLICATE_REGISTER_TASK_VARIANT, + REPLICATE_GENERATE_DYNAMIC_TRACE_ID, + REPLICATE_GENERATE_DYNAMIC_MAPPER_ID, + REPLICATE_GENERATE_DYNAMIC_PROJECTION_ID, + REPLICATE_GENERATE_DYNAMIC_SHARDING_ID, + REPLICATE_GENERATE_DYNAMIC_TASK_ID, + REPLICATE_GENERATE_DYNAMIC_REDUCTION_ID, + REPLICATE_GENERATE_DYNAMIC_SERDEZ_ID, + REPLICATE_CREATE_INDEX_SPACE, + REPLICATE_UNION_INDEX_SPACES, + REPLICATE_INTERSECT_INDEX_SPACES, + REPLICATE_SUBTRACT_INDEX_SPACES, + REPLICATE_CREATE_SHARED_OWNERSHIP, + REPLICATE_DESTROY_INDEX_SPACE, + REPLICATE_DESTROY_INDEX_PARTITION, + REPLICATE_CREATE_EQUAL_PARTITION, + REPLICATE_CREATE_PARTITION_BY_WEIGHTS, + REPLICATE_CREATE_PARTITION_BY_UNION, + REPLICATE_CREATE_PARTITION_BY_INTERSECTION, + REPLICATE_CREATE_PARTITION_BY_DIFFERENCE, + REPLICATE_CREATE_CROSS_PRODUCT_PARTITIONS, + REPLICATE_CREATE_ASSOCIATION, + REPLICATE_CREATE_RESTRICTED_PARTITION, + REPLICATE_CREATE_PARTITION_BY_DOMAIN, + REPLICATE_CREATE_PARTITION_BY_FIELD, + REPLICATE_CREATE_PARTITION_BY_IMAGE, + REPLICATE_CREATE_PARTITION_BY_IMAGE_RANGE, + REPLICATE_CREATE_PARTITION_BY_PREIMAGE, + REPLICATE_CREATE_PARTITION_BY_PREIMAGE_RANGE, + REPLICATE_CREATE_PENDING_PARTITION, + REPLICATE_CREATE_INDEX_SPACE_UNION, + REPLICATE_CREATE_INDEX_SPACE_INTERSECTION, + REPLICATE_CREATE_INDEX_SPACE_DIFFERENCE, + REPLICATE_CREATE_FIELD_SPACE, + REPLICATE_DESTROY_FIELD_SPACE, + REPLICATE_ALLOCATE_FIELD, + REPLICATE_FREE_FIELD, + REPLICATE_ALLOCATE_FIELDS, + REPLICATE_FREE_FIELDS, + REPLICATE_CREATE_LOGICAL_REGION, + REPLICATE_DESTROY_LOGICAL_REGION, + REPLICATE_CREATE_FIELD_ALLOCATOR, + REPLICATE_DESTROY_FIELD_ALLOCATOR, + }; + public: + ReplicateContext(Runtime *runtime, ShardTask *owner,int d,bool full_inner, + const std::vector &reqs, + const std::vector &parent_indexes, + const std::vector &virt_mapped, + UniqueID context_uid, ApEvent execution_fence_event, + ShardManager *manager); + ReplicateContext(const ReplicateContext &rhs); + virtual ~ReplicateContext(void); + public: + ReplicateContext& operator=(const ReplicateContext &rhs); + public: + inline int get_shard_collective_radix(void) const + { return shard_collective_radix; } + inline int get_shard_collective_log_radix(void) const + { return shard_collective_log_radix; } + inline int get_shard_collective_stages(void) const + { return shard_collective_stages; } + inline int get_shard_collective_participating_shards(void) const + { return shard_collective_participating_shards; } + inline int get_shard_collective_last_radix(void) const + { return shard_collective_last_radix; } + public: // Privilege tracker methods + virtual void receive_resources(size_t return_index, + std::map &created_regions, + std::vector &deleted_regions, + std::set > &created_fields, + std::vector > &deleted_fields, + std::map &created_field_spaces, + std::map > &latent_spaces, + std::vector &deleted_field_spaces, + std::map &created_index_spaces, + std::vector > &deleted_index_spaces, + std::map &created_partitions, + std::vector > &deleted_partitions, + std::set &preconditions); + protected: + void receive_replicate_resources(size_t return_index, + std::map &created_regions, + std::vector &deleted_regions, + std::set > &created_fields, + std::vector > &deleted_fields, + std::map &created_field_spaces, + std::map > &latent_spaces, + std::vector &deleted_field_spaces, + std::map &created_index_spaces, + std::vector > &deleted_index_spaces, + std::map &created_partitions, + std::vector > &deleted_partitions, + std::set &preconditions, RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, RtBarrier &execution_barrier); + void register_region_deletions(ApEvent precondition, + const std::map &dependences, + std::vector ®ions, + std::set &preconditions, RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, RtBarrier &execution_barrier); + void register_field_deletions(ApEvent precondition, + const std::map &dependences, + std::vector > &fields, + std::set &preconditions, RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, RtBarrier &execution_barrier); + void register_field_space_deletions(ApEvent precondition, + const std::map &dependences, + std::vector &spaces, + std::set &preconditions, RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, RtBarrier &execution_barrier); + void register_index_space_deletions(ApEvent precondition, + const std::map &dependences, + std::vector > &spaces, + std::set &preconditions, RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, RtBarrier &execution_barrier); + void register_index_partition_deletions(ApEvent precondition, + const std::map &dependences, + std::vector > &parts, + std::set &preconditions, RtBarrier &ready_barrier, + RtBarrier &mapped_barrier, RtBarrier &execution_barrier); + public: + void perform_replicated_region_deletions( + std::vector ®ions, + std::set &preconditions); + void perform_replicated_field_deletions( + std::vector > &fields, + std::set &preconditions); + void perform_replicated_field_space_deletions( + std::vector &spaces, + std::set &preconditions); + void perform_replicated_index_space_deletions( + std::vector &spaces, + std::set &preconditions); + void perform_replicated_index_partition_deletions( + std::vector &parts, + std::set &preconditions); + public: + virtual void perform_global_registration_callbacks( + Realm::DSOReferenceImplementation *dso, RtEvent local_done, + RtEvent global_done, std::set &preconditions); + virtual void handle_registration_callback_effects(RtEvent effects); + virtual void print_once(FILE *f, const char *message) const; + virtual void log_once(Realm::LoggerMessage &message) const; + virtual ShardID get_shard_id(void) const; + virtual size_t get_num_shards(void) const; + virtual Future consensus_match(const void *input, void *output, + size_t num_elements, size_t element_size); + public: + virtual VariantID register_variant(const TaskVariantRegistrar ®istrar, + const void *user_data, size_t user_data_size, + const CodeDescriptor &desc, bool ret, + VariantID vid, bool check_task_id); + virtual TraceID generate_dynamic_trace_id(void); + virtual MapperID generate_dynamic_mapper_id(void); + virtual ProjectionID generate_dynamic_projection_id(void); + virtual ShardingID generate_dynamic_sharding_id(void); + virtual TaskID generate_dynamic_task_id(void); + virtual ReductionOpID generate_dynamic_reduction_id(void); + virtual CustomSerdezID generate_dynamic_serdez_id(void); + virtual bool perform_semantic_attach(bool &global); + virtual void post_semantic_attach(void); + public: + virtual InnerContext* find_parent_physical_context(unsigned index, + LogicalRegion handle); + virtual void invalidate_region_tree_contexts(void); + public: + virtual RtEvent compute_equivalence_sets(VersionManager *manager, + RegionTreeID tree_id, IndexSpace handle, + IndexSpaceExpression *expr, const FieldMask &mask, + AddressSpaceID source); + // Interface to operations performed by a context + virtual IndexSpace create_index_space(const Domain &domain, + TypeTag type_tag); + virtual IndexSpace create_index_space(const Future &future, + TypeTag type_tag); + virtual IndexSpace union_index_spaces( + const std::vector &spaces); + virtual IndexSpace intersect_index_spaces( + const std::vector &spaces); + virtual IndexSpace subtract_index_spaces( + IndexSpace left, IndexSpace right); + virtual void create_shared_ownership(IndexSpace handle); + virtual void destroy_index_space(IndexSpace handle, + const bool unordered, + const bool recurse); + virtual void create_shared_ownership(IndexPartition handle); + virtual void destroy_index_partition(IndexPartition handle, + const bool unordered, + const bool recurse); + virtual IndexPartition create_equal_partition( + IndexSpace parent, + IndexSpace color_space, + size_t granularity, + Color color); + virtual IndexPartition create_partition_by_weights(IndexSpace parent, + const FutureMap &weights, + IndexSpace color_space, + size_t granularity, + Color color); + virtual IndexPartition create_partition_by_union( + IndexSpace parent, + IndexPartition handle1, + IndexPartition handle2, + IndexSpace color_space, + PartitionKind kind, + Color color); + virtual IndexPartition create_partition_by_intersection( + IndexSpace parent, + IndexPartition handle1, + IndexPartition handle2, + IndexSpace color_space, + PartitionKind kind, + Color color); + virtual IndexPartition create_partition_by_intersection( + IndexSpace parent, + IndexPartition partition, + PartitionKind kind, + Color color, + bool dominates); + virtual IndexPartition create_partition_by_difference( + IndexSpace parent, + IndexPartition handle1, + IndexPartition handle2, + IndexSpace color_space, + PartitionKind kind, + Color color); + virtual Color create_cross_product_partitions( + IndexPartition handle1, + IndexPartition handle2, + std::map &handles, + PartitionKind kind, + Color color); + virtual void create_association( LogicalRegion domain, + LogicalRegion domain_parent, + FieldID domain_fid, + IndexSpace range, + MapperID id, MappingTagID tag); + virtual IndexPartition create_restricted_partition( + IndexSpace parent, + IndexSpace color_space, + const void *transform, + size_t transform_size, + const void *extent, + size_t extent_size, + PartitionKind part_kind, + Color color); + virtual IndexPartition create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color); + virtual IndexPartition create_partition_by_domain( + IndexSpace parent, + const FutureMap &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color); + virtual IndexPartition create_partition_by_field( + LogicalRegion handle, + LogicalRegion parent_priv, + FieldID fid, + IndexSpace color_space, + Color color, + MapperID id, MappingTagID tag, + PartitionKind part_kind); + virtual IndexPartition create_partition_by_image( + IndexSpace handle, + LogicalPartition projection, + LogicalRegion parent, + FieldID fid, + IndexSpace color_space, + PartitionKind part_kind, + Color color, + MapperID id, MappingTagID tag); + virtual IndexPartition create_partition_by_image_range( + IndexSpace handle, + LogicalPartition projection, + LogicalRegion parent, + FieldID fid, + IndexSpace color_space, + PartitionKind part_kind, + Color color, + MapperID id, MappingTagID tag); + virtual IndexPartition create_partition_by_preimage( + IndexPartition projection, + LogicalRegion handle, + LogicalRegion parent, + FieldID fid, + IndexSpace color_space, + PartitionKind part_kind, + Color color, + MapperID id, MappingTagID tag); + virtual IndexPartition create_partition_by_preimage_range( + IndexPartition projection, + LogicalRegion handle, + LogicalRegion parent, + FieldID fid, + IndexSpace color_space, + PartitionKind part_kind, + Color color, + MapperID id, MappingTagID tag); + virtual IndexPartition create_pending_partition( + IndexSpace parent, + IndexSpace color_space, + PartitionKind part_kind, + Color color); + virtual IndexSpace create_index_space_union( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + const std::vector &handles); + virtual IndexSpace create_index_space_union( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + IndexPartition handle); + virtual IndexSpace create_index_space_intersection( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + const std::vector &handles); + virtual IndexSpace create_index_space_intersection( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + IndexPartition handle); + virtual IndexSpace create_index_space_difference( + IndexPartition parent, + const void *realm_color, + size_t color_size, + TypeTag type_tag, + IndexSpace initial, + const std::vector &handles); + virtual void verify_partition(IndexPartition pid, PartitionKind kind, + const char *function_name); + virtual FieldSpace create_field_space(void); + virtual FieldSpace create_field_space(const std::vector &sizes, + std::vector &resulting_fields, + CustomSerdezID serdez_id); + virtual FieldSpace create_field_space(const std::vector &sizes, + std::vector &resulting_fields, + CustomSerdezID serdez_id); + virtual void create_shared_ownership(FieldSpace handle); + virtual void destroy_field_space(FieldSpace handle, const bool unordered); + virtual FieldID allocate_field(FieldSpace space, size_t field_size, + FieldID fid, bool local, + CustomSerdezID serdez_id); + virtual FieldID allocate_field(FieldSpace space, const Future &field_size, + FieldID fid, bool local, + CustomSerdezID serdez_id); + virtual void free_field(FieldAllocatorImpl *allocator, FieldSpace space, + FieldID fid, const bool unordered); + virtual void allocate_fields(FieldSpace space, + const std::vector &sizes, + std::vector &resuling_fields, + bool local, CustomSerdezID serdez_id); + virtual void allocate_fields(FieldSpace space, + const std::vector &sizes, + std::vector &resuling_fields, + bool local, CustomSerdezID serdez_id); + virtual void free_fields(FieldAllocatorImpl *allocator, FieldSpace space, + const std::set &to_free, + const bool unordered); + virtual LogicalRegion create_logical_region(RegionTreeForest *forest, + IndexSpace index_space, + FieldSpace field_space, + bool task_local); + virtual void create_shared_ownership(LogicalRegion handle); + virtual void destroy_logical_region(LogicalRegion handle, + const bool unordered); + public: + virtual FieldAllocatorImpl* create_field_allocator(FieldSpace handle, + bool unordered); + virtual void destroy_field_allocator(FieldSpace handle); + public: + virtual void insert_unordered_ops(AutoLock &d_lock, const bool end_task, + const bool progress); + virtual Future execute_task(const TaskLauncher &launcher); + virtual FutureMap execute_index_space(const IndexTaskLauncher &launcher); + virtual Future execute_index_space(const IndexTaskLauncher &launcher, + ReductionOpID redop, bool deterministic); + virtual Future reduce_future_map(const FutureMap &future_map, + ReductionOpID redop, bool deterministic); + virtual PhysicalRegion map_region(const InlineLauncher &launcher); + virtual ApEvent remap_region(PhysicalRegion region); + // Unmapping region is the same as for an inner context + virtual void fill_fields(const FillLauncher &launcher); + virtual void fill_fields(const IndexFillLauncher &launcher); + virtual void issue_copy(const CopyLauncher &launcher); + virtual void issue_copy(const IndexCopyLauncher &launcher); + virtual void issue_acquire(const AcquireLauncher &launcher); + virtual void issue_release(const ReleaseLauncher &launcher); + virtual PhysicalRegion attach_resource(const AttachLauncher &launcher); + virtual Future detach_resource(PhysicalRegion region, const bool flush, + const bool unordered); + virtual FutureMap execute_must_epoch(const MustEpochLauncher &launcher); + virtual Future issue_timing_measurement(const TimingLauncher &launcher); + virtual Future issue_mapping_fence(void); + virtual Future issue_execution_fence(void); + virtual void begin_trace(TraceID tid, bool logical_only, + bool static_trace, const std::set *managed, bool dep); + virtual void end_trace(TraceID tid, bool deprecated); + virtual ApEvent add_to_dependence_queue(Operation *op, + bool unordered = false, + bool outermost = true); + public: + virtual void record_dynamic_collective_contribution(DynamicCollective dc, + const Future &f); + virtual void find_collective_contributions(DynamicCollective dc, + std::vector &contributions); + public: + // Calls for barriers and dynamic collectives + virtual ApBarrier create_phase_barrier(unsigned arrivals, + ReductionOpID redop = 0, + const void *init_value = NULL, + size_t init_size = 0); + virtual void destroy_phase_barrier(ApBarrier bar); + virtual PhaseBarrier advance_phase_barrier(PhaseBarrier bar); + virtual void arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, + unsigned count); + virtual void defer_dynamic_collective_arrival(DynamicCollective dc, + const Future &f, + unsigned count); + virtual Future get_dynamic_collective_result(DynamicCollective dc); + virtual DynamicCollective advance_dynamic_collective( + DynamicCollective dc); + public: +#ifdef DEBUG_LEGION_COLLECTIVES + virtual MergeCloseOp* get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node); +#else + virtual MergeCloseOp* get_merge_close_op(void); +#endif + public: + virtual void pack_remote_context(Serializer &rez, + AddressSpaceID target, + bool replicate = false); + public: + virtual ShardingFunction* find_sharding_function(ShardingID sid); + public: + virtual InstanceView* create_instance_top_view(PhysicalManager *manager, + AddressSpaceID source); + InstanceView* create_replicate_instance_top_view(PhysicalManager *manager, + AddressSpaceID source); + void record_replicate_instance_top_view(PhysicalManager *manager, + InstanceView *result); + public: + void exchange_common_resources(void); + void handle_collective_message(Deserializer &derez); + void handle_future_map_request(Deserializer &derez); + void handle_equivalence_set_request(Deserializer &derez); + void handle_equivalence_set_response(RegionTreeID tree_id, + EquivalenceSet *result); + static void handle_eq_response(Deserializer &derez, Runtime *rt); + void handle_resource_update(Deserializer &derez, + std::set &applied); + void handle_trace_update(Deserializer &derez, AddressSpaceID source); + ApBarrier handle_find_trace_shard_event(size_t temp_index, ApEvent event, + ShardID remote_shard); + void record_intra_space_dependence(size_t context_index, + const DomainPoint &point, RtEvent point_mapped, ShardID next_shard); + void handle_intra_space_dependence(Deserializer &derez); + public: + void increase_pending_index_spaces(unsigned count, bool double_buffer); + void increase_pending_partitions(unsigned count, bool double_buffer); + void increase_pending_field_spaces(unsigned count, bool double_buffer); + void increase_pending_fields(unsigned count, bool double_buffer); + void increase_pending_region_trees(unsigned count, bool double_buffer); + bool create_shard_partition(IndexPartition &pid, + IndexSpace parent, IndexSpace color_space, PartitionKind part_kind, + LegionColor partition_color, bool color_generated, + ValueBroadcast *disjoint_result = NULL, + ApBarrier partition_ready = ApBarrier::NO_AP_BARRIER); + public: + // Collective methods + CollectiveID get_next_collective_index(CollectiveIndexLocation loc); + void register_collective(ShardCollective *collective); + ShardCollective* find_or_buffer_collective(Deserializer &derez); + void unregister_collective(ShardCollective *collective); + public: + // Future map methods + unsigned peek_next_future_map_barrier_index(void) const; + RtBarrier get_next_future_map_barrier(void); + void register_future_map(ReplFutureMapImpl *map); + ReplFutureMapImpl* find_or_buffer_future_map_request(Deserializer &derez); + void unregister_future_map(ReplFutureMapImpl *map); + public: + // Physical template methods + size_t register_trace_template(ShardedPhysicalTemplate *phy_template); + ShardedPhysicalTemplate* find_or_buffer_trace_update(Deserializer &derez, + AddressSpaceID source); + void unregister_trace_template(size_t template_index); + public: + // Fence barrier methods + RtBarrier get_next_mapping_fence_barrier(void); + ApBarrier get_next_execution_fence_barrier(void); + RtBarrier get_next_trace_recording_barrier(void); + RtBarrier get_next_summary_fence_barrier(void); + inline void advance_replicate_barrier(RtBarrier &bar, size_t arrivals) + { + Runtime::advance_barrier(bar); + if (!bar.exists()) + create_new_replicate_barrier(bar, arrivals); + } + inline void advance_replicate_barrier(ApBarrier &bar, size_t arrivals) + { + Runtime::advance_barrier(bar); + if (!bar.exists()) + create_new_replicate_barrier(bar, arrivals); + } + protected: + // These can only be called inside the task for this context + // since they assume that all the shards are aligned and doing + // the same calls for the same operations in the same order + void create_new_replicate_barrier(RtBarrier &bar, size_t arrivals); + void create_new_replicate_barrier(ApBarrier &bar, size_t arrivals); + public: + void verify_replicable(Murmur3Hasher &hasher, const char *func_name); + public: + // A little help for ConsensusMatchExchange since it is templated + static void help_complete_future(Future &f, const void *ptr, + size_t size, bool own); + public: + ShardTask *const owner_shard; + ShardManager *const shard_manager; + const size_t total_shards; + protected: + // These barriers are used to identify when close operations are mapped + std::vector close_mapped_barriers; + unsigned next_close_mapped_bar_index; + // These barriers are for signaling when indirect copies are done + std::vector indirection_barriers; + unsigned next_indirection_bar_index; + // These barriers are used for signaling when future maps can be reclaimed + std::vector future_map_barriers; + unsigned next_future_map_bar_index; + protected: + std::map,IntraSpaceDeps> intra_space_deps; + protected: + // Store the global owner shard and local owner shard for allocation + std::map > field_allocator_owner_shards; + protected: + ShardID index_space_allocator_shard; + ShardID index_partition_allocator_shard; + ShardID field_space_allocator_shard; + ShardID field_allocator_shard; + ShardID logical_region_allocator_shard; + ShardID dynamic_id_allocator_shard; + protected: + ApBarrier pending_partition_barrier; + RtBarrier creation_barrier; + RtBarrier deletion_ready_barrier; + RtBarrier deletion_mapping_barrier; + RtBarrier deletion_execution_barrier; + RtBarrier inline_mapping_barrier; + RtBarrier external_resource_barrier; + RtBarrier mapping_fence_barrier; + RtBarrier trace_recording_barrier; + RtBarrier summary_fence_barrier; + ApBarrier execution_fence_barrier; + ApBarrier attach_broadcast_barrier; + ApBarrier attach_reduce_barrier; + RtBarrier dependent_partition_barrier; + RtBarrier semantic_attach_barrier; + ApBarrier inorder_barrier; +#ifdef DEBUG_LEGION_COLLECTIVES + protected: + RtBarrier collective_check_barrier; + RtBarrier close_check_barrier; +#endif + protected: + // local barriers to this context for handling returned + // resources from sub-tasks + RtBarrier returned_resource_ready_barrier; + RtBarrier returned_resource_mapped_barrier; + RtBarrier returned_resource_execution_barrier; + protected: + int shard_collective_radix; + int shard_collective_log_radix; + int shard_collective_stages; + int shard_collective_participating_shards; + int shard_collective_last_radix; + protected: + mutable LocalLock replication_lock; + CollectiveID next_available_collective_index; + std::map collectives; + std::map > > pending_collective_updates; + // Use this for creating new summary barriers in the dependence + // analsyis stage of the pipeline. Our use of this variable is only + // safe as long as we know we have more generations of phase barriers + // than we can have outstanding replays, which is usually very true + volatile CollectiveID trace_recording_collective_id; + volatile CollectiveID summary_collective_id; + protected: + // Pending allocations of various resources + std::deque*,bool> > + pending_index_spaces; + std::deque*,ShardID> > + pending_index_partitions; + std::deque*,bool> > + pending_field_spaces; + std::deque*,bool> > + pending_fields; + std::deque*,bool> > + pending_region_trees; + protected: + std::map future_maps; + std::map > > pending_future_map_requests; + protected: + std::map physical_templates; + struct PendingTemplateUpdate { + public: + PendingTemplateUpdate(void) + : ptr(NULL), size(0), source(0) { } + PendingTemplateUpdate(void *p, size_t s, AddressSpaceID src) + : ptr(p), size(s), source(src) { } + public: + void *ptr; + size_t size; + AddressSpaceID source; + }; + std::map > pending_template_updates; + size_t next_physical_template_index; + protected: + // Different from pending_top_views as this applies to our requests + std::map pending_request_views; + std::map pending_tree_requests; + protected: + std::map,RtBarrier> ready_clone_barriers; + std::map,RtUserEvent> pending_clone_barriers; + protected: + unsigned next_replicate_bar_index; + unsigned next_trace_bar_index; + unsigned next_summary_bar_index; + protected: + static const unsigned MIN_UNORDERED_OPS_EPOCH = 32; + static const unsigned MAX_UNORDERED_OPS_EPOCH = 32768; + unsigned unordered_ops_counter; + unsigned unordered_ops_epoch; + }; + /** * \class RemoteTask * A small helper class for giving application @@ -1446,8 +2267,13 @@ namespace Legion { AddressSpaceID source); virtual InnerContext* find_parent_physical_context(unsigned index, LogicalRegion parent); + virtual void record_using_physical_context(LogicalRegion handle); + virtual InstanceView* create_instance_top_view(PhysicalManager *manager, + AddressSpaceID source); virtual void invalidate_region_tree_contexts(void); virtual void invalidate_remote_tree_contexts(Deserializer &derez); + public: + virtual ShardingFunction* find_sharding_function(ShardingID sid); public: void unpack_local_field_update(Deserializer &derez); static void handle_local_field_update(Deserializer &derez); @@ -1463,6 +2289,7 @@ namespace Legion { protected: UniqueID parent_context_uid; TaskContext *parent_ctx; + ShardManager *shard_manager; // if we're lucky and one is already here protected: ApEvent remote_completion_event; bool top_level_context; @@ -1475,6 +2302,11 @@ namespace Legion { std::map physical_contexts; std::map pending_physical_contexts; std::set local_physical_contexts; + protected: + // For remote replicate contexts + size_t total_shards; + ReplicationID repl_id; + std::map sharding_functions; }; /** @@ -1507,7 +2339,7 @@ namespace Legion { virtual RegionTreeContext get_context(void) const; virtual ContextID get_context_id(void) const; virtual void pack_remote_context(Serializer &rez, - AddressSpaceID target); + AddressSpaceID target, bool replicate = false); virtual bool attempt_children_complete(void); virtual bool attempt_children_commit(void); virtual void inline_child_task(TaskOp *child); @@ -1580,6 +2412,13 @@ namespace Legion { size_t extent_size, PartitionKind part_kind, Color color); + virtual IndexPartition create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color); virtual IndexPartition create_partition_by_domain( IndexSpace parent, const FutureMap &domains, @@ -1639,26 +2478,31 @@ namespace Legion { virtual IndexSpace create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles); virtual IndexSpace create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle); virtual IndexSpace create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles); virtual IndexSpace create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle); virtual IndexSpace create_index_space_difference( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexSpace initial, const std::vector &handles); @@ -1702,7 +2546,8 @@ namespace Legion { virtual Future reduce_future_map(const FutureMap &future_map, ReductionOpID redop, bool deterministic); virtual FutureMap construct_future_map(const Domain &domain, - const std::map &futures, + const std::map &futures, + RtUserEvent domain_deletion = RtUserEvent::NO_RT_USER_EVENT, bool internal = false); virtual PhysicalRegion map_region(const InlineLauncher &launcher); virtual ApEvent remap_region(PhysicalRegion region); @@ -1726,6 +2571,24 @@ namespace Legion { virtual Predicate predicate_not(const Predicate &p); virtual Predicate create_predicate(const PredicateLauncher &launcher); virtual Future get_predicate_future(const Predicate &p); + public: + // Calls for barriers and dynamic collectives + virtual ApBarrier create_phase_barrier(unsigned arrivals, + ReductionOpID redop = 0, + const void *init_value = NULL, + size_t init_size = 0); + virtual void destroy_phase_barrier(ApBarrier bar); + virtual PhaseBarrier advance_phase_barrier(PhaseBarrier bar); + virtual void arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, + unsigned count); + virtual void defer_dynamic_collective_arrival(DynamicCollective dc, + const Future &f, + unsigned count); + virtual Future get_dynamic_collective_result(DynamicCollective dc); + virtual DynamicCollective advance_dynamic_collective( + DynamicCollective dc); public: // The following set of operations correspond directly // to the complete_mapping, complete_operation, and @@ -1734,10 +2597,12 @@ namespace Legion { // these calls to notify the parent context. virtual size_t register_new_child_operation(Operation *op, const std::vector *dependences); + virtual void register_new_internal_operation(InternalOp *op); virtual size_t register_new_close_operation(CloseOp *op); virtual size_t register_new_summary_operation(TraceSummaryOp *op); - virtual void add_to_dependence_queue(Operation *op, - bool unordered = false); + virtual ApEvent add_to_dependence_queue(Operation *op, + bool unordered = false, + bool outermost = true); virtual void add_to_post_task_queue(TaskContext *ctx, RtEvent wait_on, const void *result, size_t size, #ifdef LEGION_MALLOC_INSTANCES @@ -1781,6 +2646,13 @@ namespace Legion { virtual void increment_frame(void); virtual void decrement_frame(void); public: +#ifdef DEBUG_LEGION_COLLECTIVES + virtual MergeCloseOp* get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node); +#else + virtual MergeCloseOp* get_merge_close_op(void); +#endif + public: virtual InnerContext* find_parent_logical_context(unsigned index); virtual InnerContext* find_parent_physical_context(unsigned index, LogicalRegion parent); @@ -1796,7 +2668,7 @@ namespace Legion { virtual void send_back_created_state(AddressSpaceID target); public: virtual InstanceView* create_instance_top_view(PhysicalManager *manager, - AddressSpaceID source, RtEvent *ready = NULL); + AddressSpaceID source); public: virtual void end_task(const void *res, size_t res_size, bool owned, #ifdef LEGION_MALLOC_INSTANCES @@ -1809,7 +2681,6 @@ namespace Legion { const Future &f); virtual void find_collective_contributions(DynamicCollective dc, std::vector &futures); - virtual Future get_dynamic_collective_result(DynamicCollective dc); protected: mutable LocalLock leaf_lock; std::set execution_events; @@ -1850,7 +2721,7 @@ namespace Legion { virtual ContextID get_context_id(void) const; virtual UniqueID get_context_uid(void) const; virtual void pack_remote_context(Serializer &rez, - AddressSpaceID target); + AddressSpaceID target, bool replicate); virtual bool attempt_children_complete(void); virtual bool attempt_children_commit(void); virtual void inline_child_task(TaskOp *child); @@ -1931,6 +2802,13 @@ namespace Legion { size_t extent_size, PartitionKind part_kind, Color color); + virtual IndexPartition create_partition_by_domain( + IndexSpace parent, + const std::map &domains, + IndexSpace color_space, + bool perform_intersections, + PartitionKind part_kind, + Color color); virtual IndexPartition create_partition_by_domain( IndexSpace parent, const FutureMap &domains, @@ -1990,26 +2868,31 @@ namespace Legion { virtual IndexSpace create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles); virtual IndexSpace create_index_space_union( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle); virtual IndexSpace create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, const std::vector &handles); virtual IndexSpace create_index_space_intersection( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexPartition handle); virtual IndexSpace create_index_space_difference( IndexPartition parent, const void *realm_color, + size_t color_size, TypeTag type_tag, IndexSpace initial, const std::vector &handles); @@ -2056,7 +2939,8 @@ namespace Legion { virtual void create_shared_ownership(LogicalRegion handle); virtual void destroy_logical_region(LogicalRegion handle, const bool unordered); - virtual FieldAllocatorImpl* create_field_allocator(FieldSpace handle); + virtual FieldAllocatorImpl* create_field_allocator(FieldSpace handle, + bool unordered); virtual void destroy_field_allocator(FieldSpace handle); virtual void get_local_field_set(const FieldSpace handle, const std::set &indexes, @@ -2072,7 +2956,8 @@ namespace Legion { virtual Future reduce_future_map(const FutureMap &future_map, ReductionOpID redop, bool deterministic); virtual FutureMap construct_future_map(const Domain &domain, - const std::map &futures, + const std::map &futures, + RtUserEvent domain_deletion = RtUserEvent::NO_RT_USER_EVENT, bool internal = false); virtual PhysicalRegion map_region(const InlineLauncher &launcher); virtual ApEvent remap_region(PhysicalRegion region); @@ -2096,6 +2981,24 @@ namespace Legion { virtual Predicate predicate_not(const Predicate &p); virtual Predicate create_predicate(const PredicateLauncher &launcher); virtual Future get_predicate_future(const Predicate &p); + public: + // Calls for barriers and dynamic collectives + virtual ApBarrier create_phase_barrier(unsigned arrivals, + ReductionOpID redop = 0, + const void *init_value = NULL, + size_t init_size = 0); + virtual void destroy_phase_barrier(ApBarrier bar); + virtual PhaseBarrier advance_phase_barrier(PhaseBarrier bar); + virtual void arrive_dynamic_collective(DynamicCollective dc, + const void *buffer, + size_t size, + unsigned count); + virtual void defer_dynamic_collective_arrival(DynamicCollective dc, + const Future &f, + unsigned count); + virtual Future get_dynamic_collective_result(DynamicCollective dc); + virtual DynamicCollective advance_dynamic_collective( + DynamicCollective dc); public: // The following set of operations correspond directly // to the complete_mapping, complete_operation, and @@ -2104,10 +3007,12 @@ namespace Legion { // these calls to notify the parent context. virtual size_t register_new_child_operation(Operation *op, const std::vector *dependences); + virtual void register_new_internal_operation(InternalOp *op); virtual size_t register_new_close_operation(CloseOp *op); virtual size_t register_new_summary_operation(TraceSummaryOp *op); - virtual void add_to_dependence_queue(Operation *op, - bool unordered = false); + virtual ApEvent add_to_dependence_queue(Operation *op, + bool unordered = false, + bool outermost = true); virtual void add_to_post_task_queue(TaskContext *ctx, RtEvent wait_on, const void *result, size_t size, #ifdef LEGION_MALLOC_INSTANCES @@ -2151,6 +3056,13 @@ namespace Legion { virtual void increment_frame(void); virtual void decrement_frame(void); public: +#ifdef DEBUG_LEGION_COLLECTIVES + virtual MergeCloseOp* get_merge_close_op(const LogicalUser &user, + RegionTreeNode *node); +#else + virtual MergeCloseOp* get_merge_close_op(void); +#endif + public: virtual InnerContext* find_parent_logical_context(unsigned index); virtual InnerContext* find_parent_physical_context(unsigned index, LogicalRegion parent); @@ -2167,7 +3079,7 @@ namespace Legion { virtual void send_back_created_state(AddressSpaceID target); public: virtual InstanceView* create_instance_top_view(PhysicalManager *manager, - AddressSpaceID source, RtEvent *ready = NULL); + AddressSpaceID source); public: virtual const std::vector& begin_task( Legion::Runtime *&runtime); @@ -2182,7 +3094,6 @@ namespace Legion { const Future &f); virtual void find_collective_contributions(DynamicCollective dc, std::vector &futures); - virtual Future get_dynamic_collective_result(DynamicCollective dc); public: virtual TaskPriority get_current_priority(void) const; virtual void set_current_priority(TaskPriority priority); diff --git a/runtime/legion/legion_mapping.cc b/runtime/legion/legion_mapping.cc index 2cf8e832ab..9ae4acd99d 100644 --- a/runtime/legion/legion_mapping.cc +++ b/runtime/legion/legion_mapping.cc @@ -582,12 +582,11 @@ namespace Legion { } //-------------------------------------------------------------------------- - bool MapperRuntime::is_replicable_variant(MapperContext ctx, + bool MapperRuntime::is_replicable_variant(MapperContext ctx, TaskID task_id, VariantID variant_id) const //-------------------------------------------------------------------------- { - // Will be implemented in the control replication branch - return false; + return ctx->manager->is_replicable_variant(ctx, task_id, variant_id); } //-------------------------------------------------------------------------- @@ -1374,6 +1373,36 @@ namespace Legion { ctx->manager->retrieve_name(ctx, handle, result); } + //-------------------------------------------------------------------------- + bool MapperRuntime::is_MPI_interop_configured(MapperContext ctx) + //-------------------------------------------------------------------------- + { + return ctx->manager->is_MPI_interop_configured(); + } + + //-------------------------------------------------------------------------- + const std::map& MapperRuntime::find_forward_MPI_mapping( + MapperContext ctx) + //-------------------------------------------------------------------------- + { + return ctx->manager->find_forward_MPI_mapping(ctx); + } + + //-------------------------------------------------------------------------- + const std::map& MapperRuntime::find_reverse_MPI_mapping( + MapperContext ctx) + //-------------------------------------------------------------------------- + { + return ctx->manager->find_reverse_MPI_mapping(ctx); + } + + //-------------------------------------------------------------------------- + int MapperRuntime::find_local_MPI_rank(MapperContext ctx) + //-------------------------------------------------------------------------- + { + return ctx->manager->find_local_MPI_rank(); + } + }; // namespace Mapping }; // namespace Legion diff --git a/runtime/legion/legion_mapping.h b/runtime/legion/legion_mapping.h index a6876e4ed2..47c4eeb6db 100644 --- a/runtime/legion/legion_mapping.h +++ b/runtime/legion/legion_mapping.h @@ -339,6 +339,16 @@ namespace Legion { * can opt-out of receiving the valid instance information * for a task. * + * replicate default:false + * Enable replication of the individual tasks for this + * operation. This is useful for performing redundant + * computation to avoid communication. There are + * requirements on the properties of replicated tasks + * and how they are mapped. Replicated tasks are not + * allowed to have reduction-only privileges. Furthermore + * the mapper must map any regions with write privileges + * for different copies of the task to different instances. + * * parent_priority default:current * If the mapper for the parent task permits child * operations to mutate the priority of the parent task @@ -348,11 +358,11 @@ namespace Legion { struct TaskOptions { Processor initial_proc; // = current bool inline_task; // = false - bool stealable; // = false + bool stealable; // = false bool map_locally; // = false bool valid_instances; // = true bool memoize; // = false - bool replicate; // = false + bool replicate; // = false TaskPriority parent_priority; // = current }; //------------------------------------------------------------------------ @@ -360,7 +370,7 @@ namespace Legion { const Task& task, TaskOptions& output) = 0; //------------------------------------------------------------------------ - + /** * ---------------------------------------------------------------------- * Premap Task @@ -522,19 +532,19 @@ namespace Legion { * to true. */ struct MapTaskInput { - std::vector > valid_instances; - std::vector premapped_regions; + std::vector > valid_instances; + std::vector premapped_regions; }; struct MapTaskOutput { - std::vector > chosen_instances; - std::set untracked_valid_regions; - std::vector target_procs; - VariantID chosen_variant; // = 0 - TaskPriority task_priority; // = 0 - TaskPriority profiling_priority; - ProfilingRequest task_prof_requests; - ProfilingRequest copy_prof_requests; - bool postmap_task; // = false + std::vector > chosen_instances; + std::set untracked_valid_regions; + std::vector target_procs; + VariantID chosen_variant; // = 0 + TaskPriority task_priority; // = 0 + TaskPriority profiling_priority; + ProfilingRequest task_prof_requests; + ProfilingRequest copy_prof_requests; + bool postmap_task; // = false }; //------------------------------------------------------------------------ virtual void map_task(const MapperContext ctx, @@ -543,6 +553,40 @@ namespace Legion { MapTaskOutput& output) = 0; //------------------------------------------------------------------------ + + /** + * ---------------------------------------------------------------------- + * Map Replicate Task + * ---------------------------------------------------------------------- + * This mapper call is invoked instead of map_task to map multiple copies + * of a single task to run in parallel and generate multiple functionally + * equivalent copies of the output data in different locations. It is + * the responsibility of the mapper to ensure that each task gets assigned + * to exactly one processor and each copy of the task gets assigned to + * a different processor. The mapper must also guarantee that any region + * requirements with write privileges be mapped to different physical + * instances for each copy of the task. The runtime will check all these + * invariants in debug mode or if safe mapping is enabled. + * + * The mapper can also choose to make this a control replicated version + * of this task by filling in the 'control_replicate' map. This will make + * all the copies of the task work together as though they were one + * logical version of the task rather than having them all execute + * independently. The vector should be exactly the same size as the + * vector of task_mappings if it is not empty + */ + struct MapReplicateTaskOutput { + std::vector task_mappings; + std::vector control_replication_map; + }; + //------------------------------------------------------------------------ + virtual void map_replicate_task(const MapperContext ctx, + const Task& task, + const MapTaskInput& input, + const MapTaskOutput& default_output, + MapReplicateTaskOutput& output) = 0; + //------------------------------------------------------------------------ + /** * ---------------------------------------------------------------------- * Select Task Variant @@ -693,6 +737,32 @@ namespace Legion { const Task& task, const TaskProfilingInfo& input) = 0; //------------------------------------------------------------------------ + + /** + * ---------------------------------------------------------------------- + * Select Sharding Functor + * ---------------------------------------------------------------------- + * This mapper call is invoked whenever the enclosing parent + * task for the task being launched has been control replicated + * and it's up to the mapper for this task to pick a sharding + * functor to determine which shard will own the point(s) of the + * task. The mapper must return the same sharding functor for all + * copies of the task. The runtime will verify this in debug mode + * but not in release mode. + */ + struct SelectShardingFunctorInput { + std::vector shard_mapping; + }; + struct SelectShardingFunctorOutput { + ShardingID chosen_functor; + }; + //------------------------------------------------------------------------ + virtual void select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) = 0; + //------------------------------------------------------------------------ public: // Inline mapping /** * ---------------------------------------------------------------------- @@ -942,6 +1012,26 @@ namespace Legion { const Copy& copy, const CopyProfilingInfo& input) = 0; //------------------------------------------------------------------------ + + /** + * ---------------------------------------------------------------------- + * Select Sharding Functor + * ---------------------------------------------------------------------- + * This mapper call is invoked whenever the enclosing parent + * task for the copy being launched has been control replicated + * and it's up to the mapper for this copy to pick a sharding + * functor to determine which shard will own the point(s) of the + * copy. The mapper must return the same sharding functor for all + * instances of the copy. The runtime will verify this in debug mode + * but not in release mode. + */ + //------------------------------------------------------------------------ + virtual void select_sharding_functor( + const MapperContext ctx, + const Copy& copy, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) = 0; + //------------------------------------------------------------------------ public: // Close operations // These are here for backwards compatibility // The mapper call these were used by no longer exists @@ -1016,6 +1106,26 @@ namespace Legion { const Close& close, const CloseProfilingInfo& input) = 0; //------------------------------------------------------------------------ + + /** + * ---------------------------------------------------------------------- + * Select Sharding Functor + * ---------------------------------------------------------------------- + * This mapper call is invoked whenever the enclosing parent + * task for the close being launched has been control replicated + * and it's up to the mapper for this task to pick a sharding + * functor to determine which shard will own the point(s) of the + * close. The mapper must return the same sharding functor for all + * instances of the close. The runtime will verify this in debug mode + * but not in release mode. + */ + //------------------------------------------------------------------------ + virtual void select_sharding_functor( + const MapperContext ctx, + const Close& close, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) = 0; + //------------------------------------------------------------------------ public: // Acquire operations /** * ---------------------------------------------------------------------- @@ -1079,6 +1189,26 @@ namespace Legion { const Acquire& acquire, const AcquireProfilingInfo& input) = 0; //------------------------------------------------------------------------ + + /** + * ---------------------------------------------------------------------- + * Select Sharding Functor + * ---------------------------------------------------------------------- + * This mapper call is invoked whenever the enclosing parent + * task for the acquire being launched has been control replicated + * and it's up to the mapper for this task to pick a sharding + * functor to determine which shard will own the point(s) of the + * acquire . The mapper must return the same sharding functor for all + * instances of the acquire. The runtime will verify this in debug mode + * but not in release mode. + */ + //------------------------------------------------------------------------ + virtual void select_sharding_functor( + const MapperContext ctx, + const Acquire& acquire, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) = 0; + //------------------------------------------------------------------------ public: // Release operations /** * ---------------------------------------------------------------------- @@ -1179,6 +1309,26 @@ namespace Legion { const Release& release, const ReleaseProfilingInfo& input) = 0; //------------------------------------------------------------------------ + + /** + * ---------------------------------------------------------------------- + * Select Sharding Functor + * ---------------------------------------------------------------------- + * This mapper call is invoked whenever the enclosing parent + * task for the release being launched has been control replicated + * and it's up to the mapper for this task to pick a sharding + * functor to determine which shard will own the point(s) of the + * release. The mapper must return the same sharding functor for all + * instances of the release. The runtime will verify this in debug mode + * but not in release mode. + */ + //------------------------------------------------------------------------ + virtual void select_sharding_functor( + const MapperContext ctx, + const Release& release, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) = 0; + //------------------------------------------------------------------------ public: // Partition Operations /** * ---------------------------------------------------------------------- @@ -1316,6 +1466,46 @@ namespace Legion { const Partition& partition, const PartitionProfilingInfo& input) = 0; //------------------------------------------------------------------------ + + /** + * ---------------------------------------------------------------------- + * Select Sharding Functor + * ---------------------------------------------------------------------- + * This mapper call is invoked whenever the enclosing parent + * task for the partition being launched has been control replicated + * and it's up to the mapper for this task to pick a sharding + * functor to determine which shard will own the point(s) of the + * partition. The mapper must return the same sharding functor for all + * instances of the partition. The runtime will verify this in debug mode + * but not in release mode. + */ + //------------------------------------------------------------------------ + virtual void select_sharding_functor( + const MapperContext ctx, + const Partition& partition, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) = 0; + //------------------------------------------------------------------------ + public: // Fill Operations + /** + * ---------------------------------------------------------------------- + * Select Sharding Functor + * ---------------------------------------------------------------------- + * This mapper call is invoked whenever the enclosing parent + * task for the fill being launched has been control replicated + * and it's up to the mapper for this task to pick a sharding + * functor to determine which shard will own the points of the + * fill. The mapper must return the same sharding functor for all + * instances of the fill. The runtime will verify this in debug mode + * but not in release mode. + */ + //------------------------------------------------------------------------ + virtual void select_sharding_functor( + const MapperContext ctx, + const Fill& fill, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) = 0; + //------------------------------------------------------------------------ public: // Single Task Context /** * ---------------------------------------------------------------------- @@ -1404,6 +1594,36 @@ namespace Legion { SelectTunableOutput& output) = 0; //------------------------------------------------------------------------ public: // Mapping collections of operations + /** + * ---------------------------------------------------------------------- + * Select Sharding Functor + * ---------------------------------------------------------------------- + * This mapper call is invoked whenever the enclosing parent + * task for the must epoch operation being launched has been + * control replicated and it's up to the mapper for this must epoch + * operation to pick a sharding functor to determine which shard will + * own the point(s) of the must epoch operation . The mapper must return + * the same sharding functor for all instances of the must epoch + * operation. The runtime will verify this in debug mode + * but not in release mode. For this mapper call the mapper must + * also choose whether to perform the map_must_epoch call as a collective + * operation or not. If it chooses to perform it as a collective then we + * will do one map_must_epoch call on each shard with the constraints + * that apply to the points owned by the shard. The default is not to + * perform the map must epoch call as a collective operation. + */ + struct MustEpochShardingFunctorOutput : + public SelectShardingFunctorOutput { + bool collective_map_must_epoch_call; + }; + //------------------------------------------------------------------------ + virtual void select_sharding_functor( + const MapperContext ctx, + const MustEpoch& epoch, + const SelectShardingFunctorInput& input, + MustEpochShardingFunctorOutput& output) = 0; + //------------------------------------------------------------------------ + /** * ---------------------------------------------------------------------- * Map Must Epoch @@ -1419,6 +1639,23 @@ namespace Legion { * field which says which logical regions in different tasks must be * mapped to the same physical instance. The mapper is also given * the mapping tag passed at the callsite in 'mapping_tag'. + * + * A special case of map_must_epoch is when it is called as a collective + * mapping call for a must epoch launch performed inside of a control + * replicated parent task. This behavior is controlled by the result + * of select_sharding_functor for the must epoch operation (see above). + * In this case map_must_epoch will only be given 'tasks' owned by its + * shard and 'constraints' that apply to those 'tasks'. The mapper must + * still pick 'task_processors' and these processor must be unique with + * respect to any chosen for other 'tasks' by other mappers. The runime + * will check this property in debug mode. For constraints, the mapper + * may also pick optional 'constraint_mappings' for its constraints or + * rely on another mapper to pick them (it's up to the mapper to + * determine which mapper instance picks thems). The mapper can then + * specify a 'weight' for each constraint mapping. The runtime will + * do a collective reduction across all the 'constraint_mappings' taking + * the mappings with the highest weights and the lowest shard ID when + * the weights are the same. */ struct MappingConstraint { std::vector constrained_tasks; @@ -1624,23 +1861,6 @@ namespace Legion { virtual void handle_task_result(const MapperContext ctx, const MapperTaskResult& result) = 0; //------------------------------------------------------------------------ - public: - // Future structs for control replication, - // provided here for forward compatibility - struct MapReplicateTaskOutput { - std::vector task_mappings; - std::vector control_replication_map; - }; - struct SelectShardingFunctorInput { - std::vector shard_mapping; - }; - struct SelectShardingFunctorOutput { - ShardingID chosen_functor; - }; - struct MustEpochShardingFunctorOutput : - public SelectShardingFunctorOutput { - bool collective_map_must_epoch_call; - }; }; /** @@ -1764,13 +1984,13 @@ namespace Legion { const char* find_task_variant_name(MapperContext ctx, TaskID task_id, VariantID vid) const; bool is_leaf_variant(MapperContext ctx, TaskID task_id, - VariantID variant_id) const; + VariantID variant_id) const; bool is_inner_variant(MapperContext ctx, TaskID task_id, VariantID variant_id)const; bool is_idempotent_variant(MapperContext ctx, TaskID task_id, - VariantID variant_id) const; + VariantID variant_id) const; bool is_replicable_variant(MapperContext ctx, TaskID task_id, - VariantID variant_id) const; + VariantID variant_id) const; public: //------------------------------------------------------------------------ // Methods for registering variants @@ -2150,6 +2370,17 @@ namespace Legion { void retrieve_name(MapperContext ctx, LogicalPartition handle, const char *&result); + public: + //------------------------------------------------------------------------ + // Methods for MPI interoperability + //------------------------------------------------------------------------ + bool is_MPI_interop_configured(MapperContext ctx); + const std::map& + find_forward_MPI_mapping(MapperContext ctx); + + const std::map& + find_reverse_MPI_mapping(MapperContext ctx); + int find_local_MPI_rank(MapperContext ctx); public: //------------------------------------------------------------------------ // Support for packing tunable values diff --git a/runtime/legion/legion_ops.cc b/runtime/legion/legion_ops.cc index 7453d83ffd..99911f412d 100644 --- a/runtime/legion/legion_ops.cc +++ b/runtime/legion/legion_ops.cc @@ -1016,7 +1016,7 @@ namespace Legion { assert(trace != NULL); #endif if (target_gen < gen) - trace->record_dependence(this, target_gen, this, gen); + trace->record_dependence(this, target_gen, this, gen); return false; } else @@ -1038,7 +1038,7 @@ namespace Legion { #ifdef DEBUG_LEGION assert(trace != NULL); #endif - trace->record_dependence(target, target_gen, this, gen); + trace->record_dependence(target, target_gen, this, gen); // Unsound to prune when tracing prune = false; } @@ -1078,7 +1078,7 @@ namespace Legion { trace->record_region_dependence(this, target_gen, this, gen, target_idx, idx, dtype, validates, - dependent_mask); + dependent_mask); return false; } else @@ -1152,8 +1152,6 @@ namespace Legion { if (finder == outgoing.end()) { outgoing[op] = op_gen; - // Record that the operation has a mapping dependence - // on us as long as we haven't mapped tracker->add_mapping_dependence(mapped_event); tracker->add_resolution_dependence(resolved_event); // Record that we have a commit dependence on the @@ -2331,7 +2329,7 @@ namespace Legion { { if (predicate != NULL) { - register_dependence(predicate, predicate->get_generation()); + register_dependence(predicate, predicate->get_generation()); // Now we can remove our predicate reference predicate->remove_predicate_reference(); } @@ -2806,7 +2804,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - void MapOp::deactivate(void) + void MapOp::deactivate_map_op(void) //-------------------------------------------------------------------------- { deactivate_operation(); @@ -2834,6 +2832,13 @@ namespace Legion { mapper_data = NULL; mapper_data_size = 0; } + } + + //-------------------------------------------------------------------------- + void MapOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_map_op(); // Now return this operation to the queue runtime->free_map_op(this); } @@ -4034,7 +4039,9 @@ namespace Legion { #endif map_id = launcher.map_id; tag = launcher.tag; - index_point = launcher.point; + index_point = launcher.point; + index_domain = Domain(index_point, index_point); + sharding_space = launcher.sharding_space; if (runtime->legion_spy_enabled) { const unsigned copy_kind = (src_indirect_requirements.empty() ? 0 : 1) + @@ -4398,14 +4405,12 @@ namespace Legion { // Register a dependence on our predicate register_predicate_dependence(); ProjectionInfo projection_info; - src_versions.resize(src_requirements.size()); for (unsigned idx = 0; idx < src_requirements.size(); idx++) runtime->forest->perform_dependence_analysis(this, idx, src_requirements[idx], projection_info, src_privilege_paths[idx], map_applied_conditions); - dst_versions.resize(dst_requirements.size()); for (unsigned idx = 0; idx < dst_requirements.size(); idx++) { unsigned index = src_requirements.size()+idx; @@ -4447,6 +4452,16 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + void CopyOp::perform_base_dependence_analysis(void) + //-------------------------------------------------------------------------- + { + // Register a dependence on our predicate + register_predicate_dependence(); + src_versions.resize(src_requirements.size()); + dst_versions.resize(dst_requirements.size()); + } + //-------------------------------------------------------------------------- bool CopyOp::query_speculate(bool &value, bool &mapping_only) //-------------------------------------------------------------------------- @@ -6202,6 +6217,7 @@ namespace Legion { launch_space->get_launch_space_domain(index_domain); else index_domain = launcher.launch_domain; + sharding_space = launcher.sharding_space; src_requirements.resize(launcher.src_requirements.size()); dst_requirements.resize(launcher.dst_requirements.size()); src_versions.resize(launcher.src_requirements.size()); @@ -6371,9 +6387,17 @@ namespace Legion { //-------------------------------------------------------------------------- void IndexCopyOp::activate(void) //-------------------------------------------------------------------------- + { + activate_index_copy(); + } + + //-------------------------------------------------------------------------- + void IndexCopyOp::activate_index_copy(void) + //-------------------------------------------------------------------------- { activate_copy(); index_domain = Domain::NO_DOMAIN; + sharding_space = IndexSpace::NO_SPACE; launch_space = NULL; points_committed = 0; commit_request = false; @@ -6382,6 +6406,15 @@ namespace Legion { //-------------------------------------------------------------------------- void IndexCopyOp::deactivate(void) //-------------------------------------------------------------------------- + { + deactivate_index_copy(); + // Return this operation to the runtime + runtime->free_index_copy_op(this); + } + + //-------------------------------------------------------------------------- + void IndexCopyOp::deactivate_index_copy(void) + //-------------------------------------------------------------------------- { deactivate_copy(); // We can deactivate all of our point operations @@ -6401,8 +6434,6 @@ namespace Legion { interfering_requirements.clear(); if (remove_launch_space_reference(launch_space)) delete launch_space; - // Return this operation to the runtime - runtime->free_index_copy_op(this); } //-------------------------------------------------------------------------- @@ -6560,8 +6591,7 @@ namespace Legion { if (runtime->check_privileges) check_copy_privileges(true/*permit projection*/); // Register a dependence on our predicate - register_predicate_dependence(); - src_versions.resize(src_requirements.size()); + perform_base_dependence_analysis(); for (unsigned idx = 0; idx < src_requirements.size(); idx++) { ProjectionInfo src_info(runtime, src_requirements[idx], launch_space); @@ -6571,7 +6601,6 @@ namespace Legion { src_privilege_paths[idx], map_applied_conditions); } - dst_versions.resize(dst_requirements.size()); for (unsigned idx = 0; idx < dst_requirements.size(); idx++) { ProjectionInfo dst_info(runtime, dst_requirements[idx], launch_space); @@ -6714,13 +6743,18 @@ namespace Legion { void IndexCopyOp::enumerate_points(bool replaying) //-------------------------------------------------------------------------- { - size_t num_points = index_domain.get_volume(); + // Need to get the launch domain in case it is different than + // the original index domain due to control replication + Domain launch_domain; + launch_space->get_launch_space_domain(launch_domain); + // Now enumerate the points + size_t num_points = launch_domain.get_volume(); #ifdef DEBUG_LEGION assert(num_points > 0); #endif unsigned point_idx = 0; points.resize(num_points); - for (Domain::DomainPointIterator itr(index_domain); + for (Domain::DomainPointIterator itr(launch_domain); itr; itr++, point_idx++) { PointCopyOp *point = runtime->get_available_point_copy_op(); @@ -6737,7 +6771,7 @@ namespace Legion { ProjectionFunction *function = runtime->find_projection_function(src_requirements[idx].projection); function->project_points(this, idx, src_requirements[idx], - runtime, projection_points); + runtime, index_domain, projection_points); } unsigned offset = src_requirements.size(); for (unsigned idx = 0; idx < dst_requirements.size(); idx++) @@ -6748,7 +6782,7 @@ namespace Legion { runtime->find_projection_function(dst_requirements[idx].projection); function->project_points(this, offset + idx, dst_requirements[idx], runtime, - projection_points); + index_domain, projection_points); } if (!src_indirect_requirements.empty()) { @@ -6763,7 +6797,7 @@ namespace Legion { src_indirect_requirements[idx].projection); function->project_points(this, offset + idx, src_indirect_requirements[idx], runtime, - projection_points); + index_domain, projection_points); } } if (!dst_indirect_requirements.empty()) @@ -6779,7 +6813,7 @@ namespace Legion { dst_indirect_requirements[idx].projection); function->project_points(this, offset + idx, dst_indirect_requirements[idx], runtime, - projection_points); + index_domain, projection_points); } } if (runtime->legion_spy_enabled && !replaying) @@ -7090,6 +7124,7 @@ namespace Legion { own->src_requirements.size() + own->dst_requirements.size()); index_point = p; index_domain = own->index_domain; + sharding_space = own->sharding_space; owner = own; execution_fence_event = own->get_execution_fence_event(); // From Memoizable @@ -7227,7 +7262,7 @@ namespace Legion { } } } - // We can also mark this as having our resolved any predication + // We can also mark this as having resolved any predication resolve_speculation(); // Then put ourselves in the queue of operations ready to map if (!preconditions.empty()) @@ -7394,10 +7429,10 @@ namespace Legion { //-------------------------------------------------------------------------- Future FenceOp::initialize(InnerContext *ctx, FenceKind kind, - bool need_future) + bool need_future, bool track/*=true*/) //-------------------------------------------------------------------------- { - initialize_operation(ctx, true/*track*/); + initialize_operation(ctx, track); fence_kind = kind; if (need_future) { @@ -7422,12 +7457,19 @@ namespace Legion { } //-------------------------------------------------------------------------- - void FenceOp::deactivate(void) + void FenceOp::deactivate_fence(void) //-------------------------------------------------------------------------- { deactivate_operation(); map_applied_conditions.clear(); result = Future(); // clear out our future reference + } + + //-------------------------------------------------------------------------- + void FenceOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_fence(); runtime->free_fence_op(this); } @@ -7688,8 +7730,23 @@ namespace Legion { } //-------------------------------------------------------------------------- - void CreationOp::initialize_index_space( - InnerContext *ctx, IndexSpaceNode *n, const Future &f) + void CreationOp::initialize_fence(InnerContext *ctx, RtEvent precondition) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!mapping_precondition.exists()); +#endif + initialize_operation(ctx, true/*track*/); + kind = FENCE_CREATION; + mapping_precondition = precondition; + if (runtime->legion_spy_enabled) + LegionSpy::log_creation_operation(parent_ctx->get_unique_id(), + unique_op_id); + } + + //-------------------------------------------------------------------------- + void CreationOp::initialize_index_space(InnerContext *ctx, + IndexSpaceNode *n, const Future &f, bool own, ShardMapping *mapping) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -7700,6 +7757,8 @@ namespace Legion { kind = INDEX_SPACE_CREATION; index_space_node = n; futures.push_back(f); + shard_mapping = mapping; + owner = own; if (runtime->legion_spy_enabled) LegionSpy::log_creation_operation(parent_ctx->get_unique_id(), unique_op_id); @@ -7707,19 +7766,22 @@ namespace Legion { //-------------------------------------------------------------------------- void CreationOp::initialize_field(InnerContext *ctx, FieldSpaceNode *node, - FieldID fid, const Future &field_size) + FieldID fid, const Future &field_size, RtEvent precondition, bool own) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION assert(field_space_node == NULL); assert(fields.empty()); assert(futures.empty()); + assert(!mapping_precondition.exists()); #endif initialize_operation(ctx, true/*track*/); kind = FIELD_ALLOCATION; field_space_node = node; fields.push_back(fid); futures.push_back(field_size); + mapping_precondition = precondition; + owner = own; if (runtime->legion_spy_enabled) LegionSpy::log_creation_operation(parent_ctx->get_unique_id(), unique_op_id); @@ -7728,7 +7790,8 @@ namespace Legion { //-------------------------------------------------------------------------- void CreationOp::initialize_fields(InnerContext *ctx, FieldSpaceNode *node, const std::vector &fids, - const std::vector &field_sizes) + const std::vector &field_sizes, + RtEvent precondition, bool own) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -7736,12 +7799,15 @@ namespace Legion { assert(fields.empty()); assert(futures.empty()); assert(fids.size() == field_sizes.size()); + assert(!mapping_precondition.exists()); #endif initialize_operation(ctx, true/*track*/); kind = FIELD_ALLOCATION; field_space_node = node; fields = fids; futures = field_sizes; + mapping_precondition = precondition; + owner = own; if (runtime->legion_spy_enabled) LegionSpy::log_creation_operation(parent_ctx->get_unique_id(), unique_op_id); @@ -7774,6 +7840,9 @@ namespace Legion { activate_operation(); index_space_node = NULL; field_space_node = NULL; + mapping_precondition = RtEvent::NO_RT_EVENT; + shard_mapping = NULL; + owner = true; } //-------------------------------------------------------------------------- @@ -7804,9 +7873,6 @@ namespace Legion { void CreationOp::trigger_dependence_analysis(void) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(!futures.empty()); -#endif for (std::vector::const_iterator it = futures.begin(); it != futures.end(); it++) { @@ -7819,7 +7885,8 @@ namespace Legion { } // Record this with the context as an implicit dependence for all // later operations which may rely on this index space for mapping - if ((kind == INDEX_SPACE_CREATION) || (kind == FIELD_ALLOCATION)) + if ((kind == FENCE_CREATION) || (kind == INDEX_SPACE_CREATION) || + (kind == FIELD_ALLOCATION)) parent_ctx->update_current_implicit(this); } @@ -7827,7 +7894,7 @@ namespace Legion { void CreationOp::trigger_mapping(void) //-------------------------------------------------------------------------- { - complete_mapping(); + complete_mapping(mapping_precondition); switch (kind) { case INDEX_SPACE_CREATION: @@ -7835,8 +7902,8 @@ namespace Legion { #ifdef DEBUG_LEGION assert(futures.size() == 1); #endif - const ApEvent ready = futures[0].impl->subscribe(); - complete_execution(Runtime::protect_event(ready)); + const RtEvent ready = futures[0].impl->subscribe_internal(); + complete_execution(ready); break; } case FIELD_ALLOCATION: @@ -7850,6 +7917,7 @@ namespace Legion { complete_execution(); break; } + case FENCE_CREATION: case FUTURE_MAP_CREATION: { complete_execution(); @@ -7882,7 +7950,8 @@ namespace Legion { parent_ctx->get_unique_id(), sizeof(Domain)) const Domain *domain = static_cast( impl->get_untyped_result(true,NULL,true/*internal*/)); - if (index_space_node->set_domain(*domain, runtime->address_space)) + if (owner && index_space_node->set_domain(*domain, + runtime->address_space, shard_mapping)) delete index_space_node; break; } @@ -7904,12 +7973,13 @@ namespace Legion { *((const size_t*)impl->get_untyped_result(true, NULL, true)); field_space_node->update_field_size(fields[idx], field_size, complete_preconditions, runtime->address_space); - if (runtime->legion_spy_enabled) + if (runtime->legion_spy_enabled && owner) LegionSpy::log_field_creation(field_space_node->handle.id, fields[idx], field_size); } break; } + case FENCE_CREATION: case FUTURE_MAP_CREATION: // Nothing to do here break; @@ -8067,6 +8137,13 @@ namespace Legion { //-------------------------------------------------------------------------- void DeletionOp::activate(void) //-------------------------------------------------------------------------- + { + activate_deletion(); + } + + //-------------------------------------------------------------------------- + void DeletionOp::activate_deletion(void) + //-------------------------------------------------------------------------- { activate_operation(); allocator = NULL; @@ -8075,6 +8152,15 @@ namespace Legion { //-------------------------------------------------------------------------- void DeletionOp::deactivate(void) //-------------------------------------------------------------------------- + { + deactivate_deletion(); + // Return this to the available deletion ops on the queue + runtime->free_deletion_op(this); + } + + //-------------------------------------------------------------------------- + void DeletionOp::deactivate_deletion(void) + //-------------------------------------------------------------------------- { // We can remove the reference to the allocator once we are // done with all of our free operations @@ -8092,8 +8178,6 @@ namespace Legion { deletion_requirements.clear(); version_infos.clear(); map_applied_conditions.clear(); - // Return this to the available deletion ops on the queue - runtime->free_deletion_op(this); } //-------------------------------------------------------------------------- @@ -8253,10 +8337,10 @@ namespace Legion { // For this case we actually need to go through and prune out any // valid instances for these fields in the equivalence sets in order // to be able to free up the resources. - const PhysicalTraceInfo trace_info(this, -1U, false/*init*/); + const TraceInfo trace_info(this); for (unsigned idx = 0; idx < deletion_requirements.size(); idx++) runtime->forest->invalidate_fields(this, idx, version_infos[idx], - trace_info,map_applied_conditions); + PhysicalTraceInfo(trace_info, idx), map_applied_conditions); // make sure that we don't try to do the deletion calls until // after the allocator is ready if (allocator->ready_event.exists()) @@ -8415,6 +8499,7 @@ namespace Legion { #endif // We never track internal operations initialize_operation(creator->get_context(), false/*track*/); + parent_ctx->register_new_internal_operation(this); #ifdef DEBUG_LEGION assert(creator_req_idx == -1); assert(create_op == NULL); @@ -9447,11 +9532,11 @@ namespace Legion { //-------------------------------------------------------------------------- { parent_task = ctx->get_task(); + initialize_memoizable(); initialize_speculation(ctx, true/*track*/, 1/*num region requirements*/, launcher.static_dependences, launcher.predicate); - initialize_memoizable(); // Note we give it READ WRITE EXCLUSIVE to make sure that nobody // can be re-ordered around this operation for mapping or // normal dependences. We won't actually read or write anything. @@ -9499,7 +9584,7 @@ namespace Legion { void AcquireOp::activate(void) //-------------------------------------------------------------------------- { - activate_speculative(); + activate_speculative(); activate_memoizable(); mapper = NULL; outstanding_profiling_requests = 0; @@ -9667,7 +9752,7 @@ namespace Legion { return; } - std::set preconditions; + std::set preconditions; runtime->forest->perform_versioning_analysis(this, 0/*idx*/, requirement, version_info, @@ -9732,7 +9817,7 @@ namespace Legion { profiling_reported.exists()) Runtime::trigger_event(profiling_reported); if (is_recording()) - tpl->record_complete_replay(this, acquire_complete); + tpl->record_complete_replay(this, acquire_complete); // Mark that we completed mapping RtEvent mapping_applied; if (!map_applied_conditions.empty()) @@ -10318,11 +10403,11 @@ namespace Legion { //-------------------------------------------------------------------------- { parent_task = ctx->get_task(); + initialize_memoizable(); initialize_speculation(ctx, true/*track*/, 1/*num region requirements*/, launcher.static_dependences, launcher.predicate); - initialize_memoizable(); // Note we give it READ WRITE EXCLUSIVE to make sure that nobody // can be re-ordered around this operation for mapping or // normal dependences. We won't actually read or write anything. @@ -11406,16 +11491,16 @@ namespace Legion { LegionSpy::log_operation_events(unique_op_id, ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); #endif - if (valid) - set_resolved_value(get_generation(), value); - else + if (!valid) { // Launch a task to get the value add_predicate_reference(); ResolveFuturePredArgs args(this); runtime->issue_runtime_meta_task(args, LG_LATENCY_WORK_PRIORITY, - Runtime::protect_event(future.impl->subscribe())); + future.impl->subscribe_internal()); } + else + set_resolved_value(get_generation(), value); } ///////////////////////////////////////////////////////////// @@ -11666,7 +11751,7 @@ namespace Legion { { for (std::vector::const_iterator it = previous.begin(); it != previous.end(); it++) - register_dependence(*it, (*it)->get_generation()); + register_dependence(*it, (*it)->get_generation()); } //-------------------------------------------------------------------------- @@ -11928,14 +12013,14 @@ namespace Legion { //-------------------------------------------------------------------------- MustEpochOp::MustEpochOp(Runtime *rt) - : Operation(rt) + : Operation(rt), MustEpoch() //-------------------------------------------------------------------------- { } //-------------------------------------------------------------------------- MustEpochOp::MustEpochOp(const MustEpochOp &rhs) - : Operation(NULL) + : Operation(NULL), MustEpoch() //-------------------------------------------------------------------------- { // should never be called @@ -11957,6 +12042,27 @@ namespace Legion { return *this; } + //-------------------------------------------------------------------------- + UniqueID MustEpochOp::get_unique_id(void) const + //-------------------------------------------------------------------------- + { + return unique_op_id; + } + + //-------------------------------------------------------------------------- + size_t MustEpochOp::get_context_index(void) const + //-------------------------------------------------------------------------- + { + return context_index; + } + + //-------------------------------------------------------------------------- + int MustEpochOp::get_depth(void) const + //-------------------------------------------------------------------------- + { + return (parent_ctx->get_depth() + 1); + } + //-------------------------------------------------------------------------- FutureMap MustEpochOp::initialize(InnerContext *ctx, const MustEpochLauncher &launcher) @@ -11964,12 +12070,50 @@ namespace Legion { { // Initialize this operation initialize_operation(ctx, true/*track*/); + // Compute our launch domain if we need it + launch_domain = launcher.launch_domain; + RtUserEvent future_deletion; + if (!launch_domain.exists()) + { + if (!launcher.launch_space.exists()) + future_deletion = compute_launch_space(launcher); + else + runtime->forest->find_launch_space_domain(launcher.launch_space, + launch_domain); +#ifdef DEBUG_LEGION + assert(launch_domain.exists()); +#endif + } // Make a new future map for storing our results // We'll fill it in later - result_map = FutureMap(new FutureMapImpl(ctx, this, - Runtime::protect_event(get_completion_event()), runtime, - runtime->get_available_distributed_id(), - runtime->address_space)); + sharding_space = launcher.sharding_space; + result_map = FutureMap(create_future_map(ctx, launch_domain, + sharding_space, future_deletion)); + instantiate_tasks(ctx, launcher); + map_id = launcher.map_id; + tag = launcher.mapping_tag; + parent_task = ctx->get_task(); + if (runtime->legion_spy_enabled) + LegionSpy::log_must_epoch_operation(ctx->get_unique_id(), unique_op_id); + return result_map; + } + + //-------------------------------------------------------------------------- + FutureMapImpl* MustEpochOp::create_future_map(TaskContext *ctx, + const Domain &domain, IndexSpace shard_space, RtUserEvent deleted) + //-------------------------------------------------------------------------- + { + return new FutureMapImpl(ctx, this, + Runtime::protect_event(get_completion_event()), domain, + runtime, runtime->get_available_distributed_id(), + runtime->address_space, deleted); + } + + //-------------------------------------------------------------------------- + void MustEpochOp::instantiate_tasks(InnerContext *ctx, + const MustEpochLauncher &launcher) + //-------------------------------------------------------------------------- + { // Initialize operations for everything in the launcher // Note that we do not track these operations as we want them all to // appear as a single operation to the parent context in order to @@ -12002,17 +12146,6 @@ namespace Legion { index_tasks[idx]->set_trace(trace, NULL); } index_triggered.resize(index_tasks.size(), false); - mapper_id = launcher.map_id; - mapper_tag = launcher.mapping_tag; -#ifdef DEBUG_LEGION - for (unsigned idx = 0; idx < indiv_tasks.size(); idx++) - result_map.impl->add_valid_point(indiv_tasks[idx]->index_point); - for (unsigned idx = 0; idx < index_tasks.size(); idx++) - result_map.impl->add_valid_domain(index_tasks[idx]->index_domain); -#endif - if (runtime->legion_spy_enabled) - LegionSpy::log_must_epoch_operation(ctx->get_unique_id(), unique_op_id); - return result_map; } //-------------------------------------------------------------------------- @@ -12042,10 +12175,21 @@ namespace Legion { //-------------------------------------------------------------------------- void MustEpochOp::activate(void) //-------------------------------------------------------------------------- + { + activate_must_epoch_op(); + } + + //-------------------------------------------------------------------------- + void MustEpochOp::activate_must_epoch_op(void) + //-------------------------------------------------------------------------- { activate_operation(); - mapper_id = 0; - mapper_tag = 0; + map_id = 0; + tag = 0; + parent_task = NULL; + launch_domain = Domain(); + individual_tasks.clear(); + index_space_tasks.clear(); // Set to 1 to include the triggers we get for our operation remaining_subop_completes = 1; remaining_subop_commits = 1; @@ -12055,6 +12199,15 @@ namespace Legion { //-------------------------------------------------------------------------- void MustEpochOp::deactivate(void) //-------------------------------------------------------------------------- + { + deactivate_must_epoch_op(); + // Return this operation to the free list + runtime->free_epoch_op(this); + } + + //-------------------------------------------------------------------------- + void MustEpochOp::deactivate_must_epoch_op(void) + //-------------------------------------------------------------------------- { deactivate_operation(); // All the sub-operations we have will deactivate themselves @@ -12082,8 +12235,6 @@ namespace Legion { input.constraints.clear(); output.task_processors.clear(); output.constraint_mappings.clear(); - // Return this operation to the free list - runtime->free_epoch_op(this); } //-------------------------------------------------------------------------- @@ -12132,6 +12283,14 @@ namespace Legion { index_tasks[idx]->execute_dependence_analysis(); } + //-------------------------------------------------------------------------- + /*static*/ bool MustEpochOp::single_task_sorter(const Task *t1, + const Task *t2) + //-------------------------------------------------------------------------- + { + return (t1->index_point < t2->index_point); + } + //-------------------------------------------------------------------------- void MustEpochOp::trigger_mapping(void) //-------------------------------------------------------------------------- @@ -12150,12 +12309,17 @@ namespace Legion { if (!triggering_complete) { task_sets.resize(indiv_tasks.size()+index_tasks.size()); - MustEpochTriggerer triggerer(this); - triggerer.trigger_tasks(indiv_tasks, indiv_triggered, - index_tasks, index_triggered); + trigger_tasks(this, indiv_tasks, indiv_triggered, + index_tasks, index_triggered); #ifdef DEBUG_LEGION assert(!single_tasks.empty()); #endif + // Sort the points so that they are in order for determinism + // across runs and for control replication + std::sort(single_tasks.begin(), single_tasks.end(), single_task_sorter); + // Then construct the inverse mapping + for (unsigned idx = 0; idx < single_tasks.size(); idx++) + single_task_map[single_tasks[idx]] = idx; // Next build the set of single tasks and all their constraints. // Iterate over all the recorded dependences std::vector &constraints = input.constraints; @@ -12217,16 +12381,14 @@ namespace Legion { triggering_complete = true; } // Fill in the rest of the inputs to the mapper call - input.mapping_tag = mapper_tag; + input.mapping_tag = tag; input.tasks.insert(input.tasks.end(), single_tasks.begin(), single_tasks.end()); // Also resize the outputs so the mapper knows what it is doing output.constraint_mappings.resize(input.constraints.size()); output.task_processors.resize(single_tasks.size(), Processor::NO_PROC); - Processor mapper_proc = parent_ctx->get_executing_processor(); - MapperManager *mapper = runtime->find_mapper(mapper_proc, mapper_id); - // We've got all our meta-data set up so go ahead and issue the call - mapper->invoke_map_must_epoch(this, &input, &output); + // Now we can invoke the mapper + MapperManager *mapper = invoke_mapper(); // Check that all the tasks have been assigned to different processors { std::map target_procs; @@ -12261,32 +12423,10 @@ namespace Legion { task->target_proc = proc; } } - // Then we need to actually perform the mapping - { - MustEpochMapper mapper(this); - mapper.map_tasks(single_tasks, mapping_dependences); - mapping_dependences.clear(); - } - // Once all the tasks have been initialized we can defer - // our all mapped event on all their all mapped events std::set tasks_all_mapped; std::set tasks_all_complete; - for (std::vector::const_iterator it = - indiv_tasks.begin(); it != indiv_tasks.end(); it++) - { - tasks_all_mapped.insert((*it)->get_mapped_event()); - tasks_all_complete.insert((*it)->get_completion_event()); - } - for (std::vector::const_iterator it = - index_tasks.begin(); it != index_tasks.end(); it++) - { - tasks_all_mapped.insert((*it)->get_mapped_event()); - tasks_all_complete.insert((*it)->get_completion_event()); - } - // If we passed all the constraints, then kick everything off - MustEpochDistributor distributor(this); - distributor.distribute_tasks(runtime, indiv_tasks, slice_tasks); - + // Map and distribute all our tasks + map_and_distribute(tasks_all_mapped, tasks_all_complete); // Mark that we are done mapping and executing this operation RtEvent all_mapped = Runtime::merge_events(tasks_all_mapped); RtEvent all_complete = Runtime::protect_merge_events(tasks_all_complete); @@ -12298,259 +12438,367 @@ namespace Legion { } //-------------------------------------------------------------------------- - void MustEpochOp::trigger_complete(void) + void MustEpochOp::map_and_distribute(std::set &tasks_mapped, + std::set &tasks_complete) //-------------------------------------------------------------------------- { - bool need_complete; + // Perform the mapping + map_tasks(); + mapping_dependences.clear(); + // Once all the tasks have been initialized we can defer + // our all mapped event on all their all mapped events + for (std::vector::const_iterator it = + indiv_tasks.begin(); it != indiv_tasks.end(); it++) { - AutoLock o_lock(op_lock); -#ifdef DEBUG_LEGION - assert(remaining_subop_completes > 0); -#endif - remaining_subop_completes--; - need_complete = (remaining_subop_completes == 0); + tasks_mapped.insert((*it)->get_mapped_event()); + tasks_complete.insert((*it)->get_completion_event()); } - if (need_complete) + for (std::vector::const_iterator it = + index_tasks.begin(); it != index_tasks.end(); it++) { -#ifdef LEGION_SPY - // Still need this for Legion Spy - LegionSpy::log_operation_events(unique_op_id, - ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); -#endif - complete_operation(); + tasks_mapped.insert((*it)->get_mapped_event()); + tasks_complete.insert((*it)->get_completion_event()); } + // Then we can distribute the tasks + distribute_tasks(); } //-------------------------------------------------------------------------- - void MustEpochOp::trigger_commit(void) + MapperManager* MustEpochOp::invoke_mapper(void) //-------------------------------------------------------------------------- { - bool need_commit; - { - AutoLock o_lock(op_lock); -#ifdef DEBUG_LEGION - assert(remaining_subop_commits > 0); -#endif - remaining_subop_commits--; - need_commit = (remaining_subop_commits == 0); - } - if (need_commit) - commit_operation(true/*deactivate*/); + Processor mapper_proc = parent_ctx->get_executing_processor(); + MapperManager *mapper = runtime->find_mapper(mapper_proc, map_id); + // We've got all our meta-data set up so go ahead and issue the call + mapper->invoke_map_must_epoch(this, &input, &output); + return mapper; } //-------------------------------------------------------------------------- - void MustEpochOp::verify_dependence(Operation *src_op, GenerationID src_gen, - Operation *dst_op, GenerationID dst_gen) + /*static*/ void MustEpochOp::trigger_tasks(MustEpochOp *owner, + const std::vector &indiv_tasks, + std::vector &indiv_triggered, + const std::vector &index_tasks, + std::vector &index_triggered) //-------------------------------------------------------------------------- { - // If they are the same, then we can ignore them - if ((src_op == dst_op) && (src_gen == dst_gen)) - return; - // Check to see if the source is one of our operations, if it is - // then we have an actual dependence which is an error. - int src_index = find_operation_index(src_op, src_gen); - if (src_index >= 0) + const Processor current = owner->get_context()->get_executing_processor(); + std::set wait_events; + for (unsigned idx = 0; idx < indiv_triggered.size(); idx++) { - int dst_index = find_operation_index(dst_op, dst_gen); - if (dst_index >= 0) + if (!indiv_triggered[idx]) { - TaskOp *src_task = find_task_by_index(src_index); - TaskOp *dst_task = find_task_by_index(dst_index); - REPORT_LEGION_ERROR(ERROR_MUST_EPOCH_DEPENDENCE, - "MUST EPOCH ERROR: dependence between task " - "%s (ID %lld) and task %s (ID %lld)\n", - src_task->get_task_name(), src_task->get_unique_id(), - dst_task->get_task_name(), dst_task->get_unique_id()) + MustEpochIndivArgs args(current, indiv_tasks[idx], owner); + RtEvent wait = + owner->runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); } } - } - - //-------------------------------------------------------------------------- - bool MustEpochOp::record_dependence(Operation *src_op, GenerationID src_gen, - Operation *dst_op, GenerationID dst_gen, - unsigned src_idx, unsigned dst_idx, - DependenceType dtype) - //-------------------------------------------------------------------------- - { - // If they are the same we can ignore them - if ((src_op == dst_op) && (src_gen == dst_gen)) - return true; - // Check to see if the source is one of our operations - int src_index = find_operation_index(src_op, src_gen); - int dst_index = find_operation_index(dst_op, dst_gen); - if ((src_index >= 0) && (dst_index >= 0)) + for (unsigned idx = 0; idx < index_tasks.size(); idx++) { - // If it is, see what kind of dependence we have - if ((dtype == LEGION_TRUE_DEPENDENCE) || - (dtype == LEGION_ANTI_DEPENDENCE) || - (dtype == LEGION_ATOMIC_DEPENDENCE)) + if (!index_triggered[idx]) { - TaskOp *src_task = find_task_by_index(src_index); - TaskOp *dst_task = find_task_by_index(dst_index); - REPORT_LEGION_ERROR(ERROR_MUST_EPOCH_DEPENDENCE, - "MUST EPOCH ERROR: dependence between region %d " - "of task %s (ID %lld) and region %d of task %s (ID %lld) of " - " type %s", src_idx, src_task->get_task_name(), - src_task->get_unique_id(), dst_idx, - dst_task->get_task_name(), dst_task->get_unique_id(), - (dtype == LEGION_TRUE_DEPENDENCE) ? "TRUE DEPENDENCE" : - (dtype == LEGION_ANTI_DEPENDENCE) ? "ANTI DEPENDENCE" : - "ATOMIC DEPENDENCE") - } - else if (dtype == LEGION_SIMULTANEOUS_DEPENDENCE) - { - // Record the dependence kind - int dst_index = find_operation_index(dst_op, dst_gen); -#ifdef DEBUG_LEGION - assert(dst_index >= 0); -#endif - // See if the dependence record already exists - const std::pair src_key(src_index,src_idx); - const std::pair dst_key(dst_index,dst_idx); - std::map,unsigned>::iterator - src_record_finder = dependence_map.find(src_key); - if (src_record_finder != dependence_map.end()) - { - // Already have a source record, see if we have - // a destination record too - std::map,unsigned>::iterator - dst_record_finder = dependence_map.find(dst_key); - if (dst_record_finder == dependence_map.end()) - { - // Update the destination record entry - dependence_map[dst_key] = src_record_finder->second; - dependences[src_record_finder->second]->add_entry(dst_index, - dst_idx); - } -#ifdef DEBUG_LEGION - else // both already there so just assert they are the same - assert(src_record_finder->second == dst_record_finder->second); -#endif - } - else - { - // No source record - // See if we have a destination record entry - std::map,unsigned>::iterator - dst_record_finder = dependence_map.find(dst_key); - if (dst_record_finder == dependence_map.end()) - { - // Neither source nor destination have an entry so - // make a new record - DependenceRecord *new_record = new DependenceRecord(); - new_record->add_entry(src_index, src_idx); - new_record->add_entry(dst_index, dst_idx); - unsigned record_index = dependences.size(); - dependence_map[src_key] = record_index; - dependence_map[dst_key] = record_index; - dependences.push_back(new_record); - } - else - { - // Have a destination but no source, so update the source - dependence_map[src_key] = dst_record_finder->second; - dependences[dst_record_finder->second]->add_entry(src_index, - src_idx); - } - } - return false; + MustEpochIndexArgs args(current, index_tasks[idx], owner); + RtEvent wait = + owner->runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); } - // NO_DEPENDENCE and PROMOTED_DEPENDENCE are not errors - // and do not need to be recorded } - return true; + // Wait for all of the launches to be done + // We can safely block to free up the utility processor + if (!wait_events.empty()) + { + RtEvent trigger_event = Runtime::merge_events(wait_events); + trigger_event.wait(); + } } //-------------------------------------------------------------------------- - void MustEpochOp::must_epoch_map_task_callback(SingleTask *task, - Mapper::MapTaskInput &map_input, - Mapper::MapTaskOutput &map_output) + /*static*/ void MustEpochOp::handle_trigger_individual(const void *args) + //-------------------------------------------------------------------------- + { + const MustEpochIndivArgs *indiv_args = (const MustEpochIndivArgs*)args; + indiv_args->task->set_target_proc(indiv_args->current_proc); + indiv_args->task->trigger_mapping(); + } + + //-------------------------------------------------------------------------- + /*static*/ void MustEpochOp::handle_trigger_index(const void *args) + //-------------------------------------------------------------------------- + { + const MustEpochIndexArgs *index_args = (const MustEpochIndexArgs*)args; + index_args->task->set_target_proc(index_args->current_proc); + index_args->task->trigger_mapping(); + } + + //-------------------------------------------------------------------------- + void MustEpochOp::map_tasks(void) const //-------------------------------------------------------------------------- { - // We have to do three things here - // 1. Update the target processor - // 2. Mark as inputs and outputs any regions which we know - // the results for as a result of our must epoch mapping - // 3. Record that we premapped those regions - // First find the index for this task #ifdef DEBUG_LEGION - assert(single_task_map.find(task) != single_task_map.end()); + assert(single_tasks.size() == mapping_dependences.size()); #endif - unsigned index = single_task_map[task]; - // Set the target processor by the index - task->target_proc = output.task_processors[index]; - // Now iterate over the constraints figure out which ones - // apply to this task - std::pair key(index,0); - for (unsigned idx = 0; idx < task->regions.size(); idx++) + MustEpochMapArgs args(const_cast(this)); + // For correctness we still have to abide by the mapping dependences + // computed on the individual tasks while we are mapping them + std::vector mapped_events(single_tasks.size()); + for (unsigned idx = 0; idx < single_tasks.size(); idx++) { - key.second = idx; - std::map,unsigned>::const_iterator - record_finder = dependence_map.find(key); - if (record_finder != dependence_map.end()) + // Figure out our preconditions + std::set preconditions; + for (std::set::const_iterator it = + mapping_dependences[idx].begin(); it != + mapping_dependences[idx].end(); it++) { - map_input.valid_instances[idx] = - output.constraint_mappings[record_finder->second]; - map_output.chosen_instances[idx] = - output.constraint_mappings[record_finder->second]; - // Also record that we premapped this - map_input.premapped_regions.push_back(idx); +#ifdef DEBUG_LEGION + assert((*it) < idx); +#endif + preconditions.insert(mapped_events[*it]); + } + args.task = single_tasks[idx]; + if (!preconditions.empty()) + { + RtEvent precondition = Runtime::merge_events(preconditions); + mapped_events[idx] = + runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY, precondition); } + else + mapped_events[idx] = + runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + } + std::set wait_events(mapped_events.begin(), mapped_events.end()); + if (!wait_events.empty()) + { + RtEvent mapped_event = Runtime::merge_events(wait_events); + mapped_event.wait(); } } //-------------------------------------------------------------------------- - std::map* - MustEpochOp::get_acquired_instances_ref(void) + void MustEpochOp::map_single_task(SingleTask *task) //-------------------------------------------------------------------------- { - return &acquired_instances; + RtEvent done_mapping = task->perform_mapping(this); + if (done_mapping.exists()) + done_mapping.wait(); } //-------------------------------------------------------------------------- - void MustEpochOp::add_mapping_dependence(RtEvent precondition) + /*static*/ void MustEpochOp::handle_map_task(const void *args) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(mapping_tracker != NULL); -#endif - mapping_tracker->add_mapping_dependence(precondition); + const MustEpochMapArgs *map_args = (const MustEpochMapArgs*)args; + map_args->owner->map_single_task(map_args->task); } //-------------------------------------------------------------------------- - void MustEpochOp::register_single_task(SingleTask *single, unsigned index) + void MustEpochOp::distribute_tasks(void) const //-------------------------------------------------------------------------- { - // Can do the first part without the lock + MustEpochOp *owner = const_cast(this); + MustEpochDistributorArgs dist_args(owner); + MustEpochLauncherArgs launch_args(owner); + std::set wait_events; + for (std::vector::const_iterator it = + indiv_tasks.begin(); it != indiv_tasks.end(); it++) + { + if (!runtime->is_local((*it)->target_proc)) + { + dist_args.task = *it; + RtEvent wait = + runtime->issue_runtime_meta_task(dist_args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); + } + else + { + launch_args.task = *it; + RtEvent wait = + runtime->issue_runtime_meta_task(launch_args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); + } + } + for (std::set::const_iterator it = + slice_tasks.begin(); it != slice_tasks.end(); it++) + { + (*it)->update_target_processor(); + if (!runtime->is_local((*it)->target_proc)) + { + dist_args.task = *it; + RtEvent wait = + runtime->issue_runtime_meta_task(dist_args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); + } + else + { + launch_args.task = *it; + RtEvent wait = + runtime->issue_runtime_meta_task(launch_args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); + } + } + if (!wait_events.empty()) + { + RtEvent dist_event = Runtime::merge_events(wait_events); + dist_event.wait(); + } + } + + //-------------------------------------------------------------------------- + RtUserEvent MustEpochOp::compute_launch_space( + const MustEpochLauncher &launcher) + //-------------------------------------------------------------------------- + { + const size_t single_tasks = launcher.single_tasks.size(); + const size_t multi_tasks = launcher.index_tasks.size(); #ifdef DEBUG_LEGION - assert(index < task_sets.size()); + assert(!launch_domain.exists()); + assert((single_tasks > 0) || (multi_tasks > 0)); #endif - task_sets[index].insert(single); - AutoLock o_lock(op_lock); - const unsigned single_task_index = single_tasks.size(); - single_tasks.push_back(single); - single_task_map[single] = single_task_index; + RtUserEvent result; + if (multi_tasks > 0) + { + RegionTreeForest *forest = runtime->forest; + if ((single_tasks > 0) || (multi_tasks > 1)) + { + Realm::ProfilingRequestSet no_reqs; + // Need to compute the index tasks + switch (launcher.index_tasks[0].launch_domain.get_dim()) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + std::vector > \ + subspaces(single_tasks + multi_tasks); \ + for (unsigned idx = 0; idx < multi_tasks; idx++) \ + { \ + if (launcher.index_tasks[idx].launch_domain.exists()) \ + { \ + const Rect rect = \ + launcher.index_tasks[idx].launch_domain; \ + subspaces[idx] = rect; \ + } \ + else \ + { \ + Domain domain; \ + forest->find_launch_space_domain( \ + launcher.index_tasks[idx].launch_space, domain); \ + const DomainT domaint = domain; \ + subspaces[idx] = domaint; \ + } \ + } \ + for (unsigned idx = 0; idx < single_tasks; idx++) \ + { \ + const Point p = \ + launcher.single_tasks[idx].point; \ + const Rect rect(p,p); \ + subspaces[multi_tasks + idx] = \ + Realm::IndexSpace(rect); \ + } \ + Realm::IndexSpace space; \ + const RtEvent wait_on(\ + Realm::IndexSpace::compute_union( \ + subspaces, space, no_reqs)); \ + const DomainT domaint(space); \ + launch_domain = domaint; \ + if (!space.dense()) \ + { \ + result = Runtime::create_rt_user_event(); \ + space.destroy(result); \ + } \ + if (wait_on.exists()) \ + wait_on.wait(); \ + break; \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + } + else // Easy case of a single index task + { + launch_domain = launcher.index_tasks[0].launch_domain; + if (!launch_domain.exists()) + forest->find_launch_space_domain( + launcher.index_tasks[0].launch_space, launch_domain); + } + } + else + { + // These are just point tasks + if (single_tasks > 1) + { + switch (launcher.single_tasks[0].point.get_dim()) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + std::vector > points(single_tasks); \ + for (unsigned idx = 0; idx < single_tasks; idx++) \ + { \ + const Point point = \ + launcher.single_tasks[idx].point; \ + points[idx] = point; \ + } \ + Realm::IndexSpace space(points); \ + const DomainT domaint(space); \ + launch_domain = domaint; \ + if (!space.dense()) \ + { \ + result = Runtime::create_rt_user_event(); \ + space.destroy(result); \ + } \ + break; \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + } + else // Easy case of a single point task + { + DomainPoint point = launcher.single_tasks[0].point; + launch_domain = Domain(point, point); + } + } + return result; } //-------------------------------------------------------------------------- - void MustEpochOp::register_slice_task(SliceTask *slice) + /*static*/ void MustEpochOp::handle_distribute_task(const void *args) //-------------------------------------------------------------------------- { - AutoLock o_lock(op_lock); - slice_tasks.insert(slice); + const MustEpochDistributorArgs *dist_args = + (const MustEpochDistributorArgs*)args; + dist_args->task->distribute_task(); } //-------------------------------------------------------------------------- - void MustEpochOp::register_subop(Operation *op) + /*static*/ void MustEpochOp::handle_launch_task(const void *args) //-------------------------------------------------------------------------- { - AutoLock o_lock(op_lock); - remaining_subop_completes++; - remaining_subop_commits++; + const MustEpochLauncherArgs *launch_args = + (const MustEpochLauncherArgs *)args; + launch_args->task->launch_task(); } //-------------------------------------------------------------------------- - void MustEpochOp::notify_subop_complete(Operation *op) + void MustEpochOp::trigger_complete(void) //-------------------------------------------------------------------------- { bool need_complete; @@ -12574,7 +12822,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - void MustEpochOp::notify_subop_commit(Operation *op) + void MustEpochOp::trigger_commit(void) //-------------------------------------------------------------------------- { bool need_commit; @@ -12591,387 +12839,317 @@ namespace Legion { } //-------------------------------------------------------------------------- - RtUserEvent MustEpochOp::find_slice_versioning_event(UniqueID slice_id, - bool &first) - //-------------------------------------------------------------------------- - { - AutoLock o_lock(op_lock); - std::map::const_iterator finder = - slice_version_events.find(slice_id); - if (finder == slice_version_events.end()) - { - first = true; - RtUserEvent result = Runtime::create_rt_user_event(); - slice_version_events[slice_id] = result; - return result; - } - else - { - first = false; - return finder->second; - } - } - - //-------------------------------------------------------------------------- - int MustEpochOp::find_operation_index(Operation *op, GenerationID op_gen) + void MustEpochOp::verify_dependence(Operation *src_op, GenerationID src_gen, + Operation *dst_op, GenerationID dst_gen) //-------------------------------------------------------------------------- { - for (unsigned idx = 0; idx < indiv_tasks.size(); idx++) - { - if ((indiv_tasks[idx] == op) && - (indiv_tasks[idx]->get_generation() == op_gen)) - return idx; - } - for (unsigned idx = 0; idx < index_tasks.size(); idx++) + // If they are the same, then we can ignore them + if ((src_op == dst_op) && (src_gen == dst_gen)) + return; + // Check to see if the source is one of our operations, if it is + // then we have an actual dependence which is an error. + int src_index = find_operation_index(src_op, src_gen); + if (src_index >= 0) { - if ((index_tasks[idx] == op) && - (index_tasks[idx]->get_generation() == op_gen)) - return (idx+indiv_tasks.size()); + int dst_index = find_operation_index(dst_op, dst_gen); + if (dst_index >= 0) + { + TaskOp *src_task = find_task_by_index(src_index); + TaskOp *dst_task = find_task_by_index(dst_index); + REPORT_LEGION_ERROR(ERROR_MUST_EPOCH_DEPENDENCE, + "MUST EPOCH ERROR: dependence between task " + "%s (ID %lld) and task %s (ID %lld)\n", + src_task->get_task_name(), src_task->get_unique_id(), + dst_task->get_task_name(), dst_task->get_unique_id()) + } } - return -1; } - - //-------------------------------------------------------------------------- - TaskOp* MustEpochOp::find_task_by_index(int index) - //-------------------------------------------------------------------------- - { - assert(index >= 0); - if ((size_t)index < indiv_tasks.size()) - return indiv_tasks[index]; - index -= indiv_tasks.size(); - if ((size_t)index < index_tasks.size()) - return index_tasks[index]; - assert(false); - return NULL; - } - - ///////////////////////////////////////////////////////////// - // Must Epoch Triggerer - ///////////////////////////////////////////////////////////// - - //-------------------------------------------------------------------------- - MustEpochTriggerer::MustEpochTriggerer(MustEpochOp *own) - : current_proc(own->get_context()->get_executing_processor()), owner(own) - //-------------------------------------------------------------------------- - { - trigger_lock = Reservation::create_reservation(); - } - - //-------------------------------------------------------------------------- - MustEpochTriggerer::MustEpochTriggerer(const MustEpochTriggerer &rhs) - : current_proc(rhs.current_proc), owner(rhs.owner) - //-------------------------------------------------------------------------- - { - // should never be called - assert(false); - } - - //-------------------------------------------------------------------------- - MustEpochTriggerer::~MustEpochTriggerer(void) - //-------------------------------------------------------------------------- - { - trigger_lock.destroy_reservation(); - trigger_lock = Reservation::NO_RESERVATION; - } - - //-------------------------------------------------------------------------- - MustEpochTriggerer& MustEpochTriggerer::operator=( - const MustEpochTriggerer &rhs) - //-------------------------------------------------------------------------- - { - // should never be called - assert(false); - return *this; - } - + //-------------------------------------------------------------------------- - void MustEpochTriggerer::trigger_tasks( - const std::vector &indiv_tasks, - std::vector &indiv_triggered, - const std::vector &index_tasks, - std::vector &index_triggered) + bool MustEpochOp::record_dependence(Operation *src_op, GenerationID src_gen, + Operation *dst_op, GenerationID dst_gen, + unsigned src_idx, unsigned dst_idx, + DependenceType dtype) //-------------------------------------------------------------------------- { - std::set wait_events; - for (unsigned idx = 0; idx < indiv_triggered.size(); idx++) + // If they are the same we can ignore them + if ((src_op == dst_op) && (src_gen == dst_gen)) + return true; + // Check to see if the source is one of our operations + int src_index = find_operation_index(src_op, src_gen); + int dst_index = find_operation_index(dst_op, dst_gen); + if ((src_index >= 0) && (dst_index >= 0)) { - if (!indiv_triggered[idx]) + // If it is, see what kind of dependence we have + if ((dtype == LEGION_TRUE_DEPENDENCE) || + (dtype == LEGION_ANTI_DEPENDENCE) || + (dtype == LEGION_ATOMIC_DEPENDENCE)) { - MustEpochIndivArgs args(this, owner, indiv_tasks[idx]); - RtEvent wait = - owner->runtime->issue_runtime_meta_task(args, - LG_THROUGHPUT_DEFERRED_PRIORITY); - if (wait.exists()) - wait_events.insert(wait); + TaskOp *src_task = find_task_by_index(src_index); + TaskOp *dst_task = find_task_by_index(dst_index); + REPORT_LEGION_ERROR(ERROR_MUST_EPOCH_DEPENDENCE, + "MUST EPOCH ERROR: dependence between region %d " + "of task %s (ID %lld) and region %d of task %s (ID %lld) of " + " type %s", src_idx, src_task->get_task_name(), + src_task->get_unique_id(), dst_idx, + dst_task->get_task_name(), dst_task->get_unique_id(), + (dtype == LEGION_TRUE_DEPENDENCE) ? "TRUE DEPENDENCE" : + (dtype == LEGION_ANTI_DEPENDENCE) ? "ANTI DEPENDENCE" : + "ATOMIC DEPENDENCE") } - } - for (unsigned idx = 0; idx < index_tasks.size(); idx++) - { - if (!index_triggered[idx]) + else if (dtype == LEGION_SIMULTANEOUS_DEPENDENCE) { - MustEpochIndexArgs args(this, owner, index_tasks[idx]); - RtEvent wait = - owner->runtime->issue_runtime_meta_task(args, - LG_THROUGHPUT_DEFERRED_PRIORITY); - if (wait.exists()) - wait_events.insert(wait); + // Record the dependence kind + int dst_index = find_operation_index(dst_op, dst_gen); +#ifdef DEBUG_LEGION + assert(dst_index >= 0); +#endif + // See if the dependence record already exists + const std::pair src_key(src_index,src_idx); + const std::pair dst_key(dst_index,dst_idx); + std::map,unsigned>::iterator + src_record_finder = dependence_map.find(src_key); + if (src_record_finder != dependence_map.end()) + { + // Already have a source record, see if we have + // a destination record too + std::map,unsigned>::iterator + dst_record_finder = dependence_map.find(dst_key); + if (dst_record_finder == dependence_map.end()) + { + // Update the destination record entry + dependence_map[dst_key] = src_record_finder->second; + dependences[src_record_finder->second]->add_entry(dst_index, + dst_idx); + } +#ifdef DEBUG_LEGION + else // both already there so just assert they are the same + assert(src_record_finder->second == dst_record_finder->second); +#endif + } + else + { + // No source record + // See if we have a destination record entry + std::map,unsigned>::iterator + dst_record_finder = dependence_map.find(dst_key); + if (dst_record_finder == dependence_map.end()) + { + // Neither source nor destination have an entry so + // make a new record + DependenceRecord *new_record = new DependenceRecord(); + new_record->add_entry(src_index, src_idx); + new_record->add_entry(dst_index, dst_idx); + unsigned record_index = dependences.size(); + dependence_map[src_key] = record_index; + dependence_map[dst_key] = record_index; + dependences.push_back(new_record); + } + else + { + // Have a destination but no source, so update the source + dependence_map[src_key] = dst_record_finder->second; + dependences[dst_record_finder->second]->add_entry(src_index, + src_idx); + } + } + return false; } + // NO_DEPENDENCE and PROMOTED_DEPENDENCE are not errors + // and do not need to be recorded } - // Wait for all of the launches to be done - // We can safely block to free up the utility processor - if (!wait_events.empty()) - { - RtEvent trigger_event = Runtime::merge_events(wait_events); - trigger_event.wait(); - } - } - - //-------------------------------------------------------------------------- - void MustEpochTriggerer::trigger_individual(IndividualTask *task) - //-------------------------------------------------------------------------- - { - task->set_target_proc(current_proc); - task->trigger_mapping(); - } - - //-------------------------------------------------------------------------- - void MustEpochTriggerer::trigger_index(IndexTask *task) - //-------------------------------------------------------------------------- - { - task->set_target_proc(current_proc); - task->trigger_mapping(); - } - - //-------------------------------------------------------------------------- - /*static*/ void MustEpochTriggerer::handle_individual(const void *args) - //-------------------------------------------------------------------------- - { - const MustEpochIndivArgs *indiv_args = (const MustEpochIndivArgs*)args; - indiv_args->triggerer->trigger_individual(indiv_args->task); - } - - //-------------------------------------------------------------------------- - /*static*/ void MustEpochTriggerer::handle_index(const void *args) - //-------------------------------------------------------------------------- - { - const MustEpochIndexArgs *index_args = (const MustEpochIndexArgs*)args; - index_args->triggerer->trigger_index(index_args->task); - } - - ///////////////////////////////////////////////////////////// - // Must Epoch Mapper - ///////////////////////////////////////////////////////////// - - //-------------------------------------------------------------------------- - MustEpochMapper::MustEpochMapper(MustEpochOp *own) - : owner(own) - //-------------------------------------------------------------------------- - { - } - - //-------------------------------------------------------------------------- - MustEpochMapper::MustEpochMapper(const MustEpochMapper &rhs) - : owner(rhs.owner) - //-------------------------------------------------------------------------- - { - // should never be called - assert(false); + return true; } //-------------------------------------------------------------------------- - MustEpochMapper::~MustEpochMapper(void) + void MustEpochOp::must_epoch_map_task_callback(SingleTask *task, + Mapper::MapTaskInput &map_input, + Mapper::MapTaskOutput &map_output) //-------------------------------------------------------------------------- { + // We have to do three things here + // 1. Update the target processor + // 2. Mark as inputs and outputs any regions which we know + // the results for as a result of our must epoch mapping + // 3. Record that we premapped those regions + // First find the index for this task +#ifdef DEBUG_LEGION + assert(single_task_map.find(task) != single_task_map.end()); +#endif + unsigned index = single_task_map[task]; + // Set the target processor by the index + task->target_proc = output.task_processors[index]; + // Now iterate over the constraints figure out which ones + // apply to this task + std::pair key(index,0); + for (unsigned idx = 0; idx < task->regions.size(); idx++) + { + key.second = idx; + std::map,unsigned>::const_iterator + record_finder = dependence_map.find(key); + if (record_finder != dependence_map.end()) + { + map_input.valid_instances[idx] = + output.constraint_mappings[record_finder->second]; + map_output.chosen_instances[idx] = + output.constraint_mappings[record_finder->second]; + // Also record that we premapped this + map_input.premapped_regions.push_back(idx); + } + } } //-------------------------------------------------------------------------- - MustEpochMapper& MustEpochMapper::operator=(const MustEpochMapper &rhs) + std::map* + MustEpochOp::get_acquired_instances_ref(void) //-------------------------------------------------------------------------- { - // should never be called - assert(false); - return *this; + return &acquired_instances; } //-------------------------------------------------------------------------- - void MustEpochMapper::map_tasks(const std::deque &single_tasks, - const std::vector > &dependences) + void MustEpochOp::add_mapping_dependence(RtEvent precondition) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert(single_tasks.size() == dependences.size()); -#endif - MustEpochMapArgs args(this, owner); - // For correctness we still have to abide by the mapping dependences - // computed on the individual tasks while we are mapping them - std::vector mapped_events(single_tasks.size()); - for (unsigned idx = 0; idx < single_tasks.size(); idx++) - { - // Figure out our preconditions - std::set preconditions; - for (std::set::const_iterator it = - dependences[idx].begin(); it != dependences[idx].end(); it++) - { -#ifdef DEBUG_LEGION - assert((*it) < idx); + assert(mapping_tracker != NULL); #endif - preconditions.insert(mapped_events[*it]); - } - args.task = single_tasks[idx]; - if (!preconditions.empty()) - { - RtEvent precondition = Runtime::merge_events(preconditions); - mapped_events[idx] = - owner->runtime->issue_runtime_meta_task(args, - LG_THROUGHPUT_DEFERRED_PRIORITY, precondition); - } - else - mapped_events[idx] = - owner->runtime->issue_runtime_meta_task(args, - LG_THROUGHPUT_DEFERRED_PRIORITY); - } - std::set wait_events(mapped_events.begin(), mapped_events.end()); - if (!wait_events.empty()) - { - RtEvent mapped_event = Runtime::merge_events(wait_events); - mapped_event.wait(); - } + mapping_tracker->add_mapping_dependence(precondition); } //-------------------------------------------------------------------------- - void MustEpochMapper::map_task(SingleTask *task) + void MustEpochOp::register_single_task(SingleTask *single, unsigned index) //-------------------------------------------------------------------------- { - // Note we don't need to hold a lock here because this is - // a monotonic change. Once it fails for anyone then it - // fails for everyone. - RtEvent done_mapping = task->perform_mapping(owner); - if (done_mapping.exists()) - done_mapping.wait(); + // Can do the first part without the lock +#ifdef DEBUG_LEGION + assert(index < task_sets.size()); +#endif + task_sets[index].insert(single); + AutoLock o_lock(op_lock); + single_tasks.push_back(single); } //-------------------------------------------------------------------------- - /*static*/ void MustEpochMapper::handle_map_task(const void *args) + void MustEpochOp::register_slice_task(SliceTask *slice) //-------------------------------------------------------------------------- { - const MustEpochMapArgs *map_args = (const MustEpochMapArgs*)args; - map_args->mapper->map_task(map_args->task); + AutoLock o_lock(op_lock); + slice_tasks.insert(slice); } - ///////////////////////////////////////////////////////////// - // Must Epoch Distributor - ///////////////////////////////////////////////////////////// - //-------------------------------------------------------------------------- - MustEpochDistributor::MustEpochDistributor(MustEpochOp *own) - : owner(own) + void MustEpochOp::set_future(const DomainPoint &point, const void *result, + size_t result_size, bool owner) //-------------------------------------------------------------------------- { + Future f = result_map.impl->get_future(point, true/*internal*/); + f.impl->set_result(result, result_size, owner); } //-------------------------------------------------------------------------- - MustEpochDistributor::MustEpochDistributor(const MustEpochDistributor &rhs) - : owner(rhs.owner) + void MustEpochOp::register_subop(Operation *op) //-------------------------------------------------------------------------- { - // should never be called - assert(false); + AutoLock o_lock(op_lock); + remaining_subop_completes++; + remaining_subop_commits++; } //-------------------------------------------------------------------------- - MustEpochDistributor::~MustEpochDistributor(void) + void MustEpochOp::notify_subop_complete(Operation *op) //-------------------------------------------------------------------------- { + bool need_complete; + { + AutoLock o_lock(op_lock); +#ifdef DEBUG_LEGION + assert(remaining_subop_completes > 0); +#endif + remaining_subop_completes--; + need_complete = (remaining_subop_completes == 0); + } + if (need_complete) + { +#ifdef LEGION_SPY + // Still need this for Legion Spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + complete_operation(); + } } //-------------------------------------------------------------------------- - MustEpochDistributor& MustEpochDistributor::operator=( - const MustEpochDistributor &rhs) + void MustEpochOp::notify_subop_commit(Operation *op) //-------------------------------------------------------------------------- { - // should never be called - assert(false); - return *this; + bool need_commit; + { + AutoLock o_lock(op_lock); +#ifdef DEBUG_LEGION + assert(remaining_subop_commits > 0); +#endif + remaining_subop_commits--; + need_commit = (remaining_subop_commits == 0); + } + if (need_commit) + commit_operation(true/*deactivate*/); } //-------------------------------------------------------------------------- - void MustEpochDistributor::distribute_tasks(Runtime *runtime, - const std::vector &indiv_tasks, - const std::set &slice_tasks) + RtUserEvent MustEpochOp::find_slice_versioning_event(UniqueID slice_id, + bool &first) //-------------------------------------------------------------------------- { - MustEpochDistributorArgs dist_args(owner); - MustEpochLauncherArgs launch_args(owner); - std::set wait_events; - for (std::vector::const_iterator it = - indiv_tasks.begin(); it != indiv_tasks.end(); it++) - { - if (!runtime->is_local((*it)->target_proc)) - { - dist_args.task = *it; - RtEvent wait = - runtime->issue_runtime_meta_task(dist_args, - LG_THROUGHPUT_DEFERRED_PRIORITY); - if (wait.exists()) - wait_events.insert(wait); - } - else - { - launch_args.task = *it; - RtEvent wait = - runtime->issue_runtime_meta_task(launch_args, - LG_THROUGHPUT_DEFERRED_PRIORITY); - if (wait.exists()) - wait_events.insert(wait); - } - } - for (std::set::const_iterator it = - slice_tasks.begin(); it != slice_tasks.end(); it++) + AutoLock o_lock(op_lock); + std::map::const_iterator finder = + slice_version_events.find(slice_id); + if (finder == slice_version_events.end()) { - (*it)->update_target_processor(); - if (!runtime->is_local((*it)->target_proc)) - { - dist_args.task = *it; - RtEvent wait = - runtime->issue_runtime_meta_task(dist_args, - LG_THROUGHPUT_DEFERRED_PRIORITY); - if (wait.exists()) - wait_events.insert(wait); - } - else - { - launch_args.task = *it; - RtEvent wait = - runtime->issue_runtime_meta_task(launch_args, - LG_THROUGHPUT_DEFERRED_PRIORITY); - if (wait.exists()) - wait_events.insert(wait); - } + first = true; + RtUserEvent result = Runtime::create_rt_user_event(); + slice_version_events[slice_id] = result; + return result; } - if (!wait_events.empty()) + else { - RtEvent dist_event = Runtime::merge_events(wait_events); - dist_event.wait(); + first = false; + return finder->second; } } //-------------------------------------------------------------------------- - /*static*/ void MustEpochDistributor::handle_distribute_task( - const void *args) + int MustEpochOp::find_operation_index(Operation *op, GenerationID op_gen) //-------------------------------------------------------------------------- { - const MustEpochDistributorArgs *dist_args = - (const MustEpochDistributorArgs*)args; - dist_args->task->distribute_task(); + for (unsigned idx = 0; idx < indiv_tasks.size(); idx++) + { + if ((indiv_tasks[idx] == op) && + (indiv_tasks[idx]->get_generation() == op_gen)) + return idx; + } + for (unsigned idx = 0; idx < index_tasks.size(); idx++) + { + if ((index_tasks[idx] == op) && + (index_tasks[idx]->get_generation() == op_gen)) + return (idx+indiv_tasks.size()); + } + return -1; } //-------------------------------------------------------------------------- - /*static*/ void MustEpochDistributor::handle_launch_task(const void *args) + TaskOp* MustEpochOp::find_task_by_index(int index) //-------------------------------------------------------------------------- { - const MustEpochLauncherArgs *launch_args = - (const MustEpochLauncherArgs *)args; - launch_args->task->launch_task(); + assert(index >= 0); + if ((size_t)index < indiv_tasks.size()) + return indiv_tasks[index]; + index -= indiv_tasks.size(); + if ((size_t)index < index_tasks.size()) + return index_tasks[index]; + assert(false); + return NULL; } ///////////////////////////////////////////////////////////// @@ -13293,6 +13471,13 @@ namespace Legion { //-------------------------------------------------------------------------- void PendingPartitionOp::activate(void) //-------------------------------------------------------------------------- + { + activate_pending(); + } + + //-------------------------------------------------------------------------- + void PendingPartitionOp::activate_pending(void) + //-------------------------------------------------------------------------- { activate_operation(); } @@ -13300,13 +13485,20 @@ namespace Legion { //-------------------------------------------------------------------------- void PendingPartitionOp::deactivate(void) //-------------------------------------------------------------------------- + { + deactivate_pending(); + runtime->free_pending_partition_op(this); + } + + //-------------------------------------------------------------------------- + void PendingPartitionOp::deactivate_pending(void) + //-------------------------------------------------------------------------- { deactivate_operation(); if (thunk != NULL) delete thunk; thunk = NULL; future_map = FutureMap(); // clear any references - runtime->free_pending_partition_op(this); } //-------------------------------------------------------------------------- @@ -13848,14 +14040,18 @@ namespace Legion { #ifdef DEBUG_LEGION assert(requirement.handle_type == LEGION_PARTITION_PROJECTION); #endif + // Need to get the launch domain in case it is different than + // the original index domain due to control replication + Domain launch_domain; + launch_space->get_launch_space_domain(launch_domain); // Now enumerate the points and kick them off - size_t num_points = index_domain.get_volume(); + size_t num_points = launch_domain.get_volume(); #ifdef DEBUG_LEGION assert(num_points > 0); #endif unsigned point_idx = 0; points.resize(num_points); - for (Domain::DomainPointIterator itr(index_domain); + for (Domain::DomainPointIterator itr(launch_domain); itr; itr++, point_idx++) { PointDepPartOp *point = @@ -13869,7 +14065,7 @@ namespace Legion { std::vector projection_points(points.begin(), points.end()); function->project_points(this, 0/*idx*/, requirement, - runtime, projection_points); + runtime, index_domain, projection_points); // No need to check the validity of the points, we know they are good if (runtime->legion_spy_enabled) { @@ -13878,11 +14074,10 @@ namespace Legion { (*it)->log_requirement(); } // Launch the points - std::set mapped_preconditions; for (std::vector::const_iterator it = points.begin(); it != points.end(); it++) { - mapped_preconditions.insert((*it)->get_mapped_event()); + map_applied_conditions.insert((*it)->get_mapped_event()); (*it)->launch(); } #ifdef LEGION_SPY @@ -13890,7 +14085,7 @@ namespace Legion { completion_event); #endif // We are mapped when all our points are mapped - complete_mapping(Runtime::merge_events(mapped_preconditions)); + finalize_mapping(); } else { @@ -13948,13 +14143,7 @@ namespace Legion { mapped_instances, trace_info, index_point); // Once we are done running these routines, we can mark // that the handles have all been completed - RtEvent mapping_applied; - if (!map_applied_conditions.empty()) - mapping_applied = Runtime::merge_events(map_applied_conditions); - if (!acquired_instances.empty()) - mapping_applied = release_nonempty_acquired_instances(mapping_applied, - acquired_instances); - complete_mapping(mapping_applied); + finalize_mapping(); #ifdef LEGION_SPY if (runtime->legion_spy_enabled) LegionSpy::log_operation_events(unique_op_id, done_event, @@ -13964,6 +14153,19 @@ namespace Legion { complete_execution(Runtime::protect_event(done_event)); } + //-------------------------------------------------------------------------- + void DependentPartitionOp::finalize_mapping(void) + //-------------------------------------------------------------------------- + { + RtEvent mapping_applied; + if (!map_applied_conditions.empty()) + mapping_applied = Runtime::merge_events(map_applied_conditions); + if (!acquired_instances.empty()) + mapping_applied = release_nonempty_acquired_instances(mapping_applied, + acquired_instances); + complete_mapping(mapping_applied); + } + //-------------------------------------------------------------------------- ApEvent DependentPartitionOp::trigger_thunk(IndexSpace handle, const InstanceSet &mapped_insts, @@ -14444,6 +14646,9 @@ namespace Legion { launch_space = NULL; index_domain = Domain::NO_DOMAIN; parent_req_index = 0; + thunk = NULL; + // can be changed for control rep + partition_ready = get_completion_event(); mapper = NULL; points_committed = 0; commit_request = false; @@ -14944,7 +15149,7 @@ namespace Legion { std::set preconditions; runtime->forest->perform_versioning_analysis(this, 0/*idx*/, requirement, version_info, preconditions); - // We can also mark this as having our resolved any predication + // We can also mark this as having resolved any predication resolve_speculation(); // Then put ourselves in the queue of operations ready to map if (!preconditions.empty()) @@ -15201,7 +15406,10 @@ namespace Legion { wait_barriers = launcher.wait_barriers; arrive_barriers = launcher.arrive_barriers; map_id = launcher.map_id; - tag = launcher.tag; + tag = launcher.tag; + index_point = launcher.point; + index_domain = Domain(index_point, index_point); + sharding_space = launcher.sharding_space; if (runtime->legion_spy_enabled) { LegionSpy::log_fill_operation(parent_ctx->get_unique_id(), @@ -15905,6 +16113,7 @@ namespace Legion { launch_space->get_launch_space_domain(index_domain); else index_domain = launcher.launch_domain; + sharding_space = launcher.sharding_space; if (launcher.region.exists()) { #ifdef DEBUG_LEGION @@ -15952,9 +16161,17 @@ namespace Legion { //-------------------------------------------------------------------------- void IndexFillOp::activate(void) //-------------------------------------------------------------------------- + { + activate_index_fill(); + } + + //-------------------------------------------------------------------------- + void IndexFillOp::activate_index_fill(void) + //-------------------------------------------------------------------------- { activate_fill(); index_domain = Domain::NO_DOMAIN; + sharding_space = IndexSpace::NO_SPACE; launch_space = NULL; points_committed = 0; commit_request = false; @@ -15963,6 +16180,15 @@ namespace Legion { //-------------------------------------------------------------------------- void IndexFillOp::deactivate(void) //-------------------------------------------------------------------------- + { + deactivate_index_fill(); + // Return the operation to the runtime + runtime->free_index_fill_op(this); + } + + //-------------------------------------------------------------------------- + void IndexFillOp::deactivate_index_fill(void) + //-------------------------------------------------------------------------- { deactivate_fill(); // We can deactivate our point operations @@ -15972,8 +16198,6 @@ namespace Legion { points.clear(); if (remove_launch_space_reference(launch_space)) delete launch_space; - // Return the operation to the runtime - runtime->free_index_fill_op(this); } //-------------------------------------------------------------------------- @@ -16031,6 +16255,17 @@ namespace Legion { map_applied_conditions); } + //-------------------------------------------------------------------------- + void IndexFillOp::perform_base_dependence_analysis(void) + //-------------------------------------------------------------------------- + { + // Register a dependence on our predicate + register_predicate_dependence(); + // If we are waiting on a future register a dependence + if (future.impl != NULL) + future.impl->register_dependence(this); + } + //-------------------------------------------------------------------------- void IndexFillOp::trigger_ready(void) //-------------------------------------------------------------------------- @@ -16118,13 +16353,18 @@ namespace Legion { //-------------------------------------------------------------------------- { // Enumerate the points - size_t num_points = index_domain.get_volume(); + // Need to get the launch domain in case it is different than + // the original index domain due to control replication + Domain launch_domain; + launch_space->get_launch_space_domain(launch_domain); + // Now enumerate the points + size_t num_points = launch_domain.get_volume(); #ifdef DEBUG_LEGION assert(num_points > 0); #endif unsigned point_idx = 0; points.resize(num_points); - for (Domain::DomainPointIterator itr(index_domain); + for (Domain::DomainPointIterator itr(launch_domain); itr; itr++, point_idx++) { PointFillOp *point = runtime->get_available_point_fill_op(); @@ -16137,13 +16377,14 @@ namespace Legion { std::vector projection_points(points.begin(), points.end()); function->project_points(this, 0/*idx*/, requirement, - runtime, projection_points); + runtime, index_domain, projection_points); if (runtime->legion_spy_enabled && !replaying) { for (std::vector::const_iterator it = points.begin(); it != points.end(); it++) (*it)->log_fill_requirement(); } + } //-------------------------------------------------------------------------- @@ -16260,7 +16501,8 @@ namespace Legion { // Initialize the operation initialize_operation(own->get_context(), false/*track*/, 1/*regions*/); index_point = p; - index_domain = own->index_domain; + index_domain = own->index_domain; + sharding_space = own->sharding_space; owner = own; execution_fence_event = own->get_execution_fence_event(); // From Memoizable @@ -16339,7 +16581,7 @@ namespace Legion { std::set preconditions; runtime->forest->perform_versioning_analysis(this, 0/*idx*/, requirement, version_info, preconditions); - // We can also mark this as having our resolved any predication + // We can also mark this as having resolved any predication resolve_speculation(); if (!preconditions.empty()) enqueue_ready_operation(Runtime::merge_events(preconditions)); @@ -16465,6 +16707,7 @@ namespace Legion { footprint = launcher.footprint; restricted = launcher.restricted; mapping = launcher.mapped; + local_files = launcher.local_files; switch (resource) { case LEGION_EXTERNAL_POSIX_FILE: @@ -16613,17 +16856,18 @@ namespace Legion { } //-------------------------------------------------------------------------- - void AttachOp::activate(void) + void AttachOp::activate_attach_op(void) //-------------------------------------------------------------------------- { activate_operation(); file_name = NULL; footprint = 0; restricted = true; + local_files = false; } //-------------------------------------------------------------------------- - void AttachOp::deactivate(void) + void AttachOp::deactivate_attach_op(void) //-------------------------------------------------------------------------- { deactivate_operation(); @@ -16644,6 +16888,20 @@ namespace Legion { version_info.clear(); map_applied_conditions.clear(); layout_constraint_set = LayoutConstraintSet(); + } + + //-------------------------------------------------------------------------- + void AttachOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_attach_op(); + } + + //-------------------------------------------------------------------------- + void AttachOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_attach_op(); runtime->free_attach_op(this); } @@ -17175,7 +17433,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - void DetachOp::activate(void) + void DetachOp::activate_detach_op(void) //-------------------------------------------------------------------------- { activate_operation(); @@ -17183,7 +17441,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - void DetachOp::deactivate(void) + void DetachOp::deactivate_detach_op(void) //-------------------------------------------------------------------------- { deactivate_operation(); @@ -17192,6 +17450,20 @@ namespace Legion { version_info.clear(); map_applied_conditions.clear(); result = Future(); // clear any references on the future + } + + //-------------------------------------------------------------------------- + void DetachOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_detach_op(); + } + + //-------------------------------------------------------------------------- + void DetachOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_detach_op(); runtime->free_detach_op(this); } @@ -17499,6 +17771,13 @@ namespace Legion { //-------------------------------------------------------------------------- void TimingOp::activate(void) //-------------------------------------------------------------------------- + { + activate_timing(); + } + + //-------------------------------------------------------------------------- + void TimingOp::activate_timing(void) + //-------------------------------------------------------------------------- { activate_operation(); } @@ -17506,11 +17785,18 @@ namespace Legion { //-------------------------------------------------------------------------- void TimingOp::deactivate(void) //-------------------------------------------------------------------------- + { + deactivate_timing(); + runtime->free_timing_op(this); + } + + //-------------------------------------------------------------------------- + void TimingOp::deactivate_timing(void) + //-------------------------------------------------------------------------- { deactivate_operation(); preconditions.clear(); result = Future(); - runtime->free_timing_op(this); } //-------------------------------------------------------------------------- @@ -17663,16 +17949,14 @@ namespace Legion { void AllReduceOp::activate(void) //-------------------------------------------------------------------------- { - activate_operation(); + activate_all_reduce(); } //-------------------------------------------------------------------------- void AllReduceOp::deactivate(void) //-------------------------------------------------------------------------- { - deactivate_operation(); - future_map = FutureMap(); - result = Future(); + deactivate_all_reduce(); runtime->free_all_reduce_op(this); } @@ -17690,6 +17974,22 @@ namespace Legion { return ALL_REDUCE_OP_KIND; } + //-------------------------------------------------------------------------- + void AllReduceOp::activate_all_reduce(void) + //-------------------------------------------------------------------------- + { + activate_operation(); + } + + //-------------------------------------------------------------------------- + void AllReduceOp::deactivate_all_reduce(void) + //-------------------------------------------------------------------------- + { + deactivate_operation(); + future_map = FutureMap(); + result = Future(); + } + //-------------------------------------------------------------------------- void AllReduceOp::trigger_dependence_analysis(void) //-------------------------------------------------------------------------- diff --git a/runtime/legion/legion_ops.h b/runtime/legion/legion_ops.h index 3676aad871..c6df81b257 100644 --- a/runtime/legion/legion_ops.h +++ b/runtime/legion/legion_ops.h @@ -485,6 +485,7 @@ namespace Legion { if (!runtime->program_order_execution) { need_completion_trigger = false; + __sync_synchronize(); Runtime::trigger_event(NULL, completion_event, chain_event); return true; } @@ -496,6 +497,7 @@ namespace Legion { if (!runtime->program_order_execution) { need_completion_trigger = false; + __sync_synchronize(); to_trigger = completion_event; return true; } @@ -1064,6 +1066,8 @@ namespace Legion { void initialize(InnerContext *ctx, const PhysicalRegion ®ion); inline const RegionRequirement& get_requirement(void) const { return requirement; } + protected: + void deactivate_map_op(void); public: virtual void activate(void); virtual void deactivate(void); @@ -1214,6 +1218,7 @@ namespace Legion { void activate_copy(void); void deactivate_copy(void); void log_copy_requirements(void) const; + void perform_base_dependence_analysis(void); public: virtual void activate(void); virtual void deactivate(void); @@ -1298,6 +1303,8 @@ namespace Legion { virtual void handle_profiling_update(int count); virtual void pack_remote_operation(Serializer &rez, AddressSpaceID target, std::set &applied) const; + // Separate function for this so it can be called by derived classes + RtEvent perform_local_versioning_analysis(void); public: std::vector src_privilege_paths; std::vector dst_privilege_paths; @@ -1359,7 +1366,10 @@ namespace Legion { IndexSpace launch_space); public: virtual void activate(void); - virtual void deactivate(void); + virtual void deactivate(void); + protected: + void activate_index_copy(void); + void deactivate_index_copy(void); public: virtual void trigger_prepipeline_stage(void); virtual void trigger_dependence_analysis(void); @@ -1488,7 +1498,8 @@ namespace Legion { public: FenceOp& operator=(const FenceOp &rhs); public: - Future initialize(InnerContext *ctx, FenceKind kind, bool need_future); + Future initialize(InnerContext *ctx, FenceKind kind, + bool need_future, bool track=true); inline void add_mapping_applied_condition(RtEvent precondition) { map_applied_conditions.insert(precondition); } public: @@ -1502,6 +1513,8 @@ namespace Legion { #ifdef LEGION_SPY virtual void trigger_complete(void); #endif + public: + void deactivate_fence(void); protected: void perform_fence_analysis(bool update_fence = false); void update_current_fence(void); @@ -1555,6 +1568,7 @@ namespace Legion { static const AllocationType alloc_type = CREATION_OP_ALLOC; public: enum CreationKind { + FENCE_CREATION, INDEX_SPACE_CREATION, FIELD_ALLOCATION, FUTURE_MAP_CREATION, @@ -1566,13 +1580,17 @@ namespace Legion { public: CreationOp& operator=(const CreationOp &rhs); public: - void initialize_index_space( - InnerContext *ctx, IndexSpaceNode *node, const Future &future); + void initialize_fence(InnerContext *ctx, RtEvent precondition); + void initialize_index_space(InnerContext *ctx, IndexSpaceNode *node, + const Future &future, bool owner = true, + ShardMapping *shard_mapping = NULL); void initialize_field(InnerContext *ctx, FieldSpaceNode *node, - FieldID fid, const Future &field_size); + FieldID fid, const Future &field_size, + RtEvent precondition, bool owner = true); void initialize_fields(InnerContext *ctx, FieldSpaceNode *node, const std::vector &fids, - const std::vector &field_sizes); + const std::vector &field_sizes, + RtEvent precondition, bool owner = true); void initialize_map(InnerContext *ctx, const std::map &futures); public: @@ -1590,6 +1608,9 @@ namespace Legion { FieldSpaceNode *field_space_node; std::vector futures; std::vector fields; + RtEvent mapping_precondition; + ShardMapping *shard_mapping; + bool owner; }; /** @@ -1645,6 +1666,9 @@ namespace Legion { virtual void deactivate(void); virtual const char* get_logging_name(void) const; virtual OpKind get_operation_kind(void) const; + protected: + void activate_deletion(void); + void deactivate_deletion(void); public: virtual void trigger_dependence_analysis(void); virtual void trigger_ready(void); @@ -2336,7 +2360,8 @@ namespace Legion { * these operations and ensures that they can all * be run in parallel or it reports an error. */ - class MustEpochOp : public Operation, public LegionHeapify { + class MustEpochOp : public Operation, public MustEpoch, + public LegionHeapify { public: static const AllocationType alloc_type = MUST_EPOCH_OP_ALLOC; public: @@ -2348,6 +2373,62 @@ namespace Legion { std::vector op_indexes; std::vector req_indexes; }; + public: + struct MustEpochIndivArgs : public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_MUST_INDIV_ID; + public: + MustEpochIndivArgs(Processor p, IndividualTask *t, MustEpochOp *o) + : LgTaskArgs(o->get_unique_op_id()), + current_proc(p), task(t) { } + public: + const Processor current_proc; + IndividualTask *const task; + }; + struct MustEpochIndexArgs : public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_MUST_INDEX_ID; + public: + MustEpochIndexArgs(Processor p, IndexTask *t, MustEpochOp *o) + : LgTaskArgs(o->get_unique_op_id()), + current_proc(p), task(t) { } + public: + const Processor current_proc; + IndexTask *const task; + }; + struct MustEpochMapArgs : public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_MUST_MAP_ID; + public: + MustEpochMapArgs(MustEpochOp *o) + : LgTaskArgs(o->get_unique_op_id()), + owner(o), task(NULL) { } + public: + MustEpochOp *const owner; + SingleTask *task; + }; + struct MustEpochDistributorArgs : + public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_MUST_DIST_ID; + public: + MustEpochDistributorArgs(MustEpochOp *o) + : LgTaskArgs(o->get_unique_op_id()), + task(NULL) { } + public: + TaskOp *task; + }; + struct MustEpochLauncherArgs : + public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_MUST_LAUNCH_ID; + public: + MustEpochLauncherArgs(MustEpochOp *o) + : LgTaskArgs(o->get_unique_op_id()), + task(NULL) { } + public: + TaskOp *task; + }; public: MustEpochOp(Runtime *rt); MustEpochOp(const MustEpochOp &rhs); @@ -2357,13 +2438,28 @@ namespace Legion { public: inline FutureMap get_future_map(void) const { return result_map; } public: - FutureMap initialize(InnerContext *ctx, - const MustEpochLauncher &launcher); + // From MustEpoch + virtual UniqueID get_unique_id(void) const; + virtual size_t get_context_index(void) const; + virtual int get_depth(void) const; + public: + FutureMap initialize(InnerContext *ctx,const MustEpochLauncher &launcher); + // Make this a virtual method so it can be overridden for + // control replicated version of must epoch op + virtual FutureMapImpl* create_future_map(TaskContext *ctx, + const Domain &domain, IndexSpace shard_space, RtUserEvent deleted); + // Another virtual method to override for control replication + virtual void instantiate_tasks(InnerContext *ctx, + const MustEpochLauncher &launcher); void find_conflicted_regions( std::vector &unmapped); public: virtual void activate(void); virtual void deactivate(void); + public: + void activate_must_epoch_op(void); + void deactivate_must_epoch_op(void); + public: virtual const char* get_logging_name(void) const; virtual size_t get_region_count(void) const; virtual OpKind get_operation_kind(void) const; @@ -2385,10 +2481,15 @@ namespace Legion { // Get a reference to our data structure for tracking acquired instances virtual std::map* get_acquired_instances_ref(void); + public: + // Make this a virtual method to override it for control replication + virtual MapperManager* invoke_mapper(void); public: void add_mapping_dependence(RtEvent precondition); void register_single_task(SingleTask *single, unsigned index); void register_slice_task(SliceTask *slice); + void set_future(const DomainPoint &point, + const void *result, size_t result_size, bool owned); public: // Methods for keeping track of when we can complete and commit void register_subop(Operation *op); @@ -2399,6 +2500,33 @@ namespace Legion { protected: int find_operation_index(Operation *op, GenerationID generation); TaskOp* find_task_by_index(int index); + protected: + static bool single_task_sorter(const Task *t1, const Task *t2); + public: + static void trigger_tasks(MustEpochOp *owner, + const std::vector &indiv_tasks, + std::vector &indiv_triggered, + const std::vector &index_tasks, + std::vector &index_triggered); + static void handle_trigger_individual(const void *args); + static void handle_trigger_index(const void *args); + protected: + // Have a virtual function that we can override to for doing the + // mapping and distribution of the point tasks, we'll override + // this for control replication + virtual void map_and_distribute(std::set &tasks_mapped, + std::set &tasks_complete); + // Make this virtual so we can override it for control replication + void map_tasks(void) const; + void map_single_task(SingleTask *task); + public: + static void handle_map_task(const void *args); + protected: + void distribute_tasks(void) const; + RtUserEvent compute_launch_space(const MustEpochLauncher &launcher); + public: + static void handle_distribute_task(const void *args); + static void handle_launch_task(const void *args); protected: std::vector indiv_tasks; std::vector indiv_triggered; @@ -2409,12 +2537,10 @@ namespace Legion { std::set slice_tasks; // The actual base operations // Use a deque to keep everything in order - std::deque single_tasks; + std::vector single_tasks; protected: Mapper::MapMustEpochInput input; Mapper::MapMustEpochOutput output; - MapperID mapper_id; - MappingTagID mapper_tag; protected: FutureMap result_map; unsigned remaining_subop_completes; @@ -2438,130 +2564,6 @@ namespace Legion { std::map slice_version_events; }; - /** - * \class MustEpochTriggerer - * A helper class for parallelizing must epoch triggering - */ - class MustEpochTriggerer { - public: - struct MustEpochIndivArgs : public LgTaskArgs { - public: - static const LgTaskID TASK_ID = LG_MUST_INDIV_ID; - public: - MustEpochIndivArgs(MustEpochTriggerer *trig, MustEpochOp *owner, - IndividualTask *t) - : LgTaskArgs(owner->get_unique_op_id()), - triggerer(trig), task(t) { } - public: - MustEpochTriggerer *const triggerer; - IndividualTask *const task; - }; - struct MustEpochIndexArgs : public LgTaskArgs { - public: - static const LgTaskID TASK_ID = LG_MUST_INDEX_ID; - public: - MustEpochIndexArgs(MustEpochTriggerer *trig, MustEpochOp *owner, - IndexTask *t) - : LgTaskArgs(owner->get_unique_op_id()), - triggerer(trig), task(t) { } - public: - MustEpochTriggerer *const triggerer; - IndexTask *const task; - }; - public: - MustEpochTriggerer(MustEpochOp *owner); - MustEpochTriggerer(const MustEpochTriggerer &rhs); - ~MustEpochTriggerer(void); - public: - MustEpochTriggerer& operator=(const MustEpochTriggerer &rhs); - public: - void trigger_tasks(const std::vector &indiv_tasks, - std::vector &indiv_triggered, - const std::vector &index_tasks, - std::vector &index_triggered); - void trigger_individual(IndividualTask *task); - void trigger_index(IndexTask *task); - public: - static void handle_individual(const void *args); - static void handle_index(const void *args); - private: - const Processor current_proc; - MustEpochOp *const owner; - Reservation trigger_lock; - }; - - /** - * \class MustEpochMapper - * A helper class for parallelizing mapping for must epochs - */ - class MustEpochMapper { - public: - struct MustEpochMapArgs : public LgTaskArgs { - public: - static const LgTaskID TASK_ID = LG_MUST_MAP_ID; - public: - MustEpochMapArgs(MustEpochMapper *map, MustEpochOp *owner) - : LgTaskArgs(owner->get_unique_op_id()), - mapper(map) { } - public: - MustEpochMapper *const mapper; - SingleTask *task; - }; - public: - MustEpochMapper(MustEpochOp *owner); - MustEpochMapper(const MustEpochMapper &rhs); - ~MustEpochMapper(void); - public: - MustEpochMapper& operator=(const MustEpochMapper &rhs); - public: - void map_tasks(const std::deque &single_tasks, - const std::vector > &dependences); - void map_task(SingleTask *task); - public: - static void handle_map_task(const void *args); - private: - MustEpochOp *const owner; - }; - - class MustEpochDistributor { - public: - struct MustEpochDistributorArgs : - public LgTaskArgs { - public: - static const LgTaskID TASK_ID = LG_MUST_DIST_ID; - public: - MustEpochDistributorArgs(MustEpochOp *owner) - : LgTaskArgs(owner->get_unique_op_id()) { } - public: - TaskOp *task; - }; - struct MustEpochLauncherArgs : - public LgTaskArgs { - public: - static const LgTaskID TASK_ID = LG_MUST_LAUNCH_ID; - public: - MustEpochLauncherArgs(MustEpochOp *owner) - : LgTaskArgs(owner->get_unique_op_id()) { } - public: - TaskOp *task; - }; - public: - MustEpochDistributor(MustEpochOp *owner); - MustEpochDistributor(const MustEpochDistributor &rhs); - ~MustEpochDistributor(void); - public: - MustEpochDistributor& operator=(const MustEpochDistributor &rhs); - public: - void distribute_tasks(Runtime *runtime, - const std::vector &indiv_tasks, - const std::set &slice_tasks); - public: - static void handle_distribute_task(const void *args); - static void handle_launch_task(const void *args); - private: - MustEpochOp *const owner; - }; - /** * \class PendingPartitionOp * Pending partition operations are ones that must be deferred @@ -2593,6 +2595,9 @@ namespace Legion { public: virtual ApEvent perform(PendingPartitionOp *op, RegionTreeForest *forest) = 0; + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) = 0; virtual void perform_logging(PendingPartitionOp* op) = 0; }; class EqualPartitionThunk : public PendingPartitionThunk { @@ -2604,6 +2609,11 @@ namespace Legion { virtual ApEvent perform(PendingPartitionOp *op, RegionTreeForest *forest) { return forest->create_equal_partition(op, pid, granularity); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_equal_partition(op, pid, granularity, + shard, total_shards); } virtual void perform_logging(PendingPartitionOp* op); protected: IndexPartition pid; @@ -2619,6 +2629,11 @@ namespace Legion { RegionTreeForest *forest) { return forest->create_partition_by_weights(op, pid, weights, granularity); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_partition_by_weights(op, pid, weights, + granularity, shard, total_shards); } virtual void perform_logging(PendingPartitionOp *op); protected: IndexPartition pid; @@ -2635,6 +2650,11 @@ namespace Legion { virtual ApEvent perform(PendingPartitionOp *op, RegionTreeForest *forest) { return forest->create_partition_by_union(op, pid, handle1, handle2); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_partition_by_union(op, pid, handle1, handle2, + shard, total_shards); } virtual void perform_logging(PendingPartitionOp* op); protected: IndexPartition pid; @@ -2652,6 +2672,11 @@ namespace Legion { RegionTreeForest *forest) { return forest->create_partition_by_intersection(op, pid, handle1, handle2); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_partition_by_intersection(op, pid, handle1, + handle2, shard, total_shards); } virtual void perform_logging(PendingPartitionOp* op); protected: IndexPartition pid; @@ -2668,6 +2693,11 @@ namespace Legion { RegionTreeForest *forest) { return forest->create_partition_by_intersection(op, pid, part, dominates); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_partition_by_intersection(op, pid, part, + dominates, shard, total_shards); } virtual void perform_logging(PendingPartitionOp* op); protected: IndexPartition pid; @@ -2685,6 +2715,11 @@ namespace Legion { RegionTreeForest *forest) { return forest->create_partition_by_difference(op, pid, handle1, handle2); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_partition_by_difference(op, pid, handle1, + handle2, shard, total_shards); } virtual void perform_logging(PendingPartitionOp* op); protected: IndexPartition pid; @@ -2704,6 +2739,11 @@ namespace Legion { RegionTreeForest *forest) { return forest->create_partition_by_restriction(pid, transform, extent); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_partition_by_restriction(pid, transform, + extent, shard, total_shards); } virtual void perform_logging(PendingPartitionOp *op); protected: IndexPartition pid; @@ -2720,6 +2760,11 @@ namespace Legion { RegionTreeForest *forest) { return forest->create_partition_by_domain(op, pid, future_map, perform_intersections); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_partition_by_domain(op, pid, future_map, + perform_intersections, shard, total_shards); } virtual void perform_logging(PendingPartitionOp *op); protected: IndexPartition pid; @@ -2736,6 +2781,11 @@ namespace Legion { RegionTreeForest *forest) { return forest->create_cross_product_partitions(op, base, source, part_color); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->create_cross_product_partitions(op, base, source, + part_color, shard, total_shards); } virtual void perform_logging(PendingPartitionOp* op); protected: IndexPartition base; @@ -2758,6 +2808,15 @@ namespace Legion { else return forest->compute_pending_space(op, target, handles, is_union); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { if (is_partition) + return forest->compute_pending_space(op, target, handle, is_union, + shard, total_shards); + else + return forest->compute_pending_space(op, target, handles, + is_union, shard, total_shards); } virtual void perform_logging(PendingPartitionOp* op); protected: bool is_union, is_partition; @@ -2775,6 +2834,11 @@ namespace Legion { virtual ApEvent perform(PendingPartitionOp *op, RegionTreeForest *forest) { return forest->compute_pending_space(op, target, initial, handles); } + virtual ApEvent perform_shard(PendingPartitionOp *op, + RegionTreeForest *forest, + ShardID shard, size_t total_shards) + { return forest->compute_pending_space(op, target, initial, handles, + shard, total_shards); } virtual void perform_logging(PendingPartitionOp* op); protected: IndexSpace target, initial; @@ -2832,7 +2896,10 @@ namespace Legion { IndexSpace target, IndexSpace initial, const std::vector &handles); - void perform_logging(); + void perform_logging(void); + public: + void activate_pending(void); + void deactivate_pending(void); public: virtual void trigger_dependence_analysis(void); virtual void trigger_ready(void); @@ -2886,6 +2953,8 @@ namespace Legion { const std::vector &instances) = 0; virtual PartitionKind get_kind(void) const = 0; virtual IndexPartition get_partition(void) const = 0; + // This method should only be used by control replication thunks + virtual void elide_collectives(void) { assert(false); } }; class ByFieldThunk : public DepPartThunk { public: @@ -3009,6 +3078,8 @@ namespace Legion { virtual void trigger_dependence_analysis(void); virtual void trigger_ready(void); virtual void trigger_mapping(void); + // A method for override with control replication + virtual void finalize_mapping(void); virtual ApEvent trigger_thunk(IndexSpace handle, const InstanceSet &mapped_instances, const PhysicalTraceInfo &info, @@ -3029,6 +3100,9 @@ namespace Legion { virtual OpKind get_operation_kind(void) const; virtual size_t get_region_count(void) const; virtual void trigger_commit(void); + public: + void activate_dependent(void); + void deactivate_dependent(void); public: virtual void select_sources(const unsigned index, const InstanceRef &target, @@ -3081,6 +3155,7 @@ namespace Legion { std::map acquired_instances; std::set map_applied_conditions; DepPartThunk *thunk; + ApEvent partition_ready; protected: MapperManager *mapper; protected: @@ -3276,6 +3351,9 @@ namespace Legion { public: virtual void activate(void); virtual void deactivate(void); + protected: + void activate_index_fill(void); + void deactivate_index_fill(void); public: virtual void trigger_prepipeline_stage(void); virtual void trigger_dependence_analysis(void); @@ -3290,6 +3368,7 @@ namespace Legion { virtual IndexSpaceNode* get_collective_space(void) const { return launch_space; } public: + void perform_base_dependence_analysis(void); void enumerate_points(bool replaying); void handle_point_commit(void); void check_point_requirements(void); @@ -3370,6 +3449,9 @@ namespace Legion { const AttachLauncher &launcher); inline const RegionRequirement& get_requirement(void) const { return requirement; } + public: + void activate_attach_op(void); + void deactivate_attach_op(void); public: virtual void activate(void); virtual void deactivate(void); @@ -3413,6 +3495,7 @@ namespace Legion { size_t footprint; bool restricted; bool mapping; + bool local_files; }; /** @@ -3431,6 +3514,9 @@ namespace Legion { public: Future initialize_detach(InnerContext *ctx, PhysicalRegion region, const bool flush, const bool unordered); + public: + void activate_detach_op(void); + void deactivate_detach_op(void); public: virtual void activate(void); virtual void deactivate(void); @@ -3485,6 +3571,9 @@ namespace Legion { virtual void deactivate(void); virtual const char* get_logging_name(void) const; virtual OpKind get_operation_kind(void) const; + protected: + void activate_timing(void); + void deactivate_timing(void); public: virtual void trigger_dependence_analysis(void); virtual void trigger_mapping(void); @@ -3514,6 +3603,9 @@ namespace Legion { virtual void deactivate(void); virtual const char* get_logging_name(void) const; virtual OpKind get_operation_kind(void) const; + protected: + void activate_all_reduce(void); + void deactivate_all_reduce(void); public: virtual void trigger_dependence_analysis(void); virtual void trigger_mapping(void); diff --git a/runtime/legion/legion_replication.cc b/runtime/legion/legion_replication.cc new file mode 100644 index 0000000000..dbe2b30993 --- /dev/null +++ b/runtime/legion/legion_replication.cc @@ -0,0 +1,11940 @@ +/* Copyright 2020 Stanford University, NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "legion/legion_ops.h" +#include "legion/legion_trace.h" +#include "legion/legion_views.h" +#include "legion/legion_context.h" +#include "legion/legion_replication.h" + +namespace Legion { + namespace Internal { + + LEGION_EXTERN_LOGGER_DECLARATIONS + +#ifdef DEBUG_LEGION_COLLECTIVES + ///////////////////////////////////////////////////////////// + // Collective Check Reduction + ///////////////////////////////////////////////////////////// + + /*static*/ const long CollectiveCheckReduction::IDENTITY = -1; + /*static*/ const long CollectiveCheckReduction::identity = IDENTITY; + /*static*/ const long CollectiveCheckReduction::BAD = -2; + /*static*/ const ReductionOpID CollectiveCheckReduction::REDOP = + MAX_APPLICATION_REDUCTION_ID; + + //-------------------------------------------------------------------------- + template<> + /*static*/ void CollectiveCheckReduction::apply(LHS &lhs, RHS rhs) + //-------------------------------------------------------------------------- + { + assert(rhs > IDENTITY); + if (lhs != IDENTITY) + { + if (lhs != rhs) + lhs = BAD; + } + else + lhs = rhs; + } + + //-------------------------------------------------------------------------- + template<> + /*static*/ void CollectiveCheckReduction::apply(LHS &lhs, RHS rhs) + //-------------------------------------------------------------------------- + { + volatile LHS *ptr = &lhs; + LHS temp = *ptr; + while ((temp != BAD) && (temp != rhs)) + { + if (temp != IDENTITY) + temp = __sync_val_compare_and_swap(ptr, temp, BAD); + else + temp = __sync_val_compare_and_swap(ptr, temp, rhs); + } + } + + //-------------------------------------------------------------------------- + template<> + /*static*/ void CollectiveCheckReduction::fold(RHS &rhs1, RHS rhs2) + //-------------------------------------------------------------------------- + { + assert(rhs2 > IDENTITY); + if (rhs1 != IDENTITY) + { + if (rhs1 != rhs2) + rhs1 = BAD; + } + else + rhs1 = rhs2; + } + + //-------------------------------------------------------------------------- + template<> + /*static*/ void CollectiveCheckReduction::fold(RHS &rhs1, RHS rhs2) + //-------------------------------------------------------------------------- + { + volatile RHS *ptr = &rhs1; + RHS temp = *ptr; + while ((temp != BAD) && (temp != rhs2)) + { + if (temp != IDENTITY) + temp = __sync_val_compare_and_swap(ptr, temp, BAD); + else + temp = __sync_val_compare_and_swap(ptr, temp, rhs2); + } + } + + ///////////////////////////////////////////////////////////// + // Check Reduction + ///////////////////////////////////////////////////////////// + + /*static*/ const CloseCheckReduction::CloseCheckValue + CloseCheckReduction::IDENTITY = CloseCheckReduction::CloseCheckValue(); + /*static*/ const CloseCheckReduction::CloseCheckValue + CloseCheckReduction::identity = IDENTITY; + /*static*/ const ReductionOpID CloseCheckReduction::REDOP = + MAX_APPLICATION_REDUCTION_ID + 1; + + //-------------------------------------------------------------------------- + CloseCheckReduction::CloseCheckValue::CloseCheckValue(void) + : operation_index(0), region_requirement_index(0), + barrier(RtBarrier::NO_RT_BARRIER), region(LogicalRegion::NO_REGION), + partition(LogicalPartition::NO_PART), is_region(true), read_only(false) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + CloseCheckReduction::CloseCheckValue::CloseCheckValue( + const LogicalUser &user, RtBarrier bar, RegionTreeNode *node, bool read) + : operation_index(user.op->get_ctx_index()), + region_requirement_index(user.idx), barrier(bar), + is_region(node->is_region()), read_only(read) + //-------------------------------------------------------------------------- + { + if (is_region) + region = node->as_region_node()->handle; + else + partition = node->as_partition_node()->handle; + } + + //-------------------------------------------------------------------------- + bool CloseCheckReduction::CloseCheckValue::operator==(const + CloseCheckValue &rhs) const + //-------------------------------------------------------------------------- + { + if (operation_index != rhs.operation_index) + return false; + if (region_requirement_index != rhs.region_requirement_index) + return false; + if (barrier != rhs.barrier) + return false; + if (read_only != rhs.read_only) + return false; + if (is_region != rhs.is_region) + return false; + if (is_region) + { + if (region != rhs.region) + return false; + } + else + { + if (partition != rhs.partition) + return false; + } + return true; + } + + //-------------------------------------------------------------------------- + template<> + /*static*/ void CloseCheckReduction::apply(LHS &lhs, RHS rhs) + //-------------------------------------------------------------------------- + { + // Only copy over if LHS is the identity + // This will effectively do a broadcast of one value + if (lhs == IDENTITY) + lhs = rhs; + } + + //-------------------------------------------------------------------------- + template<> + /*static*/ void CloseCheckReduction::apply(LHS &lhs, RHS rhs) + //-------------------------------------------------------------------------- + { + // Not supported at the moment + assert(false); + } + + //-------------------------------------------------------------------------- + template<> + /*static*/ void CloseCheckReduction::fold(RHS &rhs1, RHS rhs2) + //-------------------------------------------------------------------------- + { + // Only copy over if RHS1 is the identity + // This will effectively do a broadcast of one value + if (rhs1 == IDENTITY) + rhs1 = rhs2; + } + + //-------------------------------------------------------------------------- + template<> + /*static*/ void CloseCheckReduction::fold(RHS &rhs1, RHS rhs2) + //-------------------------------------------------------------------------- + { + // Not supported at the moment + assert(false); + } +#endif // DEBUG_LEGION_COLLECTIVES + + ///////////////////////////////////////////////////////////// + // Repl Individual Task + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplIndividualTask::ReplIndividualTask(Runtime *rt) + : IndividualTask(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplIndividualTask::ReplIndividualTask(const ReplIndividualTask &rhs) + : IndividualTask(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplIndividualTask::~ReplIndividualTask(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplIndividualTask& ReplIndividualTask::operator=( + const ReplIndividualTask &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::activate(void) + //-------------------------------------------------------------------------- + { + activate_individual_task(); + owner_shard = 0; + sharding_functor = UINT_MAX; + sharding_function = NULL; + mapped_collective_id = UINT_MAX; + future_collective_id = UINT_MAX; + mapped_collective = NULL; + future_collective = NULL; +#ifdef DEBUG_LEGION + sharding_collective = NULL; +#endif + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::deactivate(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + delete sharding_collective; +#endif + if (mapped_collective != NULL) + delete mapped_collective; + if (future_collective != NULL) + delete future_collective; + deactivate_individual_task(); + runtime->free_repl_individual_task(this); + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::trigger_prepipeline_stage(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // We might be able to skip this if the sharding function was already + // picked for us which occurs when we're part of a must-epoch launch + if (sharding_function == NULL) + { + // Do the mapper call to get the sharding function to use + if (mapper == NULL) + mapper = runtime->find_mapper(current_proc, map_id); + Mapper::SelectShardingFunctorInput* input = repl_ctx->shard_manager; + Mapper::SelectShardingFunctorOutput output; + output.chosen_functor = UINT_MAX; + mapper->invoke_task_select_sharding_functor(this, input, &output); + if (output.chosen_functor == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s failed to pick a valid sharding functor for " + "task %s (UID %lld)", mapper->get_mapper_name(), + get_task_name(), get_unique_id()) + this->sharding_functor = output.chosen_functor; + sharding_function = + repl_ctx->shard_manager->find_sharding_function(sharding_functor); + } +#ifdef DEBUG_LEGION + assert(sharding_function != NULL); + // In debug mode we check to make sure that all the mappers + // picked the same sharding function + assert(sharding_collective != NULL); + // Contribute the result + sharding_collective->contribute(this->sharding_functor); + if (sharding_collective->is_target() && + !sharding_collective->validate(this->sharding_functor)) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s chose different sharding functions " + "for individual task %s (UID %lld) in %s " + "(UID %lld)", mapper->get_mapper_name(), get_task_name(), + get_unique_id(), parent_ctx->get_task_name(), + parent_ctx->get_unique_id()) +#endif + // Now we can do the normal prepipeline stage + IndividualTask::trigger_prepipeline_stage(); + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::trigger_ready(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(sharding_function != NULL); + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Figure out whether this shard owns this point + if (sharding_space.exists()) + { + Domain shard_domain; + runtime->forest->find_launch_space_domain(sharding_space, shard_domain); + owner_shard = sharding_function->find_owner(index_point, shard_domain); + } + else + owner_shard = sharding_function->find_owner(index_point, index_domain); + // If we're recording then record the owner shard + if (is_recording()) + { +#ifdef DEBUG_LEGION + assert(!is_remote()); + assert((tpl != NULL) && tpl->is_recording()); +#endif + tpl->record_owner_shard(trace_local_id, owner_shard); + } + if (runtime->legion_spy_enabled) + LegionSpy::log_owner_shard(get_unique_id(), owner_shard); +#ifdef DEBUG_LEGION + assert(mapped_collective == NULL); +#endif + mapped_collective = + new ShardEventTree(repl_ctx, owner_shard, mapped_collective_id); + // If we own it we go on the queue, otherwise we complete early + if (owner_shard != repl_ctx->owner_shard->shard_id) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We don't own it, so we can pretend like we + // mapped and executed this copy already + // Before we do this though we have to get the version state + // names for any writes so we can update our local state + RtEvent local_done = mapped_collective->get_local_event(); + complete_mapping(local_done); + complete_execution(); + trigger_children_complete(); + trigger_children_committed(); + } + else // We own it, so it goes on the ready queue + { + // Signal the tree when we are done our mapping + mapped_collective->signal_tree(mapped_event); + // Then we can do the normal analysis + IndividualTask::trigger_ready(); + } + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::replay_analysis(void) + //-------------------------------------------------------------------------- + { + // Figure out if we're the one to do the replay +#ifdef DEBUG_LEGION + assert(!is_remote()); + assert(tpl != NULL); + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); + assert(sharding_collective != NULL); + sharding_collective->elide_collective(); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + owner_shard = tpl->find_owner_shard(trace_local_id); + if (owner_shard != repl_ctx->owner_shard->shard_id) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We don't need to sync mapping here across shards since + // shards can replay in any order with a mapping fence + // at the end + complete_mapping(); + complete_execution(); + trigger_children_complete(); + trigger_children_committed(); + } + else + IndividualTask::replay_analysis(); + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::resolve_false(bool speculated, bool launched) + //-------------------------------------------------------------------------- + { + if (launched) + return; +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + sharding_collective->elide_collective(); +#endif + IndividualTask::resolve_false(speculated, launched); + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::trigger_task_complete(bool deferred /*=false*/) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Before doing the normal thing we have to exchange broadcast/receive + // the future result, can skip this though if we're part of a must epoch + // We should also skip this if we were predicated false + if ((must_epoch == NULL) && + ((speculation_state != RESOLVE_FALSE_STATE) || false_guard.exists())) + { + if (owner_shard == repl_ctx->owner_shard->shard_id) + { +#ifdef DEBUG_LEGION + assert(!deferred); + assert(future_collective == NULL); +#endif + future_collective = new FutureBroadcast(repl_ctx, + future_collective_id, owner_shard, result.impl); + future_collective->broadcast_future(); + } + else + { + if (!deferred) + { +#ifdef DEBUG_LEGION + assert(future_collective == NULL); +#endif + future_collective = new FutureBroadcast(repl_ctx, + future_collective_id, owner_shard, result.impl); + const RtEvent future_ready = + future_collective->perform_collective_wait(false/*block*/); + if (future_ready.exists() && !future_ready.has_triggered()) + { + DeferredTaskCompleteArgs args(this); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_DEFERRED_PRIORITY, future_ready); + return; + } + } + } + } + IndividualTask::trigger_task_complete(deferred); + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::initialize_replication(ReplicateContext *ctx) + //-------------------------------------------------------------------------- + { + mapped_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_0); + future_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_1); + } + + //-------------------------------------------------------------------------- + void ReplIndividualTask::set_sharding_function(ShardingID functor, + ShardingFunction *function) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(must_epoch != NULL); + assert(sharding_function == NULL); +#endif + sharding_functor = functor; + sharding_function = function; + } + + ///////////////////////////////////////////////////////////// + // Repl Index Task + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplIndexTask::ReplIndexTask(Runtime *rt) + : IndexTask(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplIndexTask::ReplIndexTask(const ReplIndexTask &rhs) + : IndexTask(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplIndexTask::~ReplIndexTask(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplIndexTask& ReplIndexTask::operator=(const ReplIndexTask &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::activate(void) + //-------------------------------------------------------------------------- + { + activate_index_task(); + sharding_functor = UINT_MAX; + sharding_function = NULL; + reduction_collective = NULL; +#ifdef DEBUG_LEGION + sharding_collective = NULL; +#endif + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_index_task(); + if (reduction_collective != NULL) + { + delete reduction_collective; + reduction_collective = NULL; + } +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + delete sharding_collective; +#endif + unique_intra_space_deps.clear(); + runtime->free_repl_index_task(this); + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::trigger_prepipeline_stage(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // We might be able to skip this if the sharding function was already + // picked for us which occurs when we're part of a must-epoch launch + if (sharding_function == NULL) + select_sharding_function(repl_ctx); +#ifdef DEBUG_LEGION + assert(sharding_function != NULL); + assert(sharding_collective != NULL); + sharding_collective->contribute(this->sharding_functor); + if (sharding_collective->is_target() && + !sharding_collective->validate(this->sharding_functor)) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s chose different sharding functions " + "for index task %s (UID %lld) in %s (UID %lld)", + mapper->get_mapper_name(), get_task_name(), + get_unique_id(), parent_ctx->get_task_name(), + parent_ctx->get_unique_id()) +#endif + // If we have a future map then set the sharding function + if (redop == 0) + { +#ifdef DEBUG_LEGION + assert(future_map.impl != NULL); + ReplFutureMapImpl *impl = + dynamic_cast(future_map.impl); + assert(impl != NULL); +#else + ReplFutureMapImpl *impl = + static_cast(future_map.impl); +#endif + impl->set_sharding_function(sharding_function); + } + // Now we can do the normal prepipeline stage + IndexTask::trigger_prepipeline_stage(); + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::select_sharding_function(ReplicateContext *repl_ctx) + //-------------------------------------------------------------------------- + { + // Do the mapper call to get the sharding function to use + if (mapper == NULL) + mapper = runtime->find_mapper(current_proc, map_id); + Mapper::SelectShardingFunctorInput* input = repl_ctx->shard_manager; + Mapper::SelectShardingFunctorOutput output; + output.chosen_functor = UINT_MAX; + mapper->invoke_task_select_sharding_functor(this, input, &output); + if (output.chosen_functor == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s failed to pick a valid sharding functor for " + "task %s (UID %lld)", mapper->get_mapper_name(), + get_task_name(), get_unique_id()) + this->sharding_functor = output.chosen_functor; + sharding_function = + repl_ctx->shard_manager->find_sharding_function(sharding_functor); + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::trigger_ready(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Compute the local index space of points for this shard + if (sharding_space.exists()) + internal_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, sharding_space); + else + internal_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, launch_space->handle); + // If we're recording then record the local_space + if (is_recording()) + { +#ifdef DEBUG_LEGION + assert(!is_remote()); + assert((tpl != NULL) && tpl->is_recording()); +#endif + tpl->record_local_space(trace_local_id, internal_space); + // Record the sharding function if needed for the future map + if (redop == 0) + tpl->record_sharding_function(trace_local_id, sharding_function); + } + // If it's empty we're done, otherwise we go back on the queue + if (!internal_space.exists()) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We have no local points, so we can just trigger + complete_mapping(); + complete_execution(); + trigger_children_complete(); + trigger_children_committed(); + } + else // We have valid points, so it goes on the ready queue + { + // Update the total number of points we're actually repsonsible + // for now with this shard + IndexSpaceNode *node = runtime->forest->get_node(internal_space); + total_points = node->get_volume(); +#ifdef DEBUG_LEGION + assert(total_points > 0); +#endif + enqueue_ready_operation(); + } + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::replay_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(tpl != NULL); + assert(sharding_collective != NULL); + sharding_collective->elide_collective(); +#endif + internal_space = tpl->find_local_space(trace_local_id); + if (redop == 0) + { + sharding_function = tpl->find_sharding_function(trace_local_id); +#ifdef DEBUG_LEGION + assert(future_map.impl != NULL); + ReplFutureMapImpl *impl = + dynamic_cast(future_map.impl); + assert(impl != NULL); +#else + ReplFutureMapImpl *impl = + static_cast(future_map.impl); +#endif + impl->set_sharding_function(sharding_function); + } + // If it's empty we're done, otherwise we do the replay + if (!internal_space.exists()) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We have no local points, so we can just trigger + complete_mapping(); + complete_execution(); + trigger_children_complete(); + trigger_children_committed(); + } + else + IndexTask::replay_analysis(); + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { + perform_base_dependence_analysis(); + for (unsigned idx = 0; idx < regions.size(); idx++) + { + ProjectionInfo projection_info(runtime, regions[idx], launch_space, + sharding_function, sharding_space); + runtime->forest->perform_dependence_analysis(this, idx, regions[idx], + projection_info, + privilege_paths[idx], + map_applied_conditions); + } + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::trigger_task_complete(bool deferred /*=false*/) + //-------------------------------------------------------------------------- + { + // If we have a reduction operator, exchange the future results + if (redop > 0) + { +#ifdef DEBUG_LEGION + assert(reduction_collective != NULL); +#endif + // Set the future if we actually ran the task or we speculated + if (!deferred && ((speculation_state != RESOLVE_FALSE_STATE) || + false_guard.exists())) + { + // First time through so start the exchange + if (deterministic_redop) + { + // We have to do the fold of our values here now before + // we can send them all remotely to the other nodes + for (std::map >::const_iterator + it = temporary_futures.begin(); + it != temporary_futures.end(); it++) + { + fold_reduction_future(it->second.first, it->second.second, + false/*owner*/, true/*exclusive*/); + legion_free(FUTURE_RESULT_ALLOC, + it->second.first, it->second.second); + } + // Clear these out so we don't apply them twice when + // we call the base-class version of this method + temporary_futures.clear(); + } + // The collective takes ownership of the buffer here + const RtEvent futures_ready = + reduction_collective->exchange_futures(reduction_state); + // Reinitialize the reduction state buffer so + // that all the shards can be applied to it in the same order + // so that we have bit equivalence across the shards + reduction_state = NULL; + initialize_reduction_state(); + // Now see if we need to defer this or not + if (futures_ready.exists() && !futures_ready.has_triggered()) + { + DeferredTaskCompleteArgs args(this); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_DEFERRED_PRIORITY, futures_ready); + return; + } + } + // Otherwise we fall through and we can just do our exchange + reduction_collective->reduce_futures(this); + } + // Then we do the base class thing + IndexTask::trigger_task_complete(deferred); + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::resolve_false(bool speculated, bool launched) + //-------------------------------------------------------------------------- + { + // If we already launched then we can just return + if (launched) + return; + // Otherwise, we need to update the internal space so we only set + // our local points with the predicate false result + if (redop == 0) + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = + dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + if (sharding_function == NULL) + { + select_sharding_function(repl_ctx); +#ifdef DEBUG_LEGION + assert(future_map.impl != NULL); + ReplFutureMapImpl *impl = + dynamic_cast(future_map.impl); + assert(impl != NULL); +#else + ReplFutureMapImpl *impl = + static_cast(future_map.impl); +#endif + impl->set_sharding_function(sharding_function); + } + // Compute the local index space of points for this shard + if (sharding_space.exists()) + internal_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, sharding_space); + else + internal_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, launch_space->handle); + } +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + sharding_collective->elide_collective(); +#endif + // Now continue through and do the base case + IndexTask::resolve_false(speculated, launched); + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::initialize_replication(ReplicateContext *ctx) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(reduction_collective == NULL); +#endif + // If we have a reduction op then we need an exchange + if (redop > 0) + reduction_collective = + new FutureExchange(ctx, reduction_state_size, COLLECTIVE_LOC_53); + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::set_sharding_function(ShardingID functor, + ShardingFunction *function) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(must_epoch != NULL); + assert(sharding_function == NULL); +#endif + sharding_functor = functor; + sharding_function = function; + } + + //-------------------------------------------------------------------------- + FutureMapImpl* ReplIndexTask::create_future_map(TaskContext *ctx, + IndexSpace launch_space, IndexSpace shard_space) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!future_map_ready.exists() || future_map_ready.has_triggered()); + ReplicateContext *repl_ctx = dynamic_cast(ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(ctx); +#endif + Domain shard_domain; + if (shard_space.exists() && (launch_space != shard_space)) + runtime->forest->find_launch_space_domain(shard_space, shard_domain); + else + shard_domain = index_domain; + future_map_ready = Runtime::create_rt_user_event(); + // Make a replicate future map + return new ReplFutureMapImpl(repl_ctx, this,future_map_ready,index_domain, + shard_domain, runtime, runtime->get_available_distributed_id(), + runtime->address_space); + } + + //-------------------------------------------------------------------------- + RtEvent ReplIndexTask::find_intra_space_dependence(const DomainPoint &point) + //-------------------------------------------------------------------------- + { + + AutoLock o_lock(op_lock); + // Check to see if we already have it + std::map::const_iterator finder = + intra_space_dependences.find(point); + if (finder != intra_space_dependences.end()) + return finder->second; + // Make a temporary event and then do different things depending on + // whether we own this point or whether a remote shard owns it + const RtUserEvent pending_event = Runtime::create_rt_user_event(); + intra_space_dependences[point] = pending_event; + // If not, check to see if this is a point that we expect to own +#ifdef DEBUG_LEGION + assert(sharding_function != NULL); + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + Domain launch_domain; + if (sharding_space.exists()) + runtime->forest->find_launch_space_domain(sharding_space,launch_domain); + else + launch_space->get_launch_space_domain(launch_domain); + const ShardID point_shard = + sharding_function->find_owner(point, launch_domain); + if (point_shard != repl_ctx->owner_shard->shard_id) + { + // A different shard owns it so send a message to that shard + // requesting it to fill in the dependence + Serializer rez; + rez.serialize(repl_ctx->shard_manager->repl_id); + rez.serialize(point_shard); + rez.serialize(context_index); + rez.serialize(point); + rez.serialize(pending_event); + rez.serialize(repl_ctx->owner_shard->shard_id); + repl_ctx->shard_manager->send_intra_space_dependence(point_shard, rez); + } + else // We own it so do the normal thing + pending_intra_space_dependences[point] = pending_event; + return pending_event; + } + + //-------------------------------------------------------------------------- + void ReplIndexTask::record_intra_space_dependence(const DomainPoint &point, + const DomainPoint &next, RtEvent point_mapped) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(sharding_function != NULL); + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Determine if the next point is one that we own or is one that is + // going to be coming from a remote shard + Domain launch_domain; + if (sharding_space.exists()) + runtime->forest->find_launch_space_domain(sharding_space,launch_domain); + else + launch_space->get_launch_space_domain(launch_domain); + const ShardID next_shard = + sharding_function->find_owner(next, launch_domain); + if (next_shard != repl_ctx->owner_shard->shard_id) + { + // Make sure we only send this to the repl_ctx once for each + // unique shard ID that we see for this point task + const std::pair key(point, next_shard); + bool record_dependence = true; + { + AutoLock o_lock(op_lock); + std::set >::const_iterator finder = + unique_intra_space_deps.find(key); + if (finder != unique_intra_space_deps.end()) + record_dependence = false; + else + unique_intra_space_deps.insert(key); + } + if (record_dependence) + repl_ctx->record_intra_space_dependence(context_index, point, + point_mapped, next_shard); + } + else // The next shard is ourself, so we can do the normal thing + IndexTask::record_intra_space_dependence(point, next, point_mapped); + } + + ///////////////////////////////////////////////////////////// + // Repl Merge Close Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplMergeCloseOp::ReplMergeCloseOp(Runtime *rt) + : MergeCloseOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplMergeCloseOp::ReplMergeCloseOp(const ReplMergeCloseOp &rhs) + : MergeCloseOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplMergeCloseOp::~ReplMergeCloseOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplMergeCloseOp& ReplMergeCloseOp::operator=(const ReplMergeCloseOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplMergeCloseOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_close(); + mapped_barrier = RtBarrier::NO_RT_BARRIER; + } + + //-------------------------------------------------------------------------- + void ReplMergeCloseOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_close(); + runtime->free_repl_merge_close_op(this); + } + + //-------------------------------------------------------------------------- + void ReplMergeCloseOp::set_repl_close_info(RtBarrier mapped) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!mapped_barrier.exists()); +#endif + mapped_barrier = mapped; + } + + //-------------------------------------------------------------------------- + void ReplMergeCloseOp::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(mapped_barrier.exists()); + assert(mapping_tracker != NULL); +#endif + // All we have to do is add our map precondition to the tracker + // so we know we are mapping in order with respect to other + // repl close operations that use the same close index + mapping_tracker->add_mapping_dependence( + mapped_barrier.get_previous_phase()); + } + + //-------------------------------------------------------------------------- + void ReplMergeCloseOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(mapped_barrier.exists()); +#endif + // Arrive on our barrier with the precondition + Runtime::phase_barrier_arrive(mapped_barrier, 1/*count*/); + // Then complete the mapping once the barrier has triggered + complete_mapping(mapped_barrier); + complete_execution(); + } + + ///////////////////////////////////////////////////////////// + // Repl Fill Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplFillOp::ReplFillOp(Runtime *rt) + : FillOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplFillOp::ReplFillOp(const ReplFillOp &rhs) + : FillOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplFillOp::~ReplFillOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplFillOp& ReplFillOp::operator=(const ReplFillOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplFillOp::initialize_replication(ReplicateContext *ctx) + //-------------------------------------------------------------------------- + { + mapped_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_2); + } + + //-------------------------------------------------------------------------- + void ReplFillOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_fill(); + sharding_functor = UINT_MAX; + sharding_function = NULL; + mapper = NULL; +#ifdef DEBUG_LEGION + sharding_collective = NULL; +#endif + mapped_collective_id = UINT_MAX; + mapped_collective = NULL; + } + + //-------------------------------------------------------------------------- + void ReplFillOp::deactivate(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + delete sharding_collective; +#endif + if (mapped_collective != NULL) + delete mapped_collective; + deactivate_fill(); + runtime->free_repl_fill_op(this); + } + + //-------------------------------------------------------------------------- + void ReplFillOp::trigger_prepipeline_stage(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Do the mapper call to get the sharding function to use + if (mapper == NULL) + mapper = runtime->find_mapper( + parent_ctx->get_executing_processor(), map_id); + Mapper::SelectShardingFunctorInput* input = repl_ctx->shard_manager; + Mapper::SelectShardingFunctorOutput output; + output.chosen_functor = UINT_MAX; + mapper->invoke_fill_select_sharding_functor(this, input, &output); + if (output.chosen_functor == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s failed to pick a valid sharding functor for " + "fill in task %s (UID %lld)", mapper->get_mapper_name(), + parent_ctx->get_task_name(), parent_ctx->get_unique_id()) + this->sharding_functor = output.chosen_functor; + sharding_function = + repl_ctx->shard_manager->find_sharding_function(sharding_functor); +#ifdef DEBUG_LEGION + assert(sharding_collective != NULL); + sharding_collective->contribute(this->sharding_functor); + if (sharding_collective->is_target() && + !sharding_collective->validate(this->sharding_functor)) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s chose different sharding functions " + "for fill in task %s (UID %lld)", + mapper->get_mapper_name(), parent_ctx->get_task_name(), + parent_ctx->get_unique_id()) +#endif + // Now we can do the normal prepipeline stage + FillOp::trigger_prepipeline_stage(); + } + + //-------------------------------------------------------------------------- + void ReplFillOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Figure out whether this shard owns this point + ShardID owner_shard; + if (sharding_space.exists()) + { + Domain shard_domain; + runtime->forest->find_launch_space_domain(sharding_space, shard_domain); + owner_shard = sharding_function->find_owner(index_point, shard_domain); + } + else + owner_shard = sharding_function->find_owner(index_point, index_domain); + // If we're recording then record the owner shard + if (is_recording()) + { +#ifdef DEBUG_LEGION + assert((tpl != NULL) && tpl->is_recording()); +#endif + tpl->record_owner_shard(trace_local_id, owner_shard); + } + if (runtime->legion_spy_enabled) + LegionSpy::log_owner_shard(get_unique_id(), owner_shard); +#ifdef DEBUG_LEGION + assert(mapped_collective == NULL); +#endif + mapped_collective = + new ShardEventTree(repl_ctx, owner_shard, mapped_collective_id); + // If we own it we go on the queue, otherwise we complete early + if (owner_shard != repl_ctx->owner_shard->shard_id) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We don't own it, so we can pretend like we + // mapped and executed this fill already + // Before we do this though we have to get the version state + // names for any writes so we can update our local state + RtEvent local_done = mapped_collective->get_local_event(); + complete_mapping(local_done); + complete_execution(); + } + else // We own it, so do the base call + { + // Signal the tree when we are done our mapping + mapped_collective->signal_tree(mapped_event); + // Perform the base operation + FillOp::trigger_ready(); + } + } + + //-------------------------------------------------------------------------- + void ReplFillOp::replay_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(tpl != NULL); + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); + assert(sharding_collective != NULL); + sharding_collective->elide_collective(); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + const ShardID owner_shard = tpl->find_owner_shard(trace_local_id); + if (owner_shard != repl_ctx->owner_shard->shard_id) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + complete_mapping(); + complete_execution(); + } + else // We own it, so do the base call + FillOp::replay_analysis(); + } + + //-------------------------------------------------------------------------- + void ReplFillOp::resolve_false(bool speculated, bool launched) + //-------------------------------------------------------------------------- + { + if (launched) + return; +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + sharding_collective->elide_collective(); +#endif + FillOp::resolve_false(speculated, launched); + } + + ///////////////////////////////////////////////////////////// + // Repl Index Fill Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplIndexFillOp::ReplIndexFillOp(Runtime *rt) + : IndexFillOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplIndexFillOp::ReplIndexFillOp(const ReplIndexFillOp &rhs) + : IndexFillOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplIndexFillOp::~ReplIndexFillOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplIndexFillOp& ReplIndexFillOp::operator=(const ReplIndexFillOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplIndexFillOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_index_fill(); + sharding_functor = UINT_MAX; + sharding_function = NULL; + mapper = NULL; +#ifdef DEBUG_LEGION + sharding_collective = NULL; +#endif + } + + //-------------------------------------------------------------------------- + void ReplIndexFillOp::deactivate(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + delete sharding_collective; +#endif + deactivate_index_fill(); + runtime->free_repl_index_fill_op(this); + } + + //-------------------------------------------------------------------------- + void ReplIndexFillOp::trigger_prepipeline_stage(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Do the mapper call to get the sharding function to use + if (mapper == NULL) + mapper = runtime->find_mapper( + parent_ctx->get_executing_processor(), map_id); + Mapper::SelectShardingFunctorInput* input = repl_ctx->shard_manager; + Mapper::SelectShardingFunctorOutput output; + output.chosen_functor = UINT_MAX; + mapper->invoke_fill_select_sharding_functor(this, input, &output); + if (output.chosen_functor == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s failed to pick a valid sharding functor for " + "index fill in task %s (UID %lld)", + mapper->get_mapper_name(), + parent_ctx->get_task_name(), parent_ctx->get_unique_id()) + this->sharding_functor = output.chosen_functor; + sharding_function = + repl_ctx->shard_manager->find_sharding_function(sharding_functor); +#ifdef DEBUG_LEGION + assert(sharding_collective != NULL); + sharding_collective->contribute(this->sharding_functor); + if (sharding_collective->is_target() && + !sharding_collective->validate(this->sharding_functor)) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s chose different sharding functions " + "for index fill in task %s (UID %lld)", + mapper->get_mapper_name(), parent_ctx->get_task_name(), + parent_ctx->get_unique_id()) +#endif + // Now we can do the normal prepipeline stage + IndexFillOp::trigger_prepipeline_stage(); + } + + //-------------------------------------------------------------------------- + void ReplIndexFillOp::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { + perform_base_dependence_analysis(); + ProjectionInfo projection_info(runtime, requirement, launch_space, + sharding_function, sharding_space); + runtime->forest->perform_dependence_analysis(this, 0/*idx*/, + requirement, + projection_info, + privilege_path, + map_applied_conditions); + } + + //-------------------------------------------------------------------------- + void ReplIndexFillOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); + assert(launch_space != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Compute the local index space of points for this shard + IndexSpace local_space; + if (sharding_space.exists()) + local_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, sharding_space); + else + local_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, launch_space->handle); + // If we're recording then record the local_space + if (is_recording()) + { +#ifdef DEBUG_LEGION + assert((tpl != NULL) && tpl->is_recording()); +#endif + tpl->record_local_space(trace_local_id, local_space); + } + // If it's empty we're done, otherwise we go back on the queue + if (!local_space.exists()) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We have no local points, so we can just trigger + complete_mapping(); + complete_execution(); + } + else // We have valid points, so it goes on the ready queue + { + if (remove_launch_space_reference(launch_space)) + delete launch_space; + launch_space = runtime->forest->get_node(local_space); + add_launch_space_reference(launch_space); + IndexFillOp::trigger_ready(); + } + } + + //-------------------------------------------------------------------------- + void ReplIndexFillOp::replay_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(tpl != NULL); + assert(sharding_collective != NULL); + sharding_collective->elide_collective(); +#endif + const IndexSpace local_space = tpl->find_local_space(trace_local_id); + // If it's empty we're done, otherwise we do the replay + if (!local_space.exists()) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We have no local points, so we can just trigger + complete_mapping(); + complete_execution(); + } + else + { + if (remove_launch_space_reference(launch_space)) + delete launch_space; + launch_space = runtime->forest->get_node(local_space); + add_launch_space_reference(launch_space); + IndexFillOp::replay_analysis(); + } + } + + //-------------------------------------------------------------------------- + void ReplIndexFillOp::resolve_false(bool speculated, bool launched) + //-------------------------------------------------------------------------- + { + if (launched) + return; +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + sharding_collective->elide_collective(); +#endif + IndexFillOp::resolve_false(speculated, launched); + } + + //-------------------------------------------------------------------------- + void ReplIndexFillOp::initialize_replication(ReplicateContext *ctx) + //-------------------------------------------------------------------------- + { + } + + ///////////////////////////////////////////////////////////// + // Repl Copy Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplCopyOp::ReplCopyOp(Runtime *rt) + : CopyOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplCopyOp::ReplCopyOp(const ReplCopyOp &rhs) + : CopyOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplCopyOp::~ReplCopyOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplCopyOp& ReplCopyOp::operator=(const ReplCopyOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplCopyOp::initialize_replication(ReplicateContext *ctx) + //-------------------------------------------------------------------------- + { + mapped_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_2); + // Initialize our index domain of a single point + index_domain = Domain(index_point, index_point); + } + + //-------------------------------------------------------------------------- + void ReplCopyOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_copy(); + sharding_functor = UINT_MAX; + sharding_function = NULL; +#ifdef DEBUG_LEGION + sharding_collective = NULL; +#endif + mapped_collective_id = UINT_MAX; + mapped_collective = NULL; + } + + //-------------------------------------------------------------------------- + void ReplCopyOp::deactivate(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + delete sharding_collective; +#endif + if (mapped_collective != NULL) + delete mapped_collective; + deactivate_copy(); + runtime->free_repl_copy_op(this); + } + + //-------------------------------------------------------------------------- + void ReplCopyOp::trigger_prepipeline_stage(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Do the mapper call to get the sharding function to use + if (mapper == NULL) + mapper = runtime->find_mapper( + parent_ctx->get_executing_processor(), map_id); + Mapper::SelectShardingFunctorInput* input = repl_ctx->shard_manager; + Mapper::SelectShardingFunctorOutput output; + output.chosen_functor = UINT_MAX; + mapper->invoke_copy_select_sharding_functor(this, input, &output); + if (output.chosen_functor == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s failed to pick a valid sharding functor for " + "copy in task %s (UID %lld)", mapper->get_mapper_name(), + parent_ctx->get_task_name(), parent_ctx->get_unique_id()) + this->sharding_functor = output.chosen_functor; + sharding_function = + repl_ctx->shard_manager->find_sharding_function(sharding_functor); +#ifdef DEBUG_LEGION + assert(sharding_collective != NULL); + sharding_collective->contribute(this->sharding_functor); + if (sharding_collective->is_target() && + !sharding_collective->validate(this->sharding_functor)) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s chose different sharding functions " + "for copy in task %s (UID %lld)", + mapper->get_mapper_name(), parent_ctx->get_task_name(), + parent_ctx->get_unique_id()) +#endif + // Now we can do the normal prepipeline stage + CopyOp::trigger_prepipeline_stage(); + } + + //-------------------------------------------------------------------------- + void ReplCopyOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Figure out whether this shard owns this point + ShardID owner_shard; + if (sharding_space.exists()) + { + Domain shard_domain; + runtime->forest->find_launch_space_domain(sharding_space, shard_domain); + owner_shard = sharding_function->find_owner(index_point, shard_domain); + } + else + owner_shard = sharding_function->find_owner(index_point, index_domain); + // If we're recording then record the owner shard + if (is_recording()) + { +#ifdef DEBUG_LEGION + assert((tpl != NULL) && tpl->is_recording()); +#endif + tpl->record_owner_shard(trace_local_id, owner_shard); + } + if (runtime->legion_spy_enabled) + LegionSpy::log_owner_shard(get_unique_id(), owner_shard); +#ifdef DEBUG_LEGION + assert(mapped_collective == NULL); +#endif + mapped_collective = + new ShardEventTree(repl_ctx, owner_shard, mapped_collective_id); + // If we own it we go on the queue, otherwise we complete early + if (owner_shard != repl_ctx->owner_shard->shard_id) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We don't own it, so we can pretend like we + // mapped and executed this copy already + // Before we do this though we have to get the version state + // names for any writes so we can update our local state + RtEvent local_done = mapped_collective->get_local_event(); + complete_mapping(local_done); + complete_execution(); + } + else // We own it, so do the base call + { + // Signal the tree when we are done our mapping + mapped_collective->signal_tree(mapped_event); + // Perform the base operation + CopyOp::trigger_ready(); + } + } + + //-------------------------------------------------------------------------- + void ReplCopyOp::replay_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(tpl != NULL); + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); + assert(sharding_collective != NULL); + sharding_collective->elide_collective(); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + const ShardID owner_shard = tpl->find_owner_shard(trace_local_id); + if (owner_shard != repl_ctx->owner_shard->shard_id) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + complete_mapping(); + complete_execution(); + } + else // We own it, so do the base call + CopyOp::replay_analysis(); + } + + //-------------------------------------------------------------------------- + void ReplCopyOp::resolve_false(bool speculated, bool launched) + //-------------------------------------------------------------------------- + { + if (launched) + return; +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + sharding_collective->elide_collective(); +#endif + CopyOp::resolve_false(speculated, launched); + } + + ///////////////////////////////////////////////////////////// + // Repl Index Copy Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplIndexCopyOp::ReplIndexCopyOp(Runtime *rt) + : IndexCopyOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplIndexCopyOp::ReplIndexCopyOp(const ReplIndexCopyOp &rhs) + : IndexCopyOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplIndexCopyOp::~ReplIndexCopyOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplIndexCopyOp& ReplIndexCopyOp::operator=(const ReplIndexCopyOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplIndexCopyOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_index_copy(); + sharding_functor = UINT_MAX; + sharding_function = NULL; +#ifdef DEBUG_LEGION + sharding_collective = NULL; +#endif + } + + //-------------------------------------------------------------------------- + void ReplIndexCopyOp::deactivate(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + delete sharding_collective; +#endif + indirection_barriers.clear(); + if (!src_collectives.empty()) + { + for (unsigned idx = 0; idx < src_collectives.size(); idx++) + delete src_collectives[idx]; + src_collectives.clear(); + } + if (!dst_collectives.empty()) + { + for (unsigned idx = 0; idx < dst_collectives.size(); idx++) + delete dst_collectives[idx]; + dst_collectives.clear(); + } + deactivate_index_copy(); + runtime->free_repl_index_copy_op(this); + } + + //-------------------------------------------------------------------------- + void ReplIndexCopyOp::trigger_prepipeline_stage(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Do the mapper call to get the sharding function to use + if (mapper == NULL) + mapper = runtime->find_mapper( + parent_ctx->get_executing_processor(), map_id); + Mapper::SelectShardingFunctorInput* input = repl_ctx->shard_manager; + Mapper::SelectShardingFunctorOutput output; + output.chosen_functor = UINT_MAX; + mapper->invoke_copy_select_sharding_functor(this, input, &output); + if (output.chosen_functor == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s failed to pick a valid sharding functor for " + "index copy in task %s (UID %lld)", + mapper->get_mapper_name(), + parent_ctx->get_task_name(), parent_ctx->get_unique_id()) + this->sharding_functor = output.chosen_functor; + sharding_function = + repl_ctx->shard_manager->find_sharding_function(sharding_functor); +#ifdef DEBUG_LEGION + assert(sharding_collective != NULL); + sharding_collective->contribute(this->sharding_functor); + if (sharding_collective->is_target() && + !sharding_collective->validate(this->sharding_functor)) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s chose different sharding functions " + "for index copy in task %s (UID %lld)", + mapper->get_mapper_name(), parent_ctx->get_task_name(), + parent_ctx->get_unique_id()) +#endif + // Now we can do the normal prepipeline stage + IndexCopyOp::trigger_prepipeline_stage(); + } + + //-------------------------------------------------------------------------- + void ReplIndexCopyOp::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { + perform_base_dependence_analysis(); + for (unsigned idx = 0; idx < src_requirements.size(); idx++) + { + ProjectionInfo projection_info (runtime, src_requirements[idx], + launch_space, sharding_function, sharding_space); + runtime->forest->perform_dependence_analysis(this, idx, + src_requirements[idx], + projection_info, + src_privilege_paths[idx], + map_applied_conditions); + } + for (unsigned idx = 0; idx < dst_requirements.size(); idx++) + { + ProjectionInfo projection_info(runtime, dst_requirements[idx], + launch_space, sharding_function, sharding_space); + unsigned index = src_requirements.size()+idx; + // Perform this dependence analysis as if it was READ_WRITE + // so that we can get the version numbers correct + const bool is_reduce_req = IS_REDUCE(dst_requirements[idx]); + if (is_reduce_req) + dst_requirements[idx].privilege = LEGION_READ_WRITE; + runtime->forest->perform_dependence_analysis(this, index, + dst_requirements[idx], + projection_info, + dst_privilege_paths[idx], + map_applied_conditions); + // Switch the privileges back when we are done + if (is_reduce_req) + dst_requirements[idx].privilege = LEGION_REDUCE; + } + } + + //-------------------------------------------------------------------------- + void ReplIndexCopyOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Compute the local index space of points for this shard + IndexSpace local_space; + if (sharding_space.exists()) + local_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, sharding_space); + else + local_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, launch_space->handle); + // If we're recording then record the local_space + if (is_recording()) + { +#ifdef DEBUG_LEGION + assert((tpl != NULL) && tpl->is_recording()); +#endif + tpl->record_local_space(trace_local_id, local_space); + } + // If it's empty we're done, otherwise we go back on the queue + if (!local_space.exists()) + { + // If we have indirections then we still need to participate in those + if (!src_indirect_requirements.empty() && + collective_src_indirect_points) + { + LegionVector::aligned empty_records; + for (unsigned idx = 0; idx < src_indirect_requirements.size(); idx++) + { + src_collectives[idx]->exchange_records(empty_records); + empty_records.clear(); + } + } + if (!dst_indirect_requirements.empty() && + collective_dst_indirect_points) + { + LegionVector::aligned empty_records; + for (unsigned idx = 0; idx < dst_indirect_requirements.size(); idx++) + { + dst_collectives[idx]->exchange_records(empty_records); + empty_records.clear(); + } + } + // Arrive on our indirection barriers if we have them + if (!indirection_barriers.empty()) + { + for (unsigned idx = 0; idx < indirection_barriers.size(); idx++) + Runtime::phase_barrier_arrive(indirection_barriers[idx],1/*count*/); + } +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We have no local points, so we can just trigger + complete_mapping(); + complete_execution(); + } + else // If we have any valid points do the base call + { + if (remove_launch_space_reference(launch_space)) + delete launch_space; + launch_space = runtime->forest->get_node(local_space); + add_launch_space_reference(launch_space); + IndexCopyOp::trigger_ready(); + } + } + + //-------------------------------------------------------------------------- + void ReplIndexCopyOp::replay_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(tpl != NULL); + assert(sharding_collective != NULL); + sharding_collective->elide_collective(); +#endif + const IndexSpace local_space = tpl->find_local_space(trace_local_id); + // If it's empty we're done, otherwise we do the replay + if (!local_space.exists()) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We have no local points, so we can just trigger + complete_mapping(); + complete_execution(); + } + else + { + if (remove_launch_space_reference(launch_space)) + delete launch_space; + launch_space = runtime->forest->get_node(local_space); + add_launch_space_reference(launch_space); + IndexCopyOp::replay_analysis(); + } + } + + //-------------------------------------------------------------------------- + void ReplIndexCopyOp::resolve_false(bool speculated, bool launched) + //-------------------------------------------------------------------------- + { + if (launched) + return; +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + sharding_collective->elide_collective(); +#endif + IndexCopyOp::resolve_false(speculated, launched); + } + + //-------------------------------------------------------------------------- + ApEvent ReplIndexCopyOp::exchange_indirect_records(const unsigned index, + const ApEvent local_done, const PhysicalTraceInfo &trace_info, + const InstanceSet &instances, const IndexSpace space, + const DomainPoint &key, + LegionVector::aligned &records, const bool sources) + //-------------------------------------------------------------------------- + { + if (sources && !collective_src_indirect_points) + return CopyOp::exchange_indirect_records(index, local_done, trace_info, + instances, space, key, records, sources); + if (!sources && !collective_dst_indirect_points) + return CopyOp::exchange_indirect_records(index, local_done, trace_info, + instances, space, key, records, sources); +#ifdef DEBUG_LEGION + assert(local_done.exists()); + assert(index < indirection_barriers.size()); + assert(indirection_barriers[index].exists()); +#endif + RtEvent wait_on; + RtUserEvent to_trigger; + std::set arrival_events; + { + IndexSpaceNode *node = runtime->forest->get_node(space); + ApEvent domain_ready; + const Domain dom = node->get_domain(domain_ready, true/*tight*/); + // Take the lock and record our sets and instances + AutoLock o_lock(op_lock); + if (sources) + { + if (domain_ready.exists() && !domain_ready.has_triggered()) + { + for (unsigned idx = 0; idx < instances.size(); idx++) + { + const InstanceRef &ref = instances[idx]; + const ApEvent inst_ready = ref.get_ready_event(); + if (inst_ready.exists() && !inst_ready.has_triggered()) + src_records[index].push_back(IndirectRecord( + ref.get_valid_fields(), ref.get_manager(), key, space, + Runtime::merge_events(&trace_info, domain_ready, + inst_ready), dom)); + else + src_records[index].push_back(IndirectRecord( + ref.get_valid_fields(), ref.get_manager(), key, + space, domain_ready, dom)); + } + } + else + { + for (unsigned idx = 0; idx < instances.size(); idx++) + { + const InstanceRef &ref = instances[idx]; + src_records[index].push_back(IndirectRecord( + ref.get_valid_fields(), ref.get_manager(), key, + space, ref.get_ready_event(), dom)); + } + } + src_exchange_events[index].insert(local_done); + if (!src_exchanged[index].exists()) + src_exchanged[index] = Runtime::create_rt_user_event(); + if (src_exchange_events[index].size() == points.size()) + { + to_trigger = src_exchanged[index]; + arrival_events.insert(src_exchange_events[index].begin(), + src_exchange_events[index].end()); + } + else + wait_on = src_exchanged[index]; + } + else + { + if (domain_ready.exists() && !domain_ready.has_triggered()) + { + for (unsigned idx = 0; idx < instances.size(); idx++) + { + const InstanceRef &ref = instances[idx]; + const ApEvent inst_ready = ref.get_ready_event(); + if (inst_ready.exists() && !inst_ready.has_triggered()) + dst_records[index].push_back(IndirectRecord( + ref.get_valid_fields(), ref.get_manager(), key, space, + Runtime::merge_events(&trace_info, domain_ready, + inst_ready), dom)); + else + dst_records[index].push_back(IndirectRecord( + ref.get_valid_fields(), ref.get_manager(), key, + space, domain_ready, dom)); + } + } + else + { + for (unsigned idx = 0; idx < instances.size(); idx++) + { + const InstanceRef &ref = instances[idx]; + dst_records[index].push_back(IndirectRecord( + ref.get_valid_fields(), ref.get_manager(), key, + space, ref.get_ready_event(), dom)); + } + } + dst_exchange_events[index].insert(local_done); + if (!dst_exchanged[index].exists()) + dst_exchanged[index] = Runtime::create_rt_user_event(); + if (dst_exchange_events[index].size() == points.size()) + { + to_trigger = dst_exchanged[index]; + arrival_events.insert(dst_exchange_events[index].begin(), + dst_exchange_events[index].end()); + } + else + wait_on = dst_exchanged[index]; + } + } + if (to_trigger.exists()) + { + // Perform the collective + if (sources) + src_collectives[index]->exchange_records(src_records[index]); + else + dst_collectives[index]->exchange_records(dst_records[index]); + Runtime::trigger_event(to_trigger); + if (!arrival_events.empty()) + Runtime::phase_barrier_arrive(indirection_barriers[index], + 1/*count*/, Runtime::merge_events(&trace_info, arrival_events)); + } + else if (!wait_on.has_triggered()) + wait_on.wait(); + // Once we wake up we can copy out the results + if (sources) + records = src_records[index]; + else + records = dst_records[index]; + return indirection_barriers[index]; + } + + //-------------------------------------------------------------------------- + void ReplIndexCopyOp::initialize_replication(ReplicateContext *ctx, + std::vector &indirection_bars, + unsigned &next_indirection_index) + //-------------------------------------------------------------------------- + { + if (!src_indirect_requirements.empty() && collective_src_indirect_points) + { + src_collectives.resize(src_indirect_requirements.size()); + for (unsigned idx = 0; idx < src_indirect_requirements.size(); idx++) + src_collectives[idx] = + new IndirectRecordExchange(ctx, COLLECTIVE_LOC_80); + } + if (!dst_indirect_requirements.empty() && collective_dst_indirect_points) + { + dst_collectives.resize(dst_indirect_requirements.size()); + for (unsigned idx = 0; idx < dst_indirect_requirements.size(); idx++) + dst_collectives[idx] = + new IndirectRecordExchange(ctx, COLLECTIVE_LOC_81); + } + if (!src_indirect_requirements.empty() || + !dst_indirect_requirements.empty()) + { +#ifdef DEBUG_LEGION + assert(src_indirect_requirements.empty() || + dst_indirect_requirements.empty() || + (src_indirect_requirements.size() == + dst_indirect_requirements.size())); +#endif + indirection_barriers.resize( + (src_indirect_requirements.size() > + dst_indirect_requirements.size()) ? + src_indirect_requirements.size() : + dst_indirect_requirements.size()); + for (unsigned idx = 0; idx < indirection_barriers.size(); idx++) + { + ApBarrier &next_bar = indirection_bars[next_indirection_index++]; + indirection_barriers[idx] = next_bar; + ctx->advance_replicate_barrier(next_bar, ctx->total_shards); + if (next_indirection_index == indirection_bars.size()) + next_indirection_index = 0; + } + } + } + + ///////////////////////////////////////////////////////////// + // Repl Deletion Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplDeletionOp::ReplDeletionOp(Runtime *rt) + : DeletionOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplDeletionOp::ReplDeletionOp(const ReplDeletionOp &rhs) + : DeletionOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplDeletionOp::~ReplDeletionOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplDeletionOp& ReplDeletionOp::operator=(const ReplDeletionOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplDeletionOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_deletion(); + ready_barrier = RtBarrier::NO_RT_BARRIER; + mapping_barrier = RtBarrier::NO_RT_BARRIER; + execution_barrier = RtBarrier::NO_RT_BARRIER; + is_total_sharding = false; + is_first_local_shard = false; + } + + //-------------------------------------------------------------------------- + void ReplDeletionOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_deletion(); + runtime->free_repl_deletion_op(this); + } + + //-------------------------------------------------------------------------- + void ReplDeletionOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { + if ((kind == FIELD_DELETION) || (kind == LOGICAL_REGION_DELETION)) + Runtime::phase_barrier_arrive(ready_barrier, 1/*count*/); + if (kind == FIELD_DELETION) + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = + dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Field deletions need to compute their version infos + if ((is_total_sharding && is_first_local_shard) || + (repl_ctx->owner_shard->shard_id == 0)) + { + std::set preconditions; + version_infos.resize(deletion_requirements.size()); + for (unsigned idx = 0; idx < deletion_requirements.size(); idx++) + runtime->forest->perform_versioning_analysis(this, idx, + deletion_requirements[idx], + version_infos[idx], + preconditions); + if (!preconditions.empty()) + { + preconditions.insert(ready_barrier); + enqueue_ready_operation(Runtime::merge_events(preconditions)); + return; + } + } + } + enqueue_ready_operation(ready_barrier); + } + + //-------------------------------------------------------------------------- + void ReplDeletionOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(execution_barrier.exists()); + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // There are two different implementations here depending on whether we + // know that we have a deletion operation on every shard or not + // If not, we just let the deletion for shard 0 do all the work, + // otherwise we know we can evenly distribute the work + if (kind == LOGICAL_REGION_DELETION) + { + // Just need to clean out the version managers which will free + // all the equivalence sets and allow the reference counting to + // clean everything up + if (is_first_local_shard) + { + bool has_outermost = false; + RegionTreeContext outermost_ctx; + const RegionTreeContext tree_context = parent_ctx->get_context(); + for (unsigned idx = 0; idx < deletion_requirements.size(); idx++) + { + const RegionRequirement &req = deletion_requirements[idx]; + if (returnable_privileges[idx]) + { + if (!has_outermost) + { + TaskContext *outermost = + parent_ctx->find_outermost_local_context(); + outermost_ctx = outermost->get_context(); + has_outermost = true; + } + runtime->forest->invalidate_versions(outermost_ctx, req.region); + } + else + runtime->forest->invalidate_versions(tree_context, req.region); + } + } + complete_mapping(); + } + else if (kind == FIELD_DELETION) + { +#ifdef DEBUG_LEGION + assert(mapping_barrier.exists()); +#endif + if ((is_total_sharding && is_first_local_shard) || + (repl_ctx->owner_shard->shard_id == 0)) + { + // For this case we actually need to go through and prune out any + // valid instances for these fields in the equivalence sets in order + // to be able to free up the resources. + const TraceInfo trace_info(this); + for (unsigned idx = 0; idx < deletion_requirements.size(); idx++) + runtime->forest->invalidate_fields(this, idx, version_infos[idx], + PhysicalTraceInfo(trace_info, idx), map_applied_conditions, + is_total_sharding/*collective*/); + } + // make sure that we don't try to do the deletion calls until + // after the allocator is ready + if (allocator->ready_event.exists()) + map_applied_conditions.insert(allocator->ready_event); + if (!map_applied_conditions.empty()) + Runtime::phase_barrier_arrive(mapping_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + else + Runtime::phase_barrier_arrive(mapping_barrier, 1/*count*/); + complete_mapping(mapping_barrier); + } + else + complete_mapping(); + // complete execution once all the shards are done + if (execution_precondition.exists()) + Runtime::phase_barrier_arrive(execution_barrier, 1/*count*/, + Runtime::protect_event(execution_precondition)); + else + Runtime::phase_barrier_arrive(execution_barrier, 1/*count*/); + complete_execution(execution_barrier); + } + + //-------------------------------------------------------------------------- + void ReplDeletionOp::trigger_complete(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + std::set applied; + if (is_total_sharding && is_first_local_shard) + { + switch (kind) + { + case INDEX_SPACE_DELETION: + { +#ifdef DEBUG_LEGION + assert(deletion_req_indexes.empty()); +#endif + runtime->forest->destroy_index_space(index_space, + applied, true/*collective*/); + if (!sub_partitions.empty()) + { + for (std::vector::const_iterator it = + sub_partitions.begin(); it != sub_partitions.end(); it++) + runtime->forest->destroy_index_partition(*it, applied, + true/*collective*/); + } + break; + } + case INDEX_PARTITION_DELETION: + { +#ifdef DEBUG_LEGION + assert(deletion_req_indexes.empty()); +#endif + runtime->forest->destroy_index_partition(index_part, applied, + true/*collective*/); + if (!sub_partitions.empty()) + { + for (std::vector::const_iterator it = + sub_partitions.begin(); it != sub_partitions.end(); it++) + runtime->forest->destroy_index_partition(*it, applied, + true/*collective*/); + } + break; + } + case FIELD_SPACE_DELETION: + { +#ifdef DEBUG_LEGION + assert(deletion_req_indexes.empty()); +#endif + runtime->forest->destroy_field_space(field_space, applied, + true/*collective*/); + break; + } + case FIELD_DELETION: + // Everyone is going to do the same thing for field deletions + break; + case LOGICAL_REGION_DELETION: + { + // Only do something here if we don't have any parent req indexes + // If we had no deletion requirements then we know there is + // nothing to race with and we can just do our deletion + if (parent_req_indexes.empty()) + runtime->forest->destroy_logical_region(logical_region, applied, + true/*collective*/); + break; + } + default: + assert(false); + } + } + else if (repl_ctx->owner_shard->shard_id == 0) + { + // Shard 0 will handle the actual deletions + switch (kind) + { + case INDEX_SPACE_DELETION: + { +#ifdef DEBUG_LEGION + assert(deletion_req_indexes.empty()); +#endif + runtime->forest->destroy_index_space(index_space, applied); + if (!sub_partitions.empty()) + { + for (std::vector::const_iterator it = + sub_partitions.begin(); it != sub_partitions.end(); it++) + runtime->forest->destroy_index_partition(*it, applied); + } + break; + } + case INDEX_PARTITION_DELETION: + { +#ifdef DEBUG_LEGION + assert(deletion_req_indexes.empty()); +#endif + runtime->forest->destroy_index_partition(index_part, applied); + if (!sub_partitions.empty()) + { + for (std::vector::const_iterator it = + sub_partitions.begin(); it != sub_partitions.end(); it++) + runtime->forest->destroy_index_partition(*it, applied); + } + break; + } + case FIELD_SPACE_DELETION: + { +#ifdef DEBUG_LEGION + assert(deletion_req_indexes.empty()); +#endif + runtime->forest->destroy_field_space(field_space, applied); + break; + } + case FIELD_DELETION: + // Everyone is going to do the same thing for field deletions + break; + case LOGICAL_REGION_DELETION: + { + // Only do something here if we don't have any parent req indexes + // If we had no deletion requirements then we know there is + // nothing to race with and we can just do our deletion + if (parent_req_indexes.empty()) + runtime->forest->destroy_logical_region(logical_region,applied); + break; + } + default: + assert(false); + } + } + std::vector regions_to_destroy; + // If this is a field deletion then everyone does the same thing + if (kind == FIELD_DELETION) + { + if (!local_fields.empty()) + runtime->forest->free_local_fields(field_space, local_fields, + local_field_indexes, true/*collective*/); + if (!global_fields.empty()) + runtime->forest->free_fields(field_space, global_fields, applied, + (repl_ctx->owner_shard->shard_id != 0)); + parent_ctx->remove_deleted_fields(free_fields, parent_req_indexes); + if (!local_fields.empty()) + parent_ctx->remove_deleted_local_fields(field_space, local_fields); + if (!deletion_req_indexes.empty()) + parent_ctx->remove_deleted_requirements(deletion_req_indexes, + regions_to_destroy); + } + else if ((kind == LOGICAL_REGION_DELETION) && !parent_req_indexes.empty()) + parent_ctx->remove_deleted_requirements(parent_req_indexes, + regions_to_destroy); + if (!regions_to_destroy.empty()) + { + // Only selectively delete depending on our configuration + if (is_total_sharding && is_first_local_shard) + { + for (std::vector::const_iterator it = + regions_to_destroy.begin(); it != regions_to_destroy.end(); it++) + runtime->forest->destroy_logical_region(*it, applied, + true/*collective*/); + } + else if (repl_ctx->owner_shard->shard_id == 0) + { + for (std::vector::const_iterator it = + regions_to_destroy.begin(); it != regions_to_destroy.end(); it++) + runtime->forest->destroy_logical_region(*it, applied); + } + } +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + if (!applied.empty()) + complete_operation(Runtime::merge_events(applied)); + else + complete_operation(); + } + + //-------------------------------------------------------------------------- + void ReplDeletionOp::initialize_replication(ReplicateContext *ctx, + RtBarrier &delready_barrier, + RtBarrier &delmap_barrier, + RtBarrier &delexec_barrier, + bool is_total, bool is_first, + bool unordered/*=false*/) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!ready_barrier.exists()); + assert(!mapping_barrier.exists()); + assert(!execution_barrier.exists()); +#endif + // Only field and region deletions need a ready barrier since they + // will be touching the physical states of the region tree + if ((kind == LOGICAL_REGION_DELETION) || (kind == FIELD_DELETION)) + { + ready_barrier = delready_barrier; + if (unordered) + Runtime::advance_barrier(delready_barrier); + else + ctx->advance_replicate_barrier(delready_barrier, ctx->total_shards); + // Only field deletions need a mapping barrier for downward facing + // dependences in other shards + if (kind == FIELD_DELETION) + { + mapping_barrier = delmap_barrier; + if (unordered) + Runtime::advance_barrier(delmap_barrier); + else + ctx->advance_replicate_barrier(delmap_barrier, ctx->total_shards); + } + } + // All deletion kinds need an execution barrier + execution_barrier = delexec_barrier; + if (unordered) + Runtime::advance_barrier(delexec_barrier); + else + ctx->advance_replicate_barrier(delexec_barrier, ctx->total_shards); + is_total_sharding = is_total; + is_first_local_shard = is_first; + } + + //-------------------------------------------------------------------------- + void ReplDeletionOp::record_unordered_kind( + std::map &index_space_deletions, + std::map &index_partition_deletions, + std::map field_space_deletions, + std::map,ReplDeletionOp*> &field_deletions, + std::map &logical_region_deletions) + //-------------------------------------------------------------------------- + { + switch (kind) + { + case INDEX_SPACE_DELETION: + { +#ifdef DEBUG_LEGION + assert(index_space_deletions.find(index_space) == + index_space_deletions.end()); +#endif + index_space_deletions[index_space] = this; + break; + } + case INDEX_PARTITION_DELETION: + { +#ifdef DEBUG_LEGION + assert(index_partition_deletions.find(index_part) == + index_partition_deletions.end()); +#endif + index_partition_deletions[index_part] = this; + break; + } + case FIELD_SPACE_DELETION: + { +#ifdef DEBUG_LEGION + assert(field_space_deletions.find(field_space) == + field_space_deletions.end()); +#endif + field_space_deletions[field_space] = this; + break; + } + case FIELD_DELETION: + { +#ifdef DEBUG_LEGION + assert(!free_fields.empty()); +#endif + const std::pair key(field_space, + *(free_fields.begin())); +#ifdef DEBUG_LEGION + assert(field_deletions.find(key) == field_deletions.end()); +#endif + field_deletions[key] = this; + break; + } + case LOGICAL_REGION_DELETION: + { +#ifdef DEBUG_LEGION + assert(logical_region_deletions.find(logical_region) == + logical_region_deletions.end()); +#endif + logical_region_deletions[logical_region] = this; + break; + } + default: + assert(false); // should never get here + } + } + + ///////////////////////////////////////////////////////////// + // Repl Pending Partition Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplPendingPartitionOp::ReplPendingPartitionOp(Runtime *rt) + : PendingPartitionOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplPendingPartitionOp::ReplPendingPartitionOp( + const ReplPendingPartitionOp &rhs) + : PendingPartitionOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplPendingPartitionOp::~ReplPendingPartitionOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplPendingPartitionOp& ReplPendingPartitionOp::operator=( + const ReplPendingPartitionOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplPendingPartitionOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_pending(); + } + + //-------------------------------------------------------------------------- + void ReplPendingPartitionOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_pending(); + runtime->free_repl_pending_partition_op(this); + } + + //-------------------------------------------------------------------------- + void ReplPendingPartitionOp::trigger_complete(void) + //-------------------------------------------------------------------------- + { + // We know we are in a replicate context +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Perform the partitioning operation + const ApEvent ready_event = thunk->perform_shard(this, runtime->forest, + repl_ctx->owner_shard->shard_id, repl_ctx->shard_manager->total_shards); +#ifdef LEGION_SPY + // Still have to do this call to let Legion Spy know we're done + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + complete_operation(Runtime::protect_event(ready_event)); + } + + ///////////////////////////////////////////////////////////// + // Repl Dependent Partition Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplDependentPartitionOp::ReplDependentPartitionOp(Runtime *rt) + : DependentPartitionOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplDependentPartitionOp::ReplDependentPartitionOp( + const ReplDependentPartitionOp &rhs) + : DependentPartitionOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplDependentPartitionOp::~ReplDependentPartitionOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplDependentPartitionOp& ReplDependentPartitionOp::operator=( + const ReplDependentPartitionOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::initialize_by_field(ReplicateContext *ctx, + ShardID target, + ApEvent ready_event, + IndexPartition pid, + LogicalRegion handle, + LogicalRegion parent, + FieldID fid, + MapperID id, + MappingTagID t, + RtBarrier &deppart_bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (!runtime->forest->check_partition_by_field_size(pid, + handle.get_field_space(), fid, false/*range*/, + true/*use color space*/)) + { + log_run.error("ERROR: Field size of field %d does not match the size " + "of the color space elements for 'partition_by_field' " + "call in task %s (UID %lld)", fid, ctx->get_task_name(), + ctx->get_unique_id()); + assert(false); + } +#endif + parent_task = ctx->get_task(); + initialize_operation(ctx, true/*track*/); + // Start without the projection requirement, we'll ask + // the mapper later if it wants to turn this into an index launch + requirement = + RegionRequirement(handle, LEGION_READ_ONLY, LEGION_EXCLUSIVE, parent); + requirement.add_field(fid); + map_id = id; + tag = t; +#ifdef DEBUG_LEGION + assert(thunk == NULL); +#endif + thunk = new ReplByFieldThunk(ctx, target, pid); + mapping_barrier = deppart_bar; + ctx->advance_replicate_barrier(deppart_bar, ctx->total_shards); + partition_ready = ready_event; + if (runtime->legion_spy_enabled) + perform_logging(); + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::initialize_by_image(ReplicateContext *ctx, +#ifndef SHARD_BY_IMAGE + ShardID target, +#endif + ApEvent ready_event, + IndexPartition pid, + LogicalPartition projection, + LogicalRegion parent, FieldID fid, + MapperID id, MappingTagID t, + ShardID shard, size_t total, + RtBarrier &deppart_bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (!runtime->forest->check_partition_by_field_size(pid, + projection.get_field_space(), fid, false/*range*/)) + { + log_run.error("ERROR: Field size of field %d does not match the size " + "of the destination index space elements for " + "'partition_by_image' call in task %s (UID %lld)", + fid, ctx->get_task_name(), ctx->get_unique_id()); + assert(false); + } +#endif + parent_task = ctx->get_task(); + initialize_operation(ctx, true/*track*/); + // Start without the projection requirement, we'll ask + // the mapper later if it wants to turn this into an index launch + LogicalRegion proj_parent = + runtime->forest->get_parent_logical_region(projection); + requirement = + RegionRequirement(proj_parent,LEGION_READ_ONLY,LEGION_EXCLUSIVE,parent); + requirement.add_field(fid); + map_id = id; + tag = t; +#ifdef DEBUG_LEGION + assert(thunk == NULL); +#endif +#ifdef SHARD_BY_IMAGE + thunk = new ReplByImageThunk(ctx, pid, projection.get_index_partition(), + shard, total); +#else + thunk = new ReplByImageThunk(ctx, target, pid, + projection.get_index_partition(), + shard, total); +#endif + mapping_barrier = deppart_bar; + ctx->advance_replicate_barrier(deppart_bar, ctx->total_shards); + partition_ready = ready_event; + if (runtime->legion_spy_enabled) + perform_logging(); + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::initialize_by_image_range( + ReplicateContext *ctx, +#ifndef SHARD_BY_IMAGE + ShardID target, +#endif + ApEvent ready_event, + IndexPartition pid, + LogicalPartition projection, + LogicalRegion parent, + FieldID fid, MapperID id, + MappingTagID t, ShardID shard, + size_t total_shards, + RtBarrier &deppart_bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (!runtime->forest->check_partition_by_field_size(pid, + projection.get_field_space(), fid, true/*range*/)) + { + log_run.error("ERROR: Field size of field %d does not match the size " + "of the destination index space elements for " + "'partition_by_image_range' call in task %s (UID %lld)", + fid, ctx->get_task_name(), ctx->get_unique_id()); + assert(false); + } +#endif + parent_task = ctx->get_task(); + initialize_operation(ctx, true/*track*/); + // Start without the projection requirement, we'll ask + // the mapper later if it wants to turn this into an index launch + LogicalRegion proj_parent = + runtime->forest->get_parent_logical_region(projection); + requirement = + RegionRequirement(proj_parent,LEGION_READ_ONLY,LEGION_EXCLUSIVE,parent); + requirement.add_field(fid); + map_id = id; + tag = t; +#ifdef DEBUG_LEGION + assert(thunk == NULL); +#endif +#ifdef SHARD_BY_IMAGE + thunk = new ReplByImageRangeThunk(ctx, pid, + projection.get_index_partition(), + shard, total_shards); +#else + thunk = new ReplByImageRangeThunk(ctx, target, pid, + projection.get_index_partition(), + shard, total_shards); +#endif + mapping_barrier = deppart_bar; + ctx->advance_replicate_barrier(deppart_bar, ctx->total_shards); + partition_ready = ready_event; + if (runtime->legion_spy_enabled) + perform_logging(); + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::initialize_by_preimage(ReplicateContext *ctx, + ShardID target_shard, ApEvent ready_event, + IndexPartition pid, IndexPartition proj, + LogicalRegion handle, LogicalRegion parent, + FieldID fid, MapperID id, MappingTagID t, + RtBarrier &deppart_bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (!runtime->forest->check_partition_by_field_size(proj, + handle.get_field_space(), fid, false/*range*/)) + { + log_run.error("ERROR: Field size of field %d does not match the size " + "of the range index space elements for " + "'partition_by_preimage' call in task %s (UID %lld)", + fid, ctx->get_task_name(), ctx->get_unique_id()); + assert(false); + } +#endif + parent_task = ctx->get_task(); + initialize_operation(ctx, true/*track*/); + // Start without the projection requirement, we'll ask + // the mapper later if it wants to turn this into an index launch + requirement = + RegionRequirement(handle, LEGION_READ_ONLY, LEGION_EXCLUSIVE, parent); + requirement.add_field(fid); + map_id = id; + tag = t; +#ifdef DEBUG_LEGION + assert(thunk == NULL); +#endif + thunk = new ReplByPreimageThunk(ctx, target_shard, pid, proj); + mapping_barrier = deppart_bar; + ctx->advance_replicate_barrier(deppart_bar, ctx->total_shards); + partition_ready = ready_event; + if (runtime->legion_spy_enabled) + perform_logging(); + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::initialize_by_preimage_range( + ReplicateContext *ctx, ShardID target_shard, + ApEvent ready_event, + IndexPartition pid, IndexPartition proj, + LogicalRegion handle, LogicalRegion parent, + FieldID fid, MapperID id, MappingTagID t, + RtBarrier &deppart_bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (!runtime->forest->check_partition_by_field_size(proj, + handle.get_field_space(), fid, true/*range*/)) + { + log_run.error("ERROR: Field size of field %d does not match the size " + "of the range index space elements for " + "'partition_by_preimage_range' call in task %s (UID %lld)", + fid, ctx->get_task_name(), ctx->get_unique_id()); + assert(false); + } +#endif + parent_task = ctx->get_task(); + initialize_operation(ctx, true/*track*/); + // Start without the projection requirement, we'll ask + // the mapper later if it wants to turn this into an index launch + requirement = + RegionRequirement(handle, LEGION_READ_ONLY, LEGION_EXCLUSIVE, parent); + requirement.add_field(fid); + map_id = id; + tag = t; +#ifdef DEBUG_LEGION + assert(thunk == NULL); +#endif + thunk = new ReplByPreimageRangeThunk(ctx, target_shard, pid, proj); + mapping_barrier = deppart_bar; + ctx->advance_replicate_barrier(deppart_bar, ctx->total_shards); + partition_ready = ready_event; + if (runtime->legion_spy_enabled) + perform_logging(); + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::initialize_by_association( + ReplicateContext *ctx, LogicalRegion domain, + LogicalRegion domain_parent, FieldID fid, + IndexSpace range, MapperID id, MappingTagID tag, + RtBarrier &deppart_bar) + //-------------------------------------------------------------------------- + { + mapping_barrier = deppart_bar; + ctx->advance_replicate_barrier(deppart_bar, ctx->total_shards); + DependentPartitionOp::initialize_by_association(ctx, domain, + domain_parent, fid, range, id, tag); + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_dependent_op(); + sharding_function = NULL; +#ifdef DEBUG_LEGION + sharding_collective = NULL; +#endif + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_dependent_op(); +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + delete sharding_collective; +#endif + runtime->free_repl_dependent_partition_op(this); + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::select_sharding_function(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); + assert(sharding_function == NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Do the mapper call to get the sharding function to use + if (mapper == NULL) + mapper = runtime->find_mapper( + parent_ctx->get_executing_processor(), map_id); + Mapper::SelectShardingFunctorInput* input = repl_ctx->shard_manager; + Mapper::SelectShardingFunctorOutput output; + output.chosen_functor = UINT_MAX; + mapper->invoke_partition_select_sharding_functor(this, input, &output); + if (output.chosen_functor == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s failed to pick a valid sharding functor for " + "dependent partition in task %s (UID %lld)", + mapper->get_mapper_name(), + parent_ctx->get_task_name(), parent_ctx->get_unique_id()) + sharding_function = repl_ctx->shard_manager->find_sharding_function( + output.chosen_functor); +#ifdef DEBUG_LEGION + assert(sharding_collective != NULL); + sharding_collective->contribute(output.chosen_functor); + if (sharding_collective->is_target() && + !sharding_collective->validate(output.chosen_functor)) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s chose different sharding functions " + "for dependent partition op in task %s (UID %lld)", + mapper->get_mapper_name(), parent_ctx->get_task_name(), + parent_ctx->get_unique_id()) +#endif + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { + if (runtime->check_privileges) + check_privilege(); + // Before doing the dependence analysis we have to ask the + // mapper whether it would like to make this an index space + // operation or a single operation + select_partition_projection(); + // Now that we know that we have the right region requirement we + // can ask the mapper to also pick the sharding function + select_sharding_function(); + // Do thise now that we've picked our region requirement + initialize_privilege_path(privilege_path, requirement); + if (runtime->legion_spy_enabled) + log_requirement(); + ProjectionInfo projection_info; + if (is_index_space) + projection_info = ProjectionInfo(runtime, requirement, + launch_space, sharding_function); + runtime->forest->perform_dependence_analysis(this, 0/*idx*/, + requirement, + projection_info, + privilege_path, + map_applied_conditions); + // Record this dependent partition op with the context so that it + // can track implicit dependences on it for later operations + parent_ctx->update_current_implicit(this); + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + + // Do different things if this is an index space point or a single point + if (is_index_space) + { +#ifdef DEBUG_LEGION + assert(sharding_function != NULL); +#endif + // Compute the local index space of points for this shard + IndexSpace local_space = + sharding_function->find_shard_space(repl_ctx->owner_shard->shard_id, + launch_space, launch_space->handle); + // If it's empty we're done, otherwise we go back on the queue + if (!local_space.exists()) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We aren't participating directly, but we still have to + // participate in the collective operations + const ApEvent done_event = + thunk->perform(this,runtime->forest,ApEvent::NO_AP_EVENT,instances); + // We can try to early-complete this operation too + request_early_complete(done_event); + // We have no local points, so we can just trigger + Runtime::phase_barrier_arrive(mapping_barrier, 1/*count*/); + complete_mapping(mapping_barrier); + complete_execution(Runtime::protect_event(done_event)); + } + else // If we have valid points then we do the base call + { + if (remove_launch_space_reference(launch_space)) + delete launch_space; + launch_space = runtime->forest->get_node(local_space); + add_launch_space_reference(launch_space); + // Update the index domain to match the launch space + launch_space->get_launch_space_domain(index_domain); + DependentPartitionOp::trigger_ready(); + } + } + else + { + // Inform the thunk that we're eliding collectives since this + // is a singular operation and not an index operation + thunk->elide_collectives(); + // Shard 0 always owns dependent partition operations + // If we own it we go on the queue, otherwise we complete early + if (repl_ctx->owner_shard->shard_id != 0) + { +#ifdef LEGION_SPY + // Still have to do this for legion spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif + // We don't own it, so we can pretend like we + // mapped and executed this task already + Runtime::phase_barrier_arrive(mapping_barrier, 1/*count*/); + complete_mapping(mapping_barrier); + complete_execution(); + } + else // If we're the shard then we do the base call + DependentPartitionOp::trigger_ready(); + } + } + + //-------------------------------------------------------------------------- + void ReplDependentPartitionOp::finalize_mapping(void) + //-------------------------------------------------------------------------- + { + RtEvent precondition; + if (!map_applied_conditions.empty()) + precondition = Runtime::merge_events(map_applied_conditions); + Runtime::phase_barrier_arrive(mapping_barrier, 1/*count*/, precondition); + if (!acquired_instances.empty()) + precondition = release_nonempty_acquired_instances(mapping_barrier, + acquired_instances); + else + precondition = mapping_barrier; + complete_mapping(precondition); + } + + //-------------------------------------------------------------------------- + ReplDependentPartitionOp::ReplByFieldThunk::ReplByFieldThunk( + ReplicateContext *ctx, ShardID target, IndexPartition p) + : ByFieldThunk(p), + gather_collective(FieldDescriptorGather(ctx, target, COLLECTIVE_LOC_54)) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ApEvent ReplDependentPartitionOp::ReplByFieldThunk::perform( + DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances) + //-------------------------------------------------------------------------- + { + if (op->is_index_space) + { + gather_collective.contribute(instances_ready, instances); + if (gather_collective.is_target()) + { + ApEvent all_ready; + const std::vector &full_descriptors = + gather_collective.get_full_descriptors(all_ready); + // Perform the operation + ApEvent done = forest->create_partition_by_field(op, pid, + full_descriptors, all_ready); + gather_collective.notify_remote_complete(done); + return done; + } + else // nothing else for us to do + return gather_collective.get_complete_event(); + } + else // singular so just do the normal thing + return forest->create_partition_by_field(op, pid, + instances, instances_ready); + } + + //-------------------------------------------------------------------------- +#ifdef SHARD_BY_IMAGE + ReplDependentPartitionOp::ReplByImageThunk::ReplByImageThunk( + ReplicateContext *ctx, + IndexPartition p, IndexPartition proj, + ShardID s, size_t total) + : ByImageThunk(p, proj), + collective(FieldDescriptorExchange(ctx, COLLECTIVE_LOC_55)), +#else + ReplDependentPartitionOp::ReplByImageThunk::ReplByImageThunk( + ReplicateContext *ctx, ShardID target, + IndexPartition p, IndexPartition proj, + ShardID s, size_t total) + : ByImageThunk(p, proj), + collective(FieldDescriptorGather(ctx, target, COLLECTIVE_LOC_55)), +#endif + shard_id(s), total_shards(total) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ApEvent ReplDependentPartitionOp::ReplByImageThunk::perform( + DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances) + //-------------------------------------------------------------------------- + { + if (op->is_index_space) + { +#ifdef SHARD_BY_IMAGE + // There is a special case here if we're projecting the same + // partition that we used to make the instances, if it is then + // we can avoid needing to do the exchange at all + if ((op->requirement.handle_type == PART_PROJECTION) && + (op->requirement.partition.get_index_partition() == projection)) + { + // Make sure we elide our collective to avoid leaking anything + collective.elide_collective(); + if (!instances.empty()) + return forest->create_partition_by_image_range(op, pid, projection, + instances, instances_ready, shard_id, total_shards); + else + return ApEvent::NO_AP_EVENT; + } + // Do the all-to-all gather of the field data descriptors + ApEvent all_ready = collective.exchange_descriptors(instances_ready, + instances); + ApEvent done = forest->create_partition_by_image(op, pid, projection, + collective.descriptors, all_ready, shard_id, total_shards); + return collective.exchange_completion(done); +#else + collective.contribute(instances_ready, instances); + if (collective.is_target()) + { + ApEvent all_ready; + const std::vector &full_descriptors = + collective.get_full_descriptors(all_ready); + // Perform the operation + ApEvent done = forest->create_partition_by_image(op, pid, + projection, full_descriptors, all_ready); + collective.notify_remote_complete(done); + return done; + } + else // nothing else for us to do + return collective.get_complete_event(); +#endif + } + else // singular so just do the normal thing + return forest->create_partition_by_image(op, pid, projection, + instances, instances_ready); + } + + //-------------------------------------------------------------------------- +#ifdef SHARD_BY_IMAGE + ReplDependentPartitionOp::ReplByImageRangeThunk::ReplByImageRangeThunk( + ReplicateContext *ctx, + IndexPartition p, IndexPartition proj, + ShardID s, size_t total) + : ByImageRangeThunk(p, proj), + collective(FieldDescriptorExchange(ctx, COLLECTIVE_LOC_60)), +#else + ReplDependentPartitionOp::ReplByImageRangeThunk::ReplByImageRangeThunk( + ReplicateContext *ctx, ShardID target, + IndexPartition p, IndexPartition proj, + ShardID s, size_t total) + : ByImageRangeThunk(p, proj), + collective(FieldDescriptorGather(ctx, target, COLLECTIVE_LOC_60)), +#endif + shard_id(s), total_shards(total) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ApEvent ReplDependentPartitionOp::ReplByImageRangeThunk::perform( + DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances) + //-------------------------------------------------------------------------- + { + if (op->is_index_space) + { +#ifdef SHARD_BY_IMAGE + // There is a special case here if we're projecting the same + // partition that we used to make the instances, if it is then + // we can avoid needing to do the exchange at all + if ((op->requirement.handle_type == PART_PROJECTION) && + (op->requirement.partition.get_index_partition() == projection)) + { + // Make sure we elide our collective to avoid leaking anything + collective.elide_collective(); + if (!instances.empty()) + return forest->create_partition_by_image_range(op, pid, projection, + instances, instances_ready, shard_id, total_shards); + else + return ApEvent::NO_AP_EVENT; + } + // Do the all-to-all gather of the field data descriptors + ApEvent all_ready = collective.exchange_descriptors(instances_ready, + instances); + ApEvent done = forest->create_partition_by_image_range(op, pid, + projection,collective.descriptors,all_ready,shard_id,total_shards); + return collective.exchange_completion(done); +#else + collective.contribute(instances_ready, instances); + if (collective.is_target()) + { + ApEvent all_ready; + const std::vector &full_descriptors = + collective.get_full_descriptors(all_ready); + // Perform the operation + ApEvent done = forest->create_partition_by_image_range(op, pid, + projection, full_descriptors, all_ready); + collective.notify_remote_complete(done); + return done; + } + else // nothing else for us to do + return collective.get_complete_event(); +#endif + } + else // singular so just do the normal thing + return forest->create_partition_by_image_range(op, pid, projection, + instances, instances_ready); + } + + //-------------------------------------------------------------------------- + ReplDependentPartitionOp::ReplByPreimageThunk::ReplByPreimageThunk( + ReplicateContext *ctx, ShardID target, + IndexPartition p, IndexPartition proj) + : ByPreimageThunk(p, proj), + gather_collective(FieldDescriptorGather(ctx, target, COLLECTIVE_LOC_56)) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ApEvent ReplDependentPartitionOp::ReplByPreimageThunk::perform( + DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances) + //-------------------------------------------------------------------------- + { + if (op->is_index_space) + { + gather_collective.contribute(instances_ready, instances); + if (gather_collective.is_target()) + { + ApEvent all_ready; + const std::vector &full_descriptors = + gather_collective.get_full_descriptors(all_ready); + // Perform the operation + ApEvent done = forest->create_partition_by_preimage(op, pid, + projection, full_descriptors, all_ready); + gather_collective.notify_remote_complete(done); + return done; + } + else // nothing else for us to do + return gather_collective.get_complete_event(); + } + else // singular so just do the normal thing + return forest->create_partition_by_preimage(op, pid, projection, + instances, instances_ready); + } + + //-------------------------------------------------------------------------- + ReplDependentPartitionOp::ReplByPreimageRangeThunk:: + ReplByPreimageRangeThunk(ReplicateContext *ctx, ShardID target, + IndexPartition p, IndexPartition proj) + : ByPreimageRangeThunk(p, proj), + gather_collective(FieldDescriptorGather(ctx, target, COLLECTIVE_LOC_57)) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ApEvent ReplDependentPartitionOp::ReplByPreimageRangeThunk::perform( + DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances) + //-------------------------------------------------------------------------- + { + if (op->is_index_space) + { + gather_collective.contribute(instances_ready, instances); + if (gather_collective.is_target()) + { + ApEvent all_ready; + const std::vector &full_descriptors = + gather_collective.get_full_descriptors(all_ready); + // Perform the operation + ApEvent done = forest->create_partition_by_preimage_range(op, pid, + projection, full_descriptors, all_ready); + gather_collective.notify_remote_complete(done); + return done; + } + else // nothing else for us to do + return gather_collective.get_complete_event(); + } + else // singular so just do the normal thing + return forest->create_partition_by_preimage_range(op, pid, projection, + instances, instances_ready); + } + + ///////////////////////////////////////////////////////////// + // Repl Must Epoch Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplMustEpochOp::ReplMustEpochOp(Runtime *rt) + : MustEpochOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplMustEpochOp::ReplMustEpochOp(const ReplMustEpochOp &rhs) + : MustEpochOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplMustEpochOp::~ReplMustEpochOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplMustEpochOp& ReplMustEpochOp::operator=(const ReplMustEpochOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_must_epoch_op(); + sharding_functor = UINT_MAX; + sharding_function = NULL; + mapping_collective_id = 0; + collective_map_must_epoch_call = false; + mapping_broadcast = NULL; + mapping_exchange = NULL; + dependence_exchange = NULL; + completion_exchange = NULL; +#ifdef DEBUG_LEGION + sharding_collective = NULL; +#endif + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_must_epoch_op(); + shard_single_tasks.clear(); + runtime->free_repl_epoch_op(this); + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::instantiate_tasks(InnerContext *ctx, + const MustEpochLauncher &launcher) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(ctx); +#endif + // Initialize operations for everything in the launcher + // Note that we do not track these operations as we want them all to + // appear as a single operation to the parent context in order to + // avoid deadlock with the maximum window size. + indiv_tasks.resize(launcher.single_tasks.size()); + for (unsigned idx = 0; idx < launcher.single_tasks.size(); idx++) + { + ReplIndividualTask *task = + runtime->get_available_repl_individual_task(); + task->initialize_task(ctx, launcher.single_tasks[idx], false/*track*/); + task->set_must_epoch(this, idx, true/*register*/); + // If we have a trace, set it for this operation as well + if (trace != NULL) + task->set_trace(trace, NULL); + task->must_epoch_task = true; + task->initialize_replication(repl_ctx); + task->index_domain = this->launch_domain; + task->sharding_space = launcher.sharding_space; +#ifdef DEBUG_LEGION + task->set_sharding_collective(new ShardingGatherCollective(repl_ctx, + 0/*owner shard*/, COLLECTIVE_LOC_59)); +#endif + indiv_tasks[idx] = task; + } + indiv_triggered.resize(indiv_tasks.size(), false); + index_tasks.resize(launcher.index_tasks.size()); + for (unsigned idx = 0; idx < launcher.index_tasks.size(); idx++) + { + IndexSpace launch_space = launcher.index_tasks[idx].launch_space; + if (!launch_space.exists()) + launch_space = ctx->find_index_launch_space( + launcher.index_tasks[idx].launch_domain); + ReplIndexTask *task = runtime->get_available_repl_index_task(); + task->initialize_task(ctx, launcher.index_tasks[idx], + launch_space, false/*track*/); + task->set_must_epoch(this, indiv_tasks.size()+idx, + true/*register*/); + if (trace != NULL) + task->set_trace(trace, NULL); + task->must_epoch_task = true; + task->initialize_replication(repl_ctx); + task->sharding_space = launcher.sharding_space; +#ifdef DEBUG_LEGION + task->set_sharding_collective(new ShardingGatherCollective(repl_ctx, + 0/*owner shard*/, COLLECTIVE_LOC_59)); +#endif + index_tasks[idx] = task; + } + index_triggered.resize(index_tasks.size(), false); + } + + //-------------------------------------------------------------------------- + FutureMapImpl* ReplMustEpochOp::create_future_map(TaskContext *ctx, + const Domain &domain, IndexSpace shard_space, RtUserEvent deleted) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(ctx); +#endif + Domain shard_domain; + if (shard_space.exists()) + runtime->forest->find_launch_space_domain(shard_space, shard_domain); + else + shard_domain = domain; + return new ReplFutureMapImpl(repl_ctx, this, + Runtime::protect_event(get_completion_event()), domain, shard_domain, + runtime, runtime->get_available_distributed_id(), + runtime->address_space); + } + + //-------------------------------------------------------------------------- + MapperManager* ReplMustEpochOp::invoke_mapper(void) + //-------------------------------------------------------------------------- + { + Processor mapper_proc = parent_ctx->get_executing_processor(); + MapperManager *mapper = runtime->find_mapper(mapper_proc, map_id); +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // We want to do the map must epoch call + // First find all the tasks that we own on this shard + Domain shard_domain = launch_domain; + if (sharding_space.exists()) + runtime->forest->find_launch_space_domain(sharding_space, shard_domain); + for (std::vector::const_iterator it = + single_tasks.begin(); it != single_tasks.end(); it++) + { + const ShardID shard = + sharding_function->find_owner((*it)->index_point, shard_domain); + if (runtime->legion_spy_enabled) + LegionSpy::log_owner_shard((*it)->get_unique_id(), shard); + // If it is not our shard then we don't own it + if (shard != repl_ctx->owner_shard->shard_id) + continue; + shard_single_tasks.insert(*it); + } + // Find the set of constraints that apply to our local set of tasks + std::vector local_constraints; + std::vector original_constraint_indexes; + for (unsigned idx = 0; idx < input.constraints.size(); idx++) + { + bool is_local = false; + for (std::vector::const_iterator it = + input.constraints[idx].constrained_tasks.begin(); it != + input.constraints[idx].constrained_tasks.end(); it++) + { + SingleTask *single = static_cast(const_cast(*it)); + if (shard_single_tasks.find(single) == shard_single_tasks.end()) + continue; + is_local = true; + break; + } + if (is_local) + { + local_constraints.push_back(input.constraints[idx]); + original_constraint_indexes.push_back(idx); + } + } + if (collective_map_must_epoch_call) + { + // Update the input tasks for our subset + std::vector all_tasks(shard_single_tasks.begin(), + shard_single_tasks.end()); + input.tasks.swap(all_tasks); + // Sort them again by their index points to for determinism + std::sort(input.tasks.begin(), input.tasks.end(), single_task_sorter); + // Update the constraints to contain just our subset + const size_t total_constraints = input.constraints.size(); + input.constraints.swap(local_constraints); + // Fill in our shard mapping and local shard info + input.shard_mapping = repl_ctx->shard_manager->shard_mapping; + input.local_shard = repl_ctx->owner_shard->shard_id; + // Update the outputs + output.task_processors.resize(input.tasks.size()); + output.constraint_mappings.resize(input.constraints.size()); + output.weights.resize(input.constraints.size()); + // Now we can do the mapper call + mapper->invoke_map_must_epoch(this, &input, &output); + // Now we need to exchange our mapping decisions between all the shards +#ifdef DEBUG_LEGION + assert(mapping_exchange == NULL); + assert(mapping_collective_id > 0); +#endif + mapping_exchange = + new MustEpochMappingExchange(repl_ctx, mapping_collective_id); + mapping_exchange->exchange_must_epoch_mappings( + repl_ctx->owner_shard->shard_id, + repl_ctx->shard_manager->total_shards, total_constraints, + input.tasks, all_tasks, output.task_processors, + original_constraint_indexes, output.constraint_mappings, + output.weights, *get_acquired_instances_ref()); + } + else + { +#ifdef DEBUG_LEGION + assert(mapping_broadcast == NULL); + assert(mapping_collective_id > 0); +#endif + mapping_broadcast = new MustEpochMappingBroadcast(repl_ctx, + 0/*owner shard*/, mapping_collective_id); + // Do the mapper call on shard 0 and then broadcast the results + if (repl_ctx->owner_shard->shard_id == 0) + { + mapper->invoke_map_must_epoch(this, &input, &output); + mapping_broadcast->broadcast(output.task_processors, + output.constraint_mappings); + } + else + mapping_broadcast->receive_results(output.task_processors, + original_constraint_indexes, output.constraint_mappings, + *get_acquired_instances_ref()); + } + // No need to do any checks, the base class handles that + return mapper; + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::map_and_distribute(std::set &tasks_mapped, + std::set &tasks_complete) + //-------------------------------------------------------------------------- + { + // Perform the mapping + map_replicate_tasks(); + mapping_dependences.clear(); + // We have to exchange mapping and completion events with all the + // other shards as well + std::set local_tasks_mapped; + std::set local_tasks_complete; + for (std::vector::const_iterator it = + indiv_tasks.begin(); it != indiv_tasks.end(); it++) + { + local_tasks_mapped.insert((*it)->get_mapped_event()); + local_tasks_complete.insert((*it)->get_completion_event()); + } + for (std::vector::const_iterator it = + index_tasks.begin(); it != index_tasks.end(); it++) + { + local_tasks_mapped.insert((*it)->get_mapped_event()); + local_tasks_complete.insert((*it)->get_completion_event()); + } + RtEvent local_mapped = Runtime::merge_events(local_tasks_mapped); + tasks_mapped.insert(local_mapped); + ApEvent local_complete = Runtime::merge_events(NULL,local_tasks_complete); + tasks_complete.insert(local_complete); +#ifdef DEBUG_LEGION + assert(completion_exchange != NULL); +#endif + completion_exchange->exchange_must_epoch_completion( + local_mapped, local_complete, tasks_mapped, tasks_complete); + // Then we can distribute the tasks + distribute_replicate_tasks(); + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::trigger_prepipeline_stage(void) + //-------------------------------------------------------------------------- + { + Processor mapper_proc = parent_ctx->get_executing_processor(); + MapperManager *mapper = runtime->find_mapper(mapper_proc, map_id); +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Select our sharding functor and then do the base call + this->individual_tasks.resize(indiv_tasks.size()); + for (unsigned idx = 0; idx < indiv_tasks.size(); idx++) + this->individual_tasks[idx] = indiv_tasks[idx]; + this->index_space_tasks.resize(index_tasks.size()); + for (unsigned idx = 0; idx < index_tasks.size(); idx++) + this->index_space_tasks[idx] = index_tasks[idx]; + Mapper::SelectShardingFunctorInput sharding_input; + sharding_input.shard_mapping = repl_ctx->shard_manager->shard_mapping; + Mapper::MustEpochShardingFunctorOutput sharding_output; + sharding_output.chosen_functor = UINT_MAX; + sharding_output.collective_map_must_epoch_call = false; + mapper->invoke_must_epoch_select_sharding_functor(this, + &sharding_input, &sharding_output); + // We can clear these now that we don't need them anymore + individual_tasks.clear(); + index_space_tasks.clear(); + // Check that we have a sharding ID + if (sharding_output.chosen_functor == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Invalid mapper output from invocation of " + "'map_must_epoch' on mapper %s. Mapper failed to specify " + "a valid sharding ID for a must epoch operation in control " + "replicated context of task %s (UID %lld).", + mapper->get_mapper_name(), repl_ctx->get_task_name(), + repl_ctx->get_unique_id()) + this->sharding_functor = sharding_output.chosen_functor; + this->collective_map_must_epoch_call = + sharding_output.collective_map_must_epoch_call; +#ifdef DEBUG_LEGION + assert(sharding_function == NULL); + // Check that the sharding IDs are all the same + assert(sharding_collective != NULL); + // Contribute the result + sharding_collective->contribute(this->sharding_functor); + if (sharding_collective->is_target() && + !sharding_collective->validate(this->sharding_functor)) + { + log_run.error("ERROR: Mapper %s chose different sharding functions " + "for must epoch launch in %s (UID %lld)", + mapper->get_mapper_name(), parent_ctx->get_task_name(), + parent_ctx->get_unique_id()); + assert(false); + } + ReplFutureMapImpl *impl = + dynamic_cast(result_map.impl); + assert(impl != NULL); +#else + ReplFutureMapImpl *impl = + static_cast(result_map.impl); +#endif + // Set the future map sharding functor + sharding_function = + repl_ctx->shard_manager->find_sharding_function(sharding_functor); + impl->set_sharding_function(sharding_function); + // Set the sharding functor for all the point and index tasks too + for (unsigned idx = 0; idx < indiv_tasks.size(); idx++) + { + ReplIndividualTask *task = + static_cast(indiv_tasks[idx]); + task->set_sharding_function(sharding_functor, sharding_function); + } + for (unsigned idx = 0; idx < index_tasks.size(); idx++) + { + ReplIndexTask *task = static_cast(index_tasks[idx]); + task->set_sharding_function(sharding_functor, sharding_function); + } + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::trigger_commit(void) + //-------------------------------------------------------------------------- + { + // We have to delete these here to make sure that they are + // unregistered with the context before the context is deleted + if (mapping_broadcast != NULL) + delete mapping_broadcast; + if (mapping_exchange != NULL) + delete mapping_exchange; + if (dependence_exchange != NULL) + delete dependence_exchange; + if (completion_exchange != NULL) + delete completion_exchange; +#ifdef DEBUG_LEGION + if (sharding_collective != NULL) + delete sharding_collective; +#endif + MustEpochOp::trigger_commit(); + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::map_replicate_tasks(void) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(dependence_exchange != NULL); + assert(single_tasks.size() == mapping_dependences.size()); +#endif + std::map mapped_events; + for (std::set::const_iterator it = + shard_single_tasks.begin(); it != shard_single_tasks.end(); it++) + mapped_events[(*it)->index_point] = Runtime::create_rt_user_event(); + // Now exchange completion events for the point tasks we own + // and end up with a set of the completion event for each task + // First compute the set of mapped events for the points that we own + dependence_exchange->exchange_must_epoch_dependences(mapped_events); + + MustEpochMapArgs args(const_cast(this)); + std::set local_mapped_events; + // For correctness we still have to abide by the mapping dependences + // computed on the individual tasks while we are mapping them + for (unsigned idx = 0; idx < single_tasks.size(); idx++) + { + // Check to see if it is one of the ones that we own + if (shard_single_tasks.find(single_tasks[idx]) == + shard_single_tasks.end()) + { + // We don't own this point + // We still need to do some work for individual tasks + // to exchange versioning information, but no such + // work is necessary for point tasks + SingleTask *task = single_tasks[idx]; + task->shard_off(mapped_events[task->index_point]); + continue; + } + // Figure out our preconditions + std::set preconditions; + for (std::set::const_iterator it = + mapping_dependences[idx].begin(); it != + mapping_dependences[idx].end(); it++) + { +#ifdef DEBUG_LEGION + assert((*it) < idx); +#endif + preconditions.insert(mapped_events[single_tasks[*it]->index_point]); + } + args.task = single_tasks[idx]; + RtEvent done; + if (!preconditions.empty()) + { + RtEvent precondition = Runtime::merge_events(preconditions); + done = runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY, precondition); + } + else + done = runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + local_mapped_events.insert(done); + // We can trigger our completion event once the task is done + RtUserEvent mapped = mapped_events[single_tasks[idx]->index_point]; + Runtime::trigger_event(mapped, done); + } + // Now we have to wait for all our mapping operations to be done + if (!local_mapped_events.empty()) + { + RtEvent mapped_event = Runtime::merge_events(local_mapped_events); + mapped_event.wait(); + } + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::distribute_replicate_tasks(void) const + //-------------------------------------------------------------------------- + { + // We only want to distribute the points that are owned by our shard + ReplMustEpochOp *owner = const_cast(this); + MustEpochDistributorArgs dist_args(owner); + MustEpochLauncherArgs launch_args(owner); + std::set wait_events; + for (std::vector::const_iterator it = + indiv_tasks.begin(); it != indiv_tasks.end(); it++) + { + // Skip any points that we do not own on this shard + if (shard_single_tasks.find(*it) == shard_single_tasks.end()) + continue; + if (!runtime->is_local((*it)->target_proc)) + { + dist_args.task = *it; + RtEvent wait = + runtime->issue_runtime_meta_task(dist_args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); + } + else + { + launch_args.task = *it; + RtEvent wait = + runtime->issue_runtime_meta_task(launch_args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); + } + } + for (std::set::const_iterator it = + slice_tasks.begin(); it != slice_tasks.end(); it++) + { + // Check to see if we either do or not own this slice + // We currently do not support mixed slices for which + // we only own some of the points + bool contains_any = false; + bool contains_all = true; + for (std::vector::const_iterator pit = + (*it)->points.begin(); pit != (*it)->points.end(); pit++) + { + if (shard_single_tasks.find(*pit) != shard_single_tasks.end()) + contains_any = true; + else if (contains_all) + { + contains_all = false; + if (contains_any) // At this point we have all the answers + break; + } + } + if (!contains_any) + continue; + if (!contains_all) + { + Processor mapper_proc = parent_ctx->get_executing_processor(); + MapperManager *mapper = runtime->find_mapper(mapper_proc, map_id); + REPORT_LEGION_FATAL(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s specified a slice for a must epoch " + "launch in control replicated task %s " + "(UID %lld) for which not all the points " + "mapped to the same shard. Legion does not " + "currently support this use case. Please " + "specify slices and a sharding function to " + "ensure that all the points in a slice are " + "owned by the same shard", + mapper->get_mapper_name(), + parent_ctx->get_task_name(), + parent_ctx->get_unique_id()) + } + (*it)->update_target_processor(); + if (!runtime->is_local((*it)->target_proc)) + { + dist_args.task = *it; + RtEvent wait = + runtime->issue_runtime_meta_task(dist_args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); + } + else + { + launch_args.task = *it; + RtEvent wait = + runtime->issue_runtime_meta_task(launch_args, + LG_THROUGHPUT_DEFERRED_PRIORITY); + if (wait.exists()) + wait_events.insert(wait); + } + } + if (!wait_events.empty()) + { + RtEvent dist_event = Runtime::merge_events(wait_events); + dist_event.wait(); + } + } + + //-------------------------------------------------------------------------- + void ReplMustEpochOp::initialize_replication(ReplicateContext *ctx) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(mapping_collective_id == 0); + assert(mapping_broadcast == NULL); + assert(mapping_exchange == NULL); + assert(dependence_exchange == NULL); + assert(completion_exchange == NULL); +#endif + // We can't actually make a collective for the mapping yet because we + // don't know if we are going to broadcast or exchange so we just get + // a collective ID that we will use later + mapping_collective_id = ctx->get_next_collective_index(COLLECTIVE_LOC_58); + dependence_exchange = + new MustEpochDependenceExchange(ctx, COLLECTIVE_LOC_70); + completion_exchange = + new MustEpochCompletionExchange(ctx, COLLECTIVE_LOC_73); + } + + //-------------------------------------------------------------------------- + Domain ReplMustEpochOp::get_shard_domain(void) const + //-------------------------------------------------------------------------- + { + if (sharding_space.exists()) + { + Domain shard_domain; + runtime->forest->find_launch_space_domain(sharding_space, shard_domain); + return shard_domain; + } + else + return launch_domain; + } + + ///////////////////////////////////////////////////////////// + // Repl Timing Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplTimingOp::ReplTimingOp(Runtime *rt) + : TimingOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTimingOp::ReplTimingOp(const ReplTimingOp &rhs) + : TimingOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplTimingOp::~ReplTimingOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTimingOp& ReplTimingOp::operator=(const ReplTimingOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplTimingOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_timing(); + timing_collective = NULL; + } + + //-------------------------------------------------------------------------- + void ReplTimingOp::deactivate(void) + //-------------------------------------------------------------------------- + { + if (timing_collective != NULL) + { + delete timing_collective; + timing_collective = NULL; + } + deactivate_timing(); + runtime->free_repl_timing_op(this); + } + + //-------------------------------------------------------------------------- + void ReplTimingOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Shard 0 will handle the timing operation so do the normal mapping + if (repl_ctx->owner_shard->shard_id > 0) + { + complete_mapping(); + RtEvent result_ready = + timing_collective->perform_collective_wait(false/*block*/); + if (result_ready.exists() && !result_ready.has_triggered()) + { + // Defer completion until the value is ready + DeferredExecuteArgs deferred_execute_args(this); + runtime->issue_runtime_meta_task(deferred_execute_args, + LG_THROUGHPUT_DEFERRED_PRIORITY, result_ready); + } + else + deferred_execute(); + } + else // Shard 0 does the normal timing operation + TimingOp::trigger_mapping(); + } + + //-------------------------------------------------------------------------- + void ReplTimingOp::deferred_execute(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + // Shard 0 will handle the timing operation + if (repl_ctx->owner_shard->shard_id > 0) + { + long long value = timing_collective->get_value(false/*already waited*/); + result.impl->set_result(&value, sizeof(value), false); + } + else + { + // Perform the measurement and then arrive on the barrier + // with the result to broadcast it to the other shards + switch (measurement) + { + case LEGION_MEASURE_SECONDS: + { + double value = Realm::Clock::current_time(); + result.impl->set_result(&value, sizeof(value), false); + long long *ptr = reinterpret_cast(&value); + timing_collective->broadcast(*ptr); + break; + } + case LEGION_MEASURE_MICRO_SECONDS: + { + long long value = Realm::Clock::current_time_in_microseconds(); + result.impl->set_result(&value, sizeof(value), false); + timing_collective->broadcast(value); + break; + } + case LEGION_MEASURE_NANO_SECONDS: + { + long long value = Realm::Clock::current_time_in_nanoseconds(); + result.impl->set_result(&value, sizeof(value), false); + timing_collective->broadcast(value); + break; + } + default: + assert(false); // should never get here + } + } +#ifdef LEGION_SPY + // Still have to do this call to let Legion Spy know we're done + LegionSpy::log_operation_events(unique_op_id, ApEvent::NO_AP_EVENT, + ApEvent::NO_AP_EVENT); +#endif + complete_execution(); + } + + ///////////////////////////////////////////////////////////// + // Repl All Reduce Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplAllReduceOp::ReplAllReduceOp(Runtime *rt) + : AllReduceOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplAllReduceOp::ReplAllReduceOp(const ReplAllReduceOp &rhs) + : AllReduceOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplAllReduceOp::~ReplAllReduceOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplAllReduceOp& ReplAllReduceOp::operator=(const ReplAllReduceOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplAllReduceOp::initialize_replication(ReplicateContext *ctx) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(redop != NULL); + assert(exchange_collective == NULL); + assert(all_reduce_collective == NULL); +#endif + if (deterministic) + exchange_collective = + new FutureExchange(ctx, redop->sizeof_rhs, COLLECTIVE_LOC_97); + else + all_reduce_collective = + new AllReduceOpCollective(COLLECTIVE_LOC_97, ctx, redop); + } + + //-------------------------------------------------------------------------- + void ReplAllReduceOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_all_reduce(); + result_buffer = NULL; + exchange_collective = NULL; + all_reduce_collective = NULL; + } + + //-------------------------------------------------------------------------- + void ReplAllReduceOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_all_reduce(); + if (exchange_collective != NULL) + delete exchange_collective; + if (all_reduce_collective != NULL) + delete all_reduce_collective; + runtime->free_repl_all_reduce_op(this); + } + + //-------------------------------------------------------------------------- + void ReplAllReduceOp::deferred_execute(void) + //-------------------------------------------------------------------------- + { + // See if this is our first pass through to perform the reduction + if (result_buffer == NULL) + { + // First perform the reduction on our shard local futures + std::map futures; + future_map.impl->get_shard_local_futures(futures); + result_buffer = malloc(redop->sizeof_rhs); + redop->init(result_buffer, 1/*count*/); + for (std::map::const_iterator it = + futures.begin(); it != futures.end(); it++) + { + FutureImpl *impl = it->second; + const size_t future_size = impl->get_untyped_size(true/*internal*/); + if (future_size != redop->sizeof_rhs) + REPORT_LEGION_ERROR(ERROR_FUTURE_MAP_REDOP_TYPE_MISMATCH, + "Future in future map reduction in task %s (UID %lld) does not " + "have the right input size for the given reduction operator. " + "Future has size %zd bytes but reduction operator expects " + "RHS inputs of %zd bytes.", parent_ctx->get_task_name(), + parent_ctx->get_unique_id(), future_size, redop->sizeof_rhs) + const void *data = + impl->get_untyped_result(true,NULL,true/*internal*/); + redop->fold(result_buffer, data, 1/*count*/, true/*exclusive*/); + } + if (runtime->legion_spy_enabled) + { + for (std::map::const_iterator it = + futures.begin(); it != futures.end(); it++) + { + FutureImpl *impl = it->second; + const ApEvent ready_event = impl->get_ready_event(); + if (ready_event.exists()) + LegionSpy::log_future_use(unique_op_id, ready_event); + } + } + // Now do the exchange across the shards + RtEvent defer; + if (deterministic) + defer = exchange_collective->exchange_futures(result_buffer); + else + defer = all_reduce_collective->async_reduce(result_buffer); + if (defer.exists() && !defer.has_triggered()) + { + DeferredExecuteArgs args(this); + runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY, defer); + return; + } + } + // If we make it here then we can get the results of the + // reductions across the shards + if (deterministic) + exchange_collective->reduce_futures(redop, result_buffer); + else + all_reduce_collective->sync_result(result_buffer); + // Tell the future about the final result which it will own + result.impl->set_result(result_buffer, redop->sizeof_rhs, true/*own*/); +#ifdef LEGION_SPY + // Still have to do this call to let Legion Spy know we're done + LegionSpy::log_operation_events(unique_op_id, ApEvent::NO_AP_EVENT, + ApEvent::NO_AP_EVENT); +#endif + // Mark that we are done executing which will complete the future + // as soon as this operation is complete + complete_execution(); + } + + ///////////////////////////////////////////////////////////// + // Repl Fence Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplFenceOp::ReplFenceOp(Runtime *rt) + : FenceOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplFenceOp::ReplFenceOp(const ReplFenceOp &rhs) + : FenceOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplFenceOp::~ReplFenceOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplFenceOp& ReplFenceOp::operator=(const ReplFenceOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplFenceOp::activate(void) + //-------------------------------------------------------------------------- + { + FenceOp::activate(); + mapping_fence_barrier = RtBarrier::NO_RT_BARRIER; + execution_fence_barrier = ApBarrier::NO_AP_BARRIER; + } + + //-------------------------------------------------------------------------- + void ReplFenceOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_fence(); + runtime->free_repl_fence_op(this); + } + + //-------------------------------------------------------------------------- + Future ReplFenceOp::initialize_repl_fence(ReplicateContext *ctx, + FenceKind k, bool need_future, bool track) + //-------------------------------------------------------------------------- + { + Future f = initialize(ctx, k, need_future, track); + mapping_fence_barrier = ctx->get_next_mapping_fence_barrier(); + if (fence_kind == EXECUTION_FENCE) + execution_fence_barrier = ctx->get_next_execution_fence_barrier(); + return f; + } + + //-------------------------------------------------------------------------- + void ReplFenceOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { + switch (fence_kind) + { + case MAPPING_FENCE: + { + // Do our arrival + if (!map_applied_conditions.empty()) + Runtime::phase_barrier_arrive(mapping_fence_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + else + Runtime::phase_barrier_arrive(mapping_fence_barrier, 1/*count*/); + // We're mapped when everyone is mapped + complete_mapping(mapping_fence_barrier); + complete_execution(); + break; + } + case EXECUTION_FENCE: + { + // Do our arrival on our mapping fence, we're mapped when + // everyone is mapped + if (!map_applied_conditions.empty()) + Runtime::phase_barrier_arrive(mapping_fence_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + else + Runtime::phase_barrier_arrive(mapping_fence_barrier, 1/*count*/); + complete_mapping(mapping_fence_barrier); + // We arrive on our barrier when all our previous operations + // have finished executing + Runtime::phase_barrier_arrive(execution_fence_barrier, 1/*count*/, + execution_precondition); + // We can always trigger the completion event when these are done + request_early_complete(execution_fence_barrier); + if (!execution_fence_barrier.has_triggered()) + { + RtEvent wait_on = Runtime::protect_event(execution_fence_barrier); + complete_execution(wait_on); + } + else + complete_execution(); + break; + } + default: + assert(false); // should never get here + } + } + + ///////////////////////////////////////////////////////////// + // Repl Map Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplMapOp::ReplMapOp(Runtime *rt) + : MapOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplMapOp::ReplMapOp(const ReplMapOp &rhs) + : MapOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplMapOp::~ReplMapOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplMapOp& ReplMapOp::operator=(const ReplMapOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplMapOp::initialize_replication(ReplicateContext *ctx,RtBarrier &bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(exchange == NULL); + assert(view_did_broadcast == NULL); + assert(sharded_view == NULL); +#endif + inline_barrier = bar; + ctx->advance_replicate_barrier(bar, ctx->total_shards); + // We only check the results of the mapping if the runtime requests it + // We can skip the check though if this is a read-only requirement + if (!IS_READ_ONLY(requirement)) + exchange = new ShardedMappingExchange(COLLECTIVE_LOC_74, ctx, + ctx->owner_shard->shard_id, !runtime->unsafe_mapper); + if (IS_WRITE(requirement)) + { + // We need a second generation of the barrier for writes + ctx->advance_replicate_barrier(bar, ctx->total_shards); + // We need a third generation of the barrirer if we're not discarding + // the previous version of the barrier so we can make sure all the + // updates have been performed before we register our users + if (!IS_DISCARD(requirement)) + ctx->advance_replicate_barrier(bar, ctx->total_shards); + view_did_broadcast = + new ValueBroadcast(ctx, 0/*owner*/, COLLECTIVE_LOC_75); + // if we're shard 0 then get the distributed id and send it out + if (ctx->owner_shard->shard_id == 0) + { + DistributedID view_did = runtime->get_available_distributed_id(); + // make it and register it with the runtime + sharded_view = new ShardedView(runtime->forest, + view_did, runtime->address_space, true/*register now*/); + // then broadcast the result out so the other nodes can grab it + view_did_broadcast->broadcast(sharded_view->did); + } + } + } + + //-------------------------------------------------------------------------- + void ReplMapOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(inline_barrier.exists()); +#endif + // Compute the version numbers for this mapping operation + std::set preconditions; + runtime->forest->perform_versioning_analysis(this, 0/*idx*/, + requirement, + version_info, + preconditions); + if ((view_did_broadcast != NULL) && (sharded_view == NULL)) + { + // Get the distributed ID for the sharded view and request it + const DistributedID sharded_view_did = view_did_broadcast->get_value(); + RtEvent ready; + sharded_view = static_cast( + runtime->find_or_request_logical_view(sharded_view_did, ready)); + if (ready.exists()) + preconditions.insert(ready); + } + if (!preconditions.empty()) + enqueue_ready_operation(Runtime::merge_events(preconditions)); + else + enqueue_ready_operation(); + } + + //-------------------------------------------------------------------------- + void ReplMapOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { + const PhysicalTraceInfo trace_info(this, 0/*index*/, true/*init*/); + // If we have any wait preconditions from phase barriers or + // grants then we use them to compute a precondition for doing + // any copies or anything else for this operation + ApEvent init_precondition = execution_fence_event; + if (!wait_barriers.empty() || !grants.empty()) + { + ApEvent sync_precondition = + merge_sync_preconditions(trace_info, grants, wait_barriers); + if (sync_precondition.exists()) + { + if (init_precondition.exists()) + init_precondition = Runtime::merge_events(&trace_info, + init_precondition, sync_precondition); + else + init_precondition = sync_precondition; + } + } + InstanceSet mapped_instances; + // If we are remapping then we know the answer + // so we don't need to do any premapping + bool record_valid = true; + if (remap_region) + region.impl->get_references(mapped_instances); + else + record_valid = invoke_mapper(mapped_instances); + // First kick off the exchange to get that in flight + std::vector mapped_views; + { + InnerContext *context = find_physical_context(0/*index*/, requirement); + context->convert_target_views(mapped_instances, mapped_views); + if (exchange != NULL) + exchange->initiate_exchange(mapped_instances, mapped_views); + } +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + const bool is_owner_shard = (repl_ctx->owner_shard->shard_id == 0); + ApEvent effects_done; + // What we do next depends on the privileges + if (IS_REDUCE(requirement)) + { + // Shard 0 updates the equivalence sets with its reduction buffer + // Everyone else just needs to do their registration + if (!is_owner_shard) + { + InnerContext *context = find_physical_context(0/*index*/,requirement); + context->convert_target_views(mapped_instances, mapped_views); + RegionNode *node = runtime->forest->get_node(requirement.region); + UpdateAnalysis *analysis = new UpdateAnalysis(runtime, this, + 0/*index*/, version_info, + requirement, node, mapped_instances, + mapped_views,trace_info,init_precondition, + termination_event, true/*track effects*/, + false/*check initialized*/, record_valid, + false/*skip output*/); + analysis->add_reference(); + // Note that this call will clean up the analysis allocation + effects_done = runtime->forest->physical_perform_registration( + analysis, mapped_instances, trace_info, map_applied_conditions); + } + else + effects_done = + runtime->forest->physical_perform_updates_and_registration( + requirement, version_info, this, 0/*index*/, init_precondition, + termination_event, mapped_instances, trace_info, + map_applied_conditions, +#ifdef DEBUG_LEGION + get_logging_name(), unique_op_id, +#endif + true/*track effects*/); + // Complete the exchange + exchange->complete_exchange(this, sharded_view, + mapped_instances, map_applied_conditions); + } + else if (IS_WRITE(requirement) && IS_DISCARD(requirement)) + { +#ifdef DEBUG_LEGION + assert(sharded_view != NULL); + assert(exchange != NULL); + assert(record_valid); +#endif + // All the users just need to do their registration + RegionNode *node = runtime->forest->get_node(requirement.region); + UpdateAnalysis *analysis = new UpdateAnalysis(runtime, this, + 0/*index*/, version_info, + requirement, node, mapped_instances, + mapped_views,trace_info,init_precondition, + termination_event, true/*track effects*/, + false/*check initialized*/, record_valid, + false/*skip output*/); + analysis->add_reference(); + // Note that this call will clean up the analysis allocation + effects_done = + runtime->forest->physical_perform_registration(analysis, + mapped_instances, trace_info, map_applied_conditions); + // We need to fill in the sharded view before we do the next + // call in case there are output effects due to restriction + exchange->complete_exchange(this, sharded_view, + mapped_instances, map_applied_conditions); + // We need everyone to be done mapping before we can do the overwrite + if (!map_applied_conditions.empty()) + { + Runtime::phase_barrier_arrive(inline_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + // No longer need this since one shard will wait on all of them + map_applied_conditions.clear(); + } + else + Runtime::phase_barrier_arrive(inline_barrier, 1/*count*/); + if (is_owner_shard) + { + // Wait for all the other shards to be done mapping first + inline_barrier.wait(); + effects_done = + runtime->forest->overwrite_sharded(this, 0/*index*/, requirement, + sharded_view, version_info, trace_info, init_precondition, + map_applied_conditions, false/*restrict*/); + } + Runtime::advance_barrier(inline_barrier); + } + else + { + // Everyone pretends like they are readers and does their + // separate updates as though they were going to just read + const bool is_write = IS_WRITE(requirement); + if (is_write) + requirement.privilege = LEGION_READ_ONLY; // pretend read-only for now + UpdateAnalysis *analysis = NULL; + const RtEvent registration_precondition = + runtime->forest->physical_perform_updates(requirement, version_info, + this, 0/*index*/, init_precondition, termination_event, + mapped_instances, trace_info, map_applied_conditions, analysis, +#ifdef DEBUG_LEGION + get_logging_name(), unique_op_id, +#endif + // No need to track effects since we know it can't be + // restricted in a control replicated context + // Can't track initialized here because it might not be + // correct with our altered privileges + false/*track effects*/, record_valid/*record valid*/, + false/*check initialized*/, + // We can skip output for the same reason we don't + // need to track any effects + true/*defer copies*/, true/*skip output*/); + // If we're a write, then switch back privileges + if (is_write) + { + // In the read-write case we need to make sure everyone is done + // performing their updates before anyone does a registration + if (registration_precondition.exists()) + map_applied_conditions.insert(registration_precondition); + if (!map_applied_conditions.empty()) + { + Runtime::phase_barrier_arrive(inline_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + // Don't need these anymore since we're going to wait for them + map_applied_conditions.clear(); + } + else + Runtime::phase_barrier_arrive(inline_barrier, 1/*count*/); + // Set the privilege back to read-write + requirement.privilege = LEGION_READ_WRITE; + // Reset the usage of the analysis too + analysis->usage = RegionUsage(requirement); + // Wait for everyone to finish their updates + inline_barrier.wait(); + // Advance the barrier to the next generation + Runtime::advance_barrier(inline_barrier); + } + else + { + // In the read-only case we just need to wait for our registration + // to be done before we can proceed + if (registration_precondition.exists() && + !registration_precondition.has_triggered()) + registration_precondition.wait(); + } + // Then do the registration, no need to track output effects since we + // know that this instance can't be restricted in a control + // replicated context + runtime->forest->physical_perform_registration(analysis, + mapped_instances, trace_info, map_applied_conditions); + // If we have a write then we make a sharded view and + // then shard 0 will do the overwrite + if (is_write) + { +#ifdef DEBUG_LEGION + assert(sharded_view != NULL); + assert(exchange != NULL); +#endif + // We need to fill in the sharded view before we do the next + // call in case there are output effects due to restriction + // Note this has to be done across all the shards in case + // the restricted copies go remote + exchange->complete_exchange(this, sharded_view, + mapped_instances, map_applied_conditions); + // We need everyone to be done mapping before we can do the overwrite + if (!map_applied_conditions.empty()) + { + Runtime::phase_barrier_arrive(inline_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + // No longer need this since one shard will wait on all of them + map_applied_conditions.clear(); + } + else + Runtime::phase_barrier_arrive(inline_barrier, 1/*count*/); + if (is_owner_shard) + { + // Wait for all the other shards to be done mapping first + inline_barrier.wait(); + // Now we can do the replacement + effects_done = + runtime->forest->overwrite_sharded(this, 0/*index*/, requirement, + sharded_view, version_info, trace_info, init_precondition, + map_applied_conditions, false/*restrict*/); + } + Runtime::advance_barrier(inline_barrier); + } + } +#ifdef DEBUG_LEGION + if (!IS_NO_ACCESS(requirement) && !requirement.privilege_fields.empty()) + { + assert(!mapped_instances.empty()); + dump_physical_state(&requirement, 0); + } +#endif + // Update our physical instance with the newly mapped instances + // Have to do this before triggering the mapped event + if (effects_done.exists()) + { + region.impl->reset_references(mapped_instances, termination_event, + Runtime::merge_events(&trace_info, init_precondition, effects_done)); + } + else + region.impl->reset_references(mapped_instances, termination_event, + init_precondition); + ApEvent map_complete_event = ApEvent::NO_AP_EVENT; + if (mapped_instances.size() > 1) + { + std::set mapped_events; + for (unsigned idx = 0; idx < mapped_instances.size(); idx++) + mapped_events.insert(mapped_instances[idx].get_ready_event()); + map_complete_event = Runtime::merge_events(&trace_info, mapped_events); + } + else if (!mapped_instances.empty()) + map_complete_event = mapped_instances[0].get_ready_event(); + if (runtime->legion_spy_enabled) + { + runtime->forest->log_mapping_decision(unique_op_id, parent_ctx, + 0/*idx*/, requirement, + mapped_instances); +#ifdef LEGION_SPY + LegionSpy::log_operation_events(unique_op_id, map_complete_event, + termination_event); +#endif + } + // See if we have any reservations to take as part of this map + if (!atomic_locks.empty() || !arrive_barriers.empty()) + { + if (!effects_done.exists()) + effects_done = + Runtime::merge_events(&trace_info, effects_done, termination_event); + else + effects_done = termination_event; + // They've already been sorted in order + for (std::map::const_iterator it = + atomic_locks.begin(); it != atomic_locks.end(); it++) + { + map_complete_event = + Runtime::acquire_ap_reservation(it->first, it->second, + map_complete_event); + // We can also issue the release condition on our termination + Runtime::release_reservation(it->first, effects_done); + } + for (std::vector::iterator it = + arrive_barriers.begin(); it != arrive_barriers.end(); it++) + { + if (runtime->legion_spy_enabled) + LegionSpy::log_phase_barrier_arrival(unique_op_id, + it->phase_barrier); + Runtime::phase_barrier_arrive(it->phase_barrier, 1/*count*/, + effects_done); + } + } + // Remove profiling our guard and trigger the profiling event if necessary + if ((__sync_add_and_fetch(&outstanding_profiling_requests, -1) == 0) && + profiling_reported.exists()) + Runtime::trigger_event(profiling_reported); + // Now we can trigger the mapping event and indicate + // to all our mapping dependences that we are mapped. + RtEvent mapping_applied; + if (!map_applied_conditions.empty()) + mapping_applied = Runtime::merge_events(map_applied_conditions); + if (!acquired_instances.empty()) + mapping_applied = release_nonempty_acquired_instances(mapping_applied, + acquired_instances); + complete_mapping(complete_inline_mapping(mapping_applied)); + if (!map_complete_event.has_triggered()) + { + // Issue a deferred trigger on our completion event + // and mark that we are no longer responsible for + // triggering our completion event + request_early_complete(map_complete_event); + DeferredExecuteArgs deferred_execute_args(this); + runtime->issue_runtime_meta_task(deferred_execute_args, + LG_THROUGHPUT_DEFERRED_PRIORITY, + Runtime::protect_event(map_complete_event)); + } + else + deferred_execute(); + } + + //-------------------------------------------------------------------------- + void ReplMapOp::activate(void) + //-------------------------------------------------------------------------- + { + MapOp::activate(); + exchange = NULL; + view_did_broadcast = NULL; + sharded_view = NULL; + } + + //-------------------------------------------------------------------------- + void ReplMapOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_map_op(); + if (exchange != NULL) + delete exchange; + if (view_did_broadcast != NULL) + delete view_did_broadcast; + runtime->free_repl_map_op(this); + } + + //-------------------------------------------------------------------------- + RtEvent ReplMapOp::complete_inline_mapping(RtEvent mapping_applied) + //-------------------------------------------------------------------------- + { + Runtime::phase_barrier_arrive(inline_barrier, 1/*count*/,mapping_applied); + return inline_barrier; + } + + ///////////////////////////////////////////////////////////// + // Repl Attach Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplAttachOp::ReplAttachOp(Runtime *rt) + : AttachOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplAttachOp::ReplAttachOp(const ReplAttachOp &rhs) + : AttachOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplAttachOp::~ReplAttachOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplAttachOp& ReplAttachOp::operator=(const ReplAttachOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplAttachOp::initialize_replication(ReplicateContext *ctx, + RtBarrier &resource_bar, + ApBarrier &broadcast_bar, + ApBarrier &reduce_bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(resource_bar.exists()); + assert(exchange == NULL); + assert(did_broadcast == NULL); + assert(sharded_view == NULL); +#endif + resource_barrier = resource_bar; + ctx->advance_replicate_barrier(resource_bar, ctx->total_shards); + broadcast_barrier = broadcast_bar; + ctx->advance_replicate_barrier(broadcast_bar, 1/*arrivals*/); + // No matter what we're going to need a view broadcast either to make + // an instance which everyone has the name of or a sharded view + did_broadcast = + new ValueBroadcast(ctx, 0/*owner*/, COLLECTIVE_LOC_77); + if ((resource == LEGION_EXTERNAL_INSTANCE) || local_files) + { + // In this case we need a second generation of the resource_bar + ctx->advance_replicate_barrier(resource_bar, ctx->total_shards); + exchange = new ShardedMappingExchange(COLLECTIVE_LOC_78, ctx, + ctx->owner_shard->shard_id, false/*perform checks*/); + + // if we're shard 0 then get the distributed id and send it out + if (ctx->owner_shard->shard_id == 0) + { + DistributedID view_did = runtime->get_available_distributed_id(); + // make it and register it with the runtime + sharded_view = new ShardedView(runtime->forest, + view_did, runtime->address_space, true/*register now*/); + // then broadcast the result out so the other nodes can grab it + did_broadcast->broadcast(sharded_view->did); + } + } + else + { + reduce_barrier = reduce_bar; + ctx->advance_replicate_barrier(reduce_bar, ctx->total_shards); + } + } + + //-------------------------------------------------------------------------- + void ReplAttachOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_attach_op(); + resource_barrier = RtBarrier::NO_RT_BARRIER; + repl_mapping_applied = RtUserEvent::NO_RT_USER_EVENT; + exchange = NULL; + did_broadcast = NULL; + sharded_view = NULL; + all_mapped_event = RtEvent::NO_RT_EVENT; + exchange_complete = false; + } + + //-------------------------------------------------------------------------- + void ReplAttachOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_attach_op(); + if (exchange != NULL) + delete exchange; + if (did_broadcast != NULL) + delete did_broadcast; + runtime->free_repl_attach_op(this); + } + + //-------------------------------------------------------------------------- + void ReplAttachOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { + std::set preconditions; +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + const bool owner_shard = (repl_ctx->owner_shard->shard_id == 0); + if (!owner_shard) + { + if ((resource == LEGION_EXTERNAL_INSTANCE) || local_files) + { + // Get the distributed ID for the sharded view and request it + const DistributedID sharded_did = did_broadcast->get_value(); + RtEvent ready; + sharded_view = static_cast( + runtime->find_or_request_logical_view(sharded_did, ready)); + if (ready.exists()) + preconditions.insert(ready); + } + } + else // Only need the version info on the owner node + runtime->forest->perform_versioning_analysis(this, 0/*idx*/, + requirement, + version_info, + preconditions); + if (!preconditions.empty()) + enqueue_ready_operation(Runtime::merge_events(preconditions)); + else + enqueue_ready_operation(); + } + + //-------------------------------------------------------------------------- + void ReplAttachOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + const bool is_owner_shard = (repl_ctx->owner_shard->shard_id == 0); + if ((resource == LEGION_EXTERNAL_INSTANCE) || local_files) + { +#ifdef DEBUG_LEGION + assert(!restricted); + assert(exchange != NULL); + assert(sharded_view != NULL); +#endif + switch (resource) + { + case LEGION_EXTERNAL_POSIX_FILE: + case LEGION_EXTERNAL_HDF5_FILE: + { + external_instance = + runtime->forest->create_external_instance(this, requirement, + requirement.instance_fields); + break; + } + case LEGION_EXTERNAL_INSTANCE: + { + external_instance = + runtime->forest->create_external_instance(this, requirement, + layout_constraint_set.field_constraint.field_set); + break; + } + default: + assert(false); + } + InstanceSet attach_instances(1); + attach_instances[0] = external_instance; + InnerContext *context = find_physical_context(0/*index*/, requirement); + std::vector attach_views; + context->convert_target_views(attach_instances, attach_views); + exchange->initiate_exchange(attach_instances, attach_views); + // Once we're ready to map we can tell the memory manager that + // this instance can be safely acquired for use + IndividualManager *external_manager = + external_instance.get_instance_manager()->as_individual_manager(); + MemoryManager *memory_manager = external_manager->memory_manager; + memory_manager->attach_external_instance(external_manager); + RegionNode *node = runtime->forest->get_node(requirement.region); + ApUserEvent termination_event; + if (mapping) + termination_event = Runtime::create_ap_user_event(NULL); + const PhysicalTraceInfo trace_info(this, 0/*idx*/, true/*init*/); + UpdateAnalysis *analysis = new UpdateAnalysis(runtime, this, 0/*index*/, + version_info, requirement, node, attach_instances, attach_views, + trace_info, ApEvent::NO_AP_EVENT, mapping ? termination_event : + completion_event, false/*track effects*/, + false/*check initialized*/, true/*record valid*/,true/*skip output*/); + analysis->add_reference(); + // Have each operation do its own registration + // Note this will clean up the analysis allocation above + runtime->forest->physical_perform_registration(analysis, + attach_instances, trace_info, map_applied_conditions); + exchange->complete_exchange(this, sharded_view, + attach_instances, map_applied_conditions); + // Make sure all these are done before we do the overwrite + if (!map_applied_conditions.empty()) + { + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + // No longer need this since one shard will wait on all of them + map_applied_conditions.clear(); + } + else + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/); + if (is_owner_shard) + { + // Wait for all the other shards to be done mapping first + resource_barrier.wait(); + // Now we can do the replacement + const ApEvent attach_event = + runtime->forest->overwrite_sharded(this, 0/*index*/, requirement, + sharded_view, version_info, trace_info, + ApEvent::NO_AP_EVENT, map_applied_conditions, restricted); + Runtime::phase_barrier_arrive(broadcast_barrier, 1/*count*/, + attach_event); + } + Runtime::advance_barrier(resource_barrier); +#ifdef DEBUG_LEGION + assert(external_instance.has_ref()); +#endif + // This operation is ready once the file is attached + if (mapping) + { + attach_instances[0].set_ready_event(broadcast_barrier); + region.impl->reset_references(attach_instances, termination_event, + broadcast_barrier); + } + else + region.impl->set_reference(external_instance); + // Also set the sharded view in this case + region.impl->set_sharded_view(sharded_view); + // Make sure that all the attach operations are done mapping + // before we consider this attach operation done + if (!map_applied_conditions.empty()) + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + else + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/); + complete_mapping(resource_barrier); + request_early_complete(broadcast_barrier); + complete_execution(Runtime::protect_event(broadcast_barrier)); + } + else + { + ApUserEvent termination_event; + if (mapping) + { + termination_event = Runtime::create_ap_user_event(NULL); + Runtime::phase_barrier_arrive(reduce_barrier, 1/*count*/, + termination_event); + } + if (is_owner_shard) + { + // Make our instance now and send out the DID + switch (resource) + { + case LEGION_EXTERNAL_POSIX_FILE: + case LEGION_EXTERNAL_HDF5_FILE: + { + external_instance = + runtime->forest->create_external_instance(this, requirement, + requirement.instance_fields); + break; + } + // No external instances here by definition + default: + assert(false); + } + + InstanceSet attach_instances(1); + attach_instances[0] = external_instance; + // Once we're ready to map we can tell the memory manager that + // this instance can be safely acquired for use + IndividualManager *external_manager = + external_instance.get_instance_manager()->as_individual_manager(); + MemoryManager *memory_manager = external_manager->memory_manager; + memory_manager->attach_external_instance(external_manager); + // We can't broadcast the DID until after doing the attach + // to the memory in case we update the reference state + did_broadcast->broadcast(external_instance.get_manager()->did); + const PhysicalTraceInfo trace_info(this, 0/*idx*/, true/*init*/); + InnerContext *context = find_physical_context(0/*index*/,requirement); + std::vector attach_views; + context->convert_target_views(attach_instances, attach_views); +#ifdef DEBUG_LEGION + assert(attach_views.size() == 1); +#endif + ApEvent attach_event = runtime->forest->attach_external(this,0/*idx*/, + requirement, + attach_views[0], + attach_views[0], + mapping ? + (ApEvent)reduce_barrier + : completion_event, + version_info, + trace_info, + map_applied_conditions, + restricted); +#ifdef DEBUG_LEGION + assert(external_instance.has_ref()); +#endif + Runtime::phase_barrier_arrive(broadcast_barrier, 1/*count*/, + attach_event); + // Save the instance information out to region + if (mapping) + { + attach_instances[0].set_ready_event(broadcast_barrier); + region.impl->reset_references(attach_instances, termination_event, + broadcast_barrier); + } + else + region.impl->set_reference(external_instance); + // This operation is ready once the file is attached + // Make sure that all the attach operations are done mapping + // before we consider this attach operation done + if (!map_applied_conditions.empty()) + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + else + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/); + complete_mapping(resource_barrier); + request_early_complete(broadcast_barrier); + complete_execution(Runtime::protect_event(broadcast_barrier)); + } + else + { + FieldSpaceNode *node = + runtime->forest->get_node(requirement.region.get_field_space()); + FieldMask instance_fields = + node->get_field_mask(requirement.privilege_fields); + // Get the DID for the common manager and request it + DistributedID manager_did = did_broadcast->get_value(); + RtEvent ready; + PhysicalManager *manager = + runtime->find_or_request_instance_manager(manager_did, ready); + // Wait for the manager to be ready + if (ready.exists()) + ready.wait(); + external_instance = InstanceRef(manager, instance_fields); + // Save the instance information out to region + if (mapping) + { + InstanceSet attach_instances(1); + attach_instances[0] = external_instance; + attach_instances[0].set_ready_event(broadcast_barrier); + region.impl->reset_references(attach_instances, termination_event, + broadcast_barrier); + } + else + region.impl->set_reference(external_instance); + // Record that we're mapped once everyone else does + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/); + complete_mapping(resource_barrier); + complete_execution(Runtime::protect_event(broadcast_barrier)); + } + } + } + + ///////////////////////////////////////////////////////////// + // Repl Detach Op + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplDetachOp::ReplDetachOp(Runtime *rt) + : DetachOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplDetachOp::ReplDetachOp(const ReplDetachOp &rhs) + : DetachOp(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplDetachOp::~ReplDetachOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplDetachOp& ReplDetachOp::operator=(const ReplDetachOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplDetachOp::initialize_replication(ReplicateContext *ctx, + RtBarrier &resource_bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(resource_bar.exists()); +#endif + resource_barrier = resource_bar; + ctx->advance_replicate_barrier(resource_bar, ctx->total_shards); + } + + //-------------------------------------------------------------------------- + void ReplDetachOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_detach_op(); + resource_barrier = RtBarrier::NO_RT_BARRIER; + } + + //-------------------------------------------------------------------------- + void ReplDetachOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_detach_op(); + runtime->free_repl_detach_op(this); + } + + //-------------------------------------------------------------------------- + void ReplDetachOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { + std::set preconditions; + runtime->forest->perform_versioning_analysis(this, 0/*idx*/, + requirement, + version_info, + preconditions); + if (!preconditions.empty()) + enqueue_ready_operation(Runtime::merge_events(preconditions)); + else + enqueue_ready_operation(); + } + + //-------------------------------------------------------------------------- + void ReplDetachOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + const bool is_owner_shard = (repl_ctx->owner_shard->shard_id == 0); + const PhysicalTraceInfo trace_info(this, 0/*index*/, true/*init*/); + // Actual unmap of an inline mapped region was deferred to here + if (region.impl->is_mapped()) + region.impl->unmap_region(); + // Now we can get the reference we need for the detach operation + InstanceSet references; + region.impl->get_references(references); +#ifdef DEBUG_LEGION + assert(references.size() == 1); +#endif + InstanceRef reference = references[0]; + // Check that this is actually a file + PhysicalManager *manager = reference.get_instance_manager(); +#ifdef DEBUG_LEGION + assert(!manager->is_reduction_manager()); +#endif + ShardedView *sharded_view = region.impl->get_sharded_view(); + ApEvent detach_event; + if ((sharded_view != NULL) || (is_owner_shard)) + { + // Everybody does registration and filtering in the case + // where there is a sharded view because there are different + // instances for each shard + // Only the owner does it in the case where there isn't a + // sharded view because there is only one instance for all shards + InnerContext *context = find_physical_context(0/*index*/, requirement); + std::vector inst_views; + context->convert_target_views(references, inst_views); + detach_event = runtime->forest->detach_external(requirement, + this, 0/*index*/, version_info, inst_views[0], + trace_info, map_applied_conditions, sharded_view); + // Also tell the runtime to detach the external instance from memory + // This has to be done before we can consider this mapped + RtEvent detached_event = manager->detach_external_instance(); + if (detached_event.exists()) + map_applied_conditions.insert(detached_event); + if (runtime->legion_spy_enabled) + { + runtime->forest->log_mapping_decision(unique_op_id, parent_ctx, + 0/*idx*/, requirement, references); +#ifdef LEGION_SPY + LegionSpy::log_operation_events(unique_op_id, detach_event, + completion_event); +#endif + } + } + // Make sure that all the detach operations are done before + // we count any of them as being mapped + if (!map_applied_conditions.empty()) + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/, + Runtime::merge_events(map_applied_conditions)); + else + Runtime::phase_barrier_arrive(resource_barrier, 1/*count*/); + complete_mapping(resource_barrier); + + request_early_complete(detach_event); + complete_execution(Runtime::protect_event(detach_event)); + } + + //-------------------------------------------------------------------------- + void ReplDetachOp::select_sources(const unsigned index, + const InstanceRef &target, + const InstanceSet &sources, + std::vector &ranking) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(index == 0); +#endif + // Pick any instances other than external ones + std::vector remote_ranking; + for (unsigned idx = 0; idx < sources.size(); idx++) + { + const InstanceRef &ref = sources[idx]; + PhysicalManager *manager = ref.get_instance_manager(); + if (manager->is_external_instance()) + continue; + if (manager->owner_space == runtime->address_space) + ranking.push_back(idx); + else + remote_ranking.push_back(idx); + } + if (!remote_ranking.empty()) + ranking.insert(ranking.end(), + remote_ranking.begin(), remote_ranking.end()); + } + + //-------------------------------------------------------------------------- + void ReplDetachOp::record_unordered_kind( + std::map,ReplDetachOp*> &detachments) + //-------------------------------------------------------------------------- + { + const RegionRequirement &req = region.impl->get_requirement(); +#ifdef DEBUG_LEGION + assert(!req.privilege_fields.empty()); +#endif + const std::pair key(req.region, + *(req.privilege_fields.begin())); +#ifdef DEBUG_LEGION + assert(detachments.find(key) == detachments.end()); +#endif + detachments[key] = this; + } + + ///////////////////////////////////////////////////////////// + // ReplTraceOp + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplTraceOp::ReplTraceOp(Runtime *rt) + : ReplFenceOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceOp::ReplTraceOp(const ReplTraceOp &rhs) + : ReplFenceOp(NULL) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplTraceOp::~ReplTraceOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceOp& ReplTraceOp::operator=(const ReplTraceOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplTraceOp::execute_dependence_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(mapping_tracker == NULL); +#endif + // Make a dependence tracker + mapping_tracker = new MappingDependenceTracker(); + // See if we have any fence dependences + execution_fence_event = parent_ctx->register_implicit_dependences(this); + parent_ctx->invalidate_trace_cache(local_trace, this); + + trigger_dependence_analysis(); + end_dependence_analysis(); + } + + //-------------------------------------------------------------------------- + void ReplTraceOp::sync_for_replayable_check(void) + //-------------------------------------------------------------------------- + { + // Should only be called by derived classes + assert(false); + } + + //-------------------------------------------------------------------------- + bool ReplTraceOp::exchange_replayable(ReplicateContext *ctx,bool replayable) + //-------------------------------------------------------------------------- + { + // Should only be called by derived classes + assert(false); + return false; + } + + //-------------------------------------------------------------------------- + void ReplTraceOp::elide_fences_pre_sync(void) + //-------------------------------------------------------------------------- + { + // Should only be called by derived classes + assert(false); + } + + //-------------------------------------------------------------------------- + void ReplTraceOp::elide_fences_post_sync(void) + //-------------------------------------------------------------------------- + { + // Should only be called by derived classes + assert(false); + } + + ///////////////////////////////////////////////////////////// + // ReplTraceCaptureOp + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplTraceCaptureOp::ReplTraceCaptureOp(Runtime *rt) + : ReplTraceOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceCaptureOp::ReplTraceCaptureOp(const ReplTraceCaptureOp &rhs) + : ReplTraceOp(NULL) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplTraceCaptureOp::~ReplTraceCaptureOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceCaptureOp& ReplTraceCaptureOp::operator=( + const ReplTraceCaptureOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplTraceCaptureOp::initialize_capture(ReplicateContext *ctx, + bool has_block, bool remove_trace_ref) + //-------------------------------------------------------------------------- + { + initialize_repl_fence(ctx, EXECUTION_FENCE, false/*need future*/); +#ifdef DEBUG_LEGION + assert(trace != NULL); +#endif + local_trace = trace; + // Now mark our trace as NULL to avoid registering this operation + trace = NULL; + tracing = false; + current_template = NULL; + has_blocking_call = has_block; + remove_trace_reference = remove_trace_ref; + // Get a collective ID to use for check all replayable + replayable_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_85); + replay_sync_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_91); + pre_elide_fences_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_92); + post_elide_fences_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_93); + } + + //-------------------------------------------------------------------------- + void ReplTraceCaptureOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_operation(); + current_template = NULL; + replayable_collective_id = 0; + has_blocking_call = false; + remove_trace_reference = false; + } + + //-------------------------------------------------------------------------- + void ReplTraceCaptureOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_operation(); + runtime->free_repl_capture_op(this); + } + + //-------------------------------------------------------------------------- + const char* ReplTraceCaptureOp::get_logging_name(void) const + //-------------------------------------------------------------------------- + { + return op_names[TRACE_CAPTURE_OP_KIND]; + } + + //-------------------------------------------------------------------------- + Operation::OpKind ReplTraceCaptureOp::get_operation_kind(void) const + //-------------------------------------------------------------------------- + { + return TRACE_CAPTURE_OP_KIND; + } + + //-------------------------------------------------------------------------- + void ReplTraceCaptureOp::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(trace == NULL); + assert(local_trace != NULL); +#endif + // Indicate that we are done capturing this trace + local_trace->end_trace_capture(); + // Register this fence with all previous users in the parent's context + ReplFenceOp::trigger_dependence_analysis(); + parent_ctx->record_previous_trace(local_trace); + if (local_trace->is_recording()) + { + PhysicalTrace *physical_trace = local_trace->get_physical_trace(); +#ifdef DEBUG_LEGION + assert(physical_trace != NULL); +#endif + current_template = physical_trace->get_current_template(); + physical_trace->record_previous_template_completion( + get_completion_event()); + physical_trace->clear_cached_template(); + } + } + + //-------------------------------------------------------------------------- + void ReplTraceCaptureOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { + // Now finish capturing the physical trace + if (local_trace->is_recording()) + { + PhysicalTrace *physical_trace = local_trace->get_physical_trace(); +#ifdef DEBUG_LEGION + assert(physical_trace != NULL); + assert(current_template != NULL); + assert(local_trace->get_physical_trace() != NULL); + assert(current_template->is_recording()); +#endif + current_template->finalize(has_blocking_call, this); + if (!current_template->is_replayable()) + { + const RtEvent pending_deletion = + current_template->defer_template_deletion(); + if (pending_deletion.exists()) + execution_precondition = Runtime::merge_events(NULL, + execution_precondition, ApEvent(pending_deletion)); + physical_trace->record_failed_capture(current_template); + } + else + physical_trace->record_replayable_capture(current_template); + // Reset the local trace + local_trace->initialize_tracing_state(); + } + if (remove_trace_reference && local_trace->remove_reference()) + delete local_trace; + ReplFenceOp::trigger_mapping(); + } + + //-------------------------------------------------------------------------- + void ReplTraceCaptureOp::sync_for_replayable_check(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + SlowBarrier replay_sync_barrier(repl_ctx, replay_sync_collective_id); + replay_sync_barrier.perform_collective_sync(); + } + + //-------------------------------------------------------------------------- + bool ReplTraceCaptureOp::exchange_replayable(ReplicateContext *repl_ctx, + bool shard_replayable) + //-------------------------------------------------------------------------- + { + // Check to see if this template is replayable across all the shards + AllReduceCollective > + all_replayable_collective(repl_ctx, replayable_collective_id); + return all_replayable_collective.sync_all_reduce(shard_replayable); + } + + //-------------------------------------------------------------------------- + void ReplTraceCaptureOp::elide_fences_pre_sync(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + SlowBarrier pre_sync_barrier(repl_ctx, pre_elide_fences_collective_id); + pre_sync_barrier.perform_collective_sync(); + } + + //-------------------------------------------------------------------------- + void ReplTraceCaptureOp::elide_fences_post_sync(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + SlowBarrier post_sync_barrier(repl_ctx, post_elide_fences_collective_id); + post_sync_barrier.perform_collective_sync(); + } + + ///////////////////////////////////////////////////////////// + // ReplTraceCompleteOp + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplTraceCompleteOp::ReplTraceCompleteOp(Runtime *rt) + : ReplTraceOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceCompleteOp::ReplTraceCompleteOp(const ReplTraceCompleteOp &rhs) + : ReplTraceOp(NULL) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplTraceCompleteOp::~ReplTraceCompleteOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceCompleteOp& ReplTraceCompleteOp::operator=( + const ReplTraceCompleteOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplTraceCompleteOp::initialize_complete(ReplicateContext *ctx, + bool has_block) + //-------------------------------------------------------------------------- + { + initialize_repl_fence(ctx, EXECUTION_FENCE, false/*need future*/); +#ifdef DEBUG_LEGION + assert(trace != NULL); +#endif + local_trace = trace; + // Now mark our trace as NULL to avoid registering this operation + trace = NULL; + tracing = false; + current_template = NULL; + template_completion = ApEvent::NO_AP_EVENT; + replayed = false; + has_blocking_call = has_block; + // Get a collective ID to use for check all replayable + replayable_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_86); + replay_sync_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_91); + pre_elide_fences_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_92); + post_elide_fences_collective_id = + ctx->get_next_collective_index(COLLECTIVE_LOC_93); + } + + //-------------------------------------------------------------------------- + void ReplTraceCompleteOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_operation(); + current_template = NULL; + template_completion = ApEvent::NO_AP_EVENT; + replayable_collective_id = 0; + replayed = false; + has_blocking_call = false; + } + + //-------------------------------------------------------------------------- + void ReplTraceCompleteOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_operation(); + runtime->free_repl_trace_op(this); + } + + //-------------------------------------------------------------------------- + const char* ReplTraceCompleteOp::get_logging_name(void) const + //-------------------------------------------------------------------------- + { + return op_names[TRACE_COMPLETE_OP_KIND]; + } + + //-------------------------------------------------------------------------- + Operation::OpKind ReplTraceCompleteOp::get_operation_kind(void) const + //-------------------------------------------------------------------------- + { + return TRACE_COMPLETE_OP_KIND; + } + + //-------------------------------------------------------------------------- + void ReplTraceCompleteOp::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(trace == NULL); + assert(local_trace != NULL); +#endif + // Indicate that this trace is done being captured + // This also registers that we have dependences on all operations + // in the trace. + local_trace->end_trace_execution(this); + parent_ctx->record_previous_trace(local_trace); + + if (local_trace->is_replaying()) + { + PhysicalTrace *physical_trace = local_trace->get_physical_trace(); +#ifdef DEBUG_LEGION + assert(physical_trace != NULL); +#endif + PhysicalTemplate *current_template = + physical_trace->get_current_template(); +#ifdef DEBUG_LEGION + assert(current_template != NULL); +#endif +#ifdef LEGION_SPY + local_trace->perform_logging( + current_template->get_fence_uid(), unique_op_id); +#endif + current_template->execute_all(); + template_completion = current_template->get_completion(); + // Trigger the execution fence barrier with this event + Runtime::phase_barrier_arrive(execution_fence_barrier, 1/*count*/, + template_completion); + need_completion_trigger = false; + Runtime::trigger_event(NULL, completion_event, execution_fence_barrier); + local_trace->end_trace_execution(this); + parent_ctx->update_current_fence(this, true, true); + parent_ctx->record_previous_trace(local_trace); + physical_trace->record_previous_template_completion( + execution_fence_barrier); + local_trace->initialize_tracing_state(); + replayed = true; + return; + } + else if (local_trace->is_recording()) + { + PhysicalTrace *physical_trace = local_trace->get_physical_trace(); +#ifdef DEBUG_LEGION + assert(physical_trace != NULL); +#endif + current_template = physical_trace->get_current_template(); + physical_trace->record_previous_template_completion( + get_completion_event()); + physical_trace->clear_cached_template(); + } + + // If this is a static trace, then we remove our reference when we're done + if (local_trace->is_static_trace()) + { + StaticTrace *static_trace = static_cast(local_trace); + if (static_trace->remove_reference()) + delete static_trace; + } + ReplFenceOp::trigger_dependence_analysis(); + } + + //-------------------------------------------------------------------------- + void ReplTraceCompleteOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { + // Now finish capturing the physical trace + if (local_trace->is_recording()) + { + PhysicalTrace *physical_trace = local_trace->get_physical_trace(); +#ifdef DEBUG_LEGION + assert(physical_trace != NULL); + assert(current_template != NULL); + assert(local_trace->get_physical_trace() != NULL); + assert(current_template->is_recording()); +#endif + current_template->finalize(has_blocking_call, this); + if (!current_template->is_replayable()) + { + const RtEvent pending_deletion = + current_template->defer_template_deletion(); + if (pending_deletion.exists()) + execution_precondition = Runtime::merge_events(NULL, + execution_precondition, ApEvent(pending_deletion)); + physical_trace->record_failed_capture(current_template); + } + else + physical_trace->record_replayable_capture(current_template); + local_trace->initialize_tracing_state(); + } + else if (replayed) + { + if (has_blocking_call) + REPORT_LEGION_ERROR(ERROR_INVALID_PHYSICAL_TRACING, + "Physical tracing violation! Trace %d in task %s (UID %lld) " + "encountered a blocking API call that was unseen when it was " + "recorded. It is required that traces do not change their " + "behavior.", local_trace->get_trace_id(), + parent_ctx->get_task_name(), parent_ctx->get_unique_id()) + // Do our arrival on the mapping fence + Runtime::phase_barrier_arrive(mapping_fence_barrier, 1/*count*/); + complete_mapping(mapping_fence_barrier); + if (!execution_fence_barrier.has_triggered()) + { + RtEvent wait_on = Runtime::protect_event(execution_fence_barrier); + complete_execution(wait_on); + } + else + complete_execution(); + return; + } + ReplFenceOp::trigger_mapping(); + } + + //-------------------------------------------------------------------------- + void ReplTraceCompleteOp::sync_for_replayable_check(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + SlowBarrier replay_sync_barrier(repl_ctx, replay_sync_collective_id); + replay_sync_barrier.perform_collective_sync(); + } + + //-------------------------------------------------------------------------- + bool ReplTraceCompleteOp::exchange_replayable(ReplicateContext *repl_ctx, + bool shard_replayable) + //-------------------------------------------------------------------------- + { + // Check to see if this template is replayable across all the shards + AllReduceCollective > + all_replayable_collective(repl_ctx, replayable_collective_id); + return all_replayable_collective.sync_all_reduce(shard_replayable); + } + + //-------------------------------------------------------------------------- + void ReplTraceCompleteOp::elide_fences_pre_sync(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + SlowBarrier pre_sync_barrier(repl_ctx, pre_elide_fences_collective_id); + pre_sync_barrier.perform_collective_sync(); + } + + //-------------------------------------------------------------------------- + void ReplTraceCompleteOp::elide_fences_post_sync(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + SlowBarrier post_sync_barrier(repl_ctx, post_elide_fences_collective_id); + post_sync_barrier.perform_collective_sync(); + } + + ///////////////////////////////////////////////////////////// + // ReplTraceReplayOp + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplTraceReplayOp::ReplTraceReplayOp(Runtime *rt) + : ReplTraceOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceReplayOp::ReplTraceReplayOp(const ReplTraceReplayOp &rhs) + : ReplTraceOp(NULL) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplTraceReplayOp::~ReplTraceReplayOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceReplayOp& ReplTraceReplayOp::operator=( + const ReplTraceReplayOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplTraceReplayOp::initialize_replay(ReplicateContext *ctx, + LegionTrace *trace) + //-------------------------------------------------------------------------- + { + initialize_repl_fence(ctx, EXECUTION_FENCE, false/*need future*/); +#ifdef DEBUG_LEGION + assert(trace != NULL); +#endif + local_trace = trace; + for (int idx = 0; idx < TRACE_SELECTION_ROUNDS; idx++) + trace_selection_collective_ids[idx] = + ctx->get_next_collective_index(COLLECTIVE_LOC_87); + } + + //-------------------------------------------------------------------------- + void ReplTraceReplayOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_operation(); + } + + //-------------------------------------------------------------------------- + void ReplTraceReplayOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_operation(); + runtime->free_repl_replay_op(this); + } + + //-------------------------------------------------------------------------- + const char* ReplTraceReplayOp::get_logging_name(void) const + //-------------------------------------------------------------------------- + { + return op_names[TRACE_REPLAY_OP_KIND]; + } + + //-------------------------------------------------------------------------- + Operation::OpKind ReplTraceReplayOp::get_operation_kind(void) const + //-------------------------------------------------------------------------- + { + return TRACE_REPLAY_OP_KIND; + } + + //-------------------------------------------------------------------------- + void ReplTraceReplayOp::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(trace == NULL); + assert(local_trace != NULL); +#endif + PhysicalTrace *physical_trace = local_trace->get_physical_trace(); +#ifdef DEBUG_LEGION + assert(physical_trace != NULL); +#endif + bool recurrent = true; + bool fence_registered = false; + bool is_recording = local_trace->is_recording(); + if ((physical_trace->get_current_template() == NULL) || is_recording) + { + recurrent = false; + { + // Wait for the previous recordings to be done before checking + // template preconditions, otherwise no template would exist. + RtEvent mapped_event = parent_ctx->get_current_mapping_fence_event(); + if (mapped_event.exists()) + mapped_event.wait(); + } +#ifdef DEBUG_LEGION + assert(!(local_trace->is_recording() || local_trace->is_replaying())); + ReplicateContext *repl_ctx =dynamic_cast(parent_ctx); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(parent_ctx); +#endif + + if (physical_trace->get_current_template() == NULL) + { + int selected_template_index = -2; + std::vector viable_templates; + for (int round = 0; round < TRACE_SELECTION_ROUNDS; round++) + { + // Exponential back-off: the more rounds we go, the + // more templates we try to find to build consensus + const unsigned number_to_find = 1 << round; + if ((viable_templates.empty() || (viable_templates.back() >= 0)) && + physical_trace->find_viable_templates(this, + map_applied_conditions, + number_to_find, + viable_templates)) + { + // If we checked all the templates figure out what kind of + // guard to add: + // Use -1 to indicate that we're done but have viable templates + // Use -2 to indicate we have no viable templates + if (!viable_templates.empty()) + viable_templates.push_back(-1); + else + viable_templates.push_back(-2); + } +#ifdef DEBUG_LEGION + assert(!viable_templates.empty()); +#endif + // Perform an exchange to see if we have consensus + TemplateIndexExchange index_exchange(repl_ctx, + trace_selection_collective_ids[round]); + index_exchange.initiate_exchange(viable_templates); + std::map result_templates; + index_exchange.complete_exchange(result_templates); + // First, if we have at least one shard that says that it + // has no viable templates then we're done + if (result_templates.find(-2) == result_templates.end()) + { + // Otherwise go through in reverse order and look for one that + // has consensus from all the shards + const size_t total_shards = repl_ctx->shard_manager->total_shards; + for (std::map::reverse_iterator rit = + result_templates.rbegin(); rit != + result_templates.rend(); rit++) + { +#ifdef DEBUG_LEGION + assert(rit->second <= total_shards); +#endif + // If we have a template that is viable for all the shards + // then we've succesffully identified a template to use + if (rit->second == total_shards) + { + // Note this could also be -1 in the case were all + // the shards have identified all their viable templates + selected_template_index = rit->first; + break; + } + } + } + else + selected_template_index = -1; + // If we picked an index then we're done + if (selected_template_index != -2) + break; + } + // If we successfully identified a template for all the shards + // to use then we record that in the trace + if (selected_template_index >= 0) + { + PhysicalTemplate *t = + physical_trace->select_template(selected_template_index); +#ifdef DEBUG_LEGION + ShardedPhysicalTemplate *tpl = + dynamic_cast(t); + assert(tpl != NULL); +#else + ShardedPhysicalTemplate *tpl = + static_cast(t); +#endif + tpl->record_replayed(); + } + } +#ifdef DEBUG_LEGION + assert(physical_trace->get_current_template() == NULL || + !physical_trace->get_current_template()->is_recording()); +#endif + execution_precondition = + parent_ctx->perform_fence_analysis(this, true, true); + physical_trace->set_current_execution_fence_event( + get_completion_event()); + fence_registered = true; + } + + if (physical_trace->get_current_template() != NULL) + { + if (!fence_registered) + execution_precondition = + parent_ctx->get_current_execution_fence_event(); + ApEvent fence_completion = recurrent ? + physical_trace->get_previous_template_completion() : + get_completion_event(); + physical_trace->initialize_template(fence_completion, recurrent); + local_trace->set_state_replay(); +#ifdef LEGION_SPY + physical_trace->get_current_template()->set_fence_uid(unique_op_id); +#endif + } + else if (!fence_registered) + { + execution_precondition = + parent_ctx->perform_fence_analysis(this, true, true); + physical_trace->set_current_execution_fence_event( + get_completion_event()); + } + + // Now update the parent context with this fence before we can complete + // the dependence analysis and possibly be deactivated + parent_ctx->update_current_fence(this, true, true); + } + + //-------------------------------------------------------------------------- + void ReplTraceReplayOp::pack_remote_operation(Serializer &rez, + AddressSpaceID target, std::set &applied_events) const + //-------------------------------------------------------------------------- + { + pack_local_remote_operation(rez); + } + + ///////////////////////////////////////////////////////////// + // ReplTraceBeginOp + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplTraceBeginOp::ReplTraceBeginOp(Runtime *rt) + : ReplTraceOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceBeginOp::ReplTraceBeginOp(const ReplTraceBeginOp &rhs) + : ReplTraceOp(NULL) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplTraceBeginOp::~ReplTraceBeginOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceBeginOp& ReplTraceBeginOp::operator=(const ReplTraceBeginOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplTraceBeginOp::initialize_begin(ReplicateContext *ctx, + LegionTrace *trace) + //-------------------------------------------------------------------------- + { + initialize_repl_fence(ctx, MAPPING_FENCE, false/*need future*/); +#ifdef DEBUG_LEGION + assert(trace != NULL); +#endif + local_trace = trace; + trace = NULL; + tracing = false; + } + + //-------------------------------------------------------------------------- + void ReplTraceBeginOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_operation(); + } + + //-------------------------------------------------------------------------- + void ReplTraceBeginOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_operation(); + runtime->free_repl_begin_op(this); + } + + //-------------------------------------------------------------------------- + const char* ReplTraceBeginOp::get_logging_name(void) const + //-------------------------------------------------------------------------- + { + return op_names[TRACE_BEGIN_OP_KIND]; + } + + //-------------------------------------------------------------------------- + Operation::OpKind ReplTraceBeginOp::get_operation_kind(void) const + //-------------------------------------------------------------------------- + { + return TRACE_BEGIN_OP_KIND; + } + + ///////////////////////////////////////////////////////////// + // ReplTraceSummaryOp + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ReplTraceSummaryOp::ReplTraceSummaryOp(Runtime *rt) + : ReplTraceOp(rt) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceSummaryOp::ReplTraceSummaryOp(const ReplTraceSummaryOp &rhs) + : ReplTraceOp(NULL) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ReplTraceSummaryOp::~ReplTraceSummaryOp(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ReplTraceSummaryOp& ReplTraceSummaryOp::operator=( + const ReplTraceSummaryOp &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ReplTraceSummaryOp::initialize_summary(ReplicateContext *ctx, + ShardedPhysicalTemplate *tpl, + Operation *invalidator) + //-------------------------------------------------------------------------- + { + // Do NOT call initialize_repl_fence here, we're in the dependence + // analysis stage of the pipeline and we need to get our mapping + // fence from a different location to avoid racing with the application + initialize(ctx, MAPPING_FENCE, false/*need future*/, false/*track*/); + mapping_fence_barrier = ctx->get_next_summary_fence_barrier(); + context_index = invalidator->get_ctx_index(); + current_template = tpl; + // The summary could have been marked as being traced, + // so here we forcibly clear them out. + trace = NULL; + tracing = false; + } + + //-------------------------------------------------------------------------- + void ReplTraceSummaryOp::activate(void) + //-------------------------------------------------------------------------- + { + activate_operation(); + current_template = NULL; + } + + //-------------------------------------------------------------------------- + void ReplTraceSummaryOp::deactivate(void) + //-------------------------------------------------------------------------- + { + deactivate_fence(); + runtime->free_repl_summary_op(this); + } + + //-------------------------------------------------------------------------- + const char* ReplTraceSummaryOp::get_logging_name(void) const + //-------------------------------------------------------------------------- + { + return op_names[TRACE_SUMMARY_OP_KIND]; + } + + //-------------------------------------------------------------------------- + Operation::OpKind ReplTraceSummaryOp::get_operation_kind(void) const + //-------------------------------------------------------------------------- + { + return TRACE_SUMMARY_OP_KIND; + } + + //-------------------------------------------------------------------------- + void ReplTraceSummaryOp::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { + perform_fence_analysis(true/*register fence also*/); + } + + //-------------------------------------------------------------------------- + void ReplTraceSummaryOp::trigger_ready(void) + //-------------------------------------------------------------------------- + { + enqueue_ready_operation(); + } + + //-------------------------------------------------------------------------- + void ReplTraceSummaryOp::trigger_mapping(void) + //-------------------------------------------------------------------------- + { + if (current_template->is_replayable()) + current_template->apply_postcondition(this, map_applied_conditions); + ReplFenceOp::trigger_mapping(); + } + + //-------------------------------------------------------------------------- + void ReplTraceSummaryOp::pack_remote_operation(Serializer &rez, + AddressSpaceID target, std::set &applied_events) const + //-------------------------------------------------------------------------- + { + pack_local_remote_operation(rez); + } + + ///////////////////////////////////////////////////////////// + // Shard Manager + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardMapping::ShardMapping(void) + : Collectable() + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardMapping::ShardMapping(const ShardMapping &rhs) + : Collectable() + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ShardMapping::ShardMapping(const std::vector &spaces) + : Collectable(), address_spaces(spaces) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardMapping::~ShardMapping(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardMapping& ShardMapping::operator=(const ShardMapping &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + AddressSpaceID ShardMapping::operator[](unsigned idx) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(idx < address_spaces.size()); +#endif + return address_spaces[idx]; + } + + //-------------------------------------------------------------------------- + AddressSpaceID& ShardMapping::operator[](unsigned idx) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(idx < address_spaces.size()); +#endif + return address_spaces[idx]; + } + + //-------------------------------------------------------------------------- + void ShardMapping::pack_mapping(Serializer &rez) const + //-------------------------------------------------------------------------- + { + rez.serialize(address_spaces.size()); + for (std::vector::const_iterator it = + address_spaces.begin(); it != address_spaces.end(); it++) + rez.serialize(*it); + } + + //-------------------------------------------------------------------------- + void ShardMapping::unpack_mapping(Deserializer &derez) + //-------------------------------------------------------------------------- + { + size_t num_spaces; + derez.deserialize(num_spaces); + address_spaces.resize(num_spaces); + for (unsigned idx = 0; idx < num_spaces; idx++) + derez.deserialize(address_spaces[idx]); + } + + ///////////////////////////////////////////////////////////// + // Shard Manager + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardManager::ShardManager(Runtime *rt, ReplicationID id, bool control, + bool top, size_t total, AddressSpaceID owner, + SingleTask *original/*= NULL*/, RtBarrier bar) + : runtime(rt), repl_id(id), owner_space(owner), total_shards(total), + original_task(original),control_replicated(control), + top_level_task(top), address_spaces(NULL), + local_mapping_complete(0), remote_mapping_complete(0), + local_execution_complete(0), remote_execution_complete(0), + trigger_local_complete(0), trigger_remote_complete(0), + trigger_local_commit(0), trigger_remote_commit(0), + remote_constituents(0), semantic_attach_counter(0), + local_future_result(NULL), local_future_size(0), + local_future_set(false), startup_barrier(bar) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(total_shards > 0); +#endif + // Add an extra reference if we're not the owner manager + if (owner_space != runtime->address_space) + add_reference(); + runtime->register_shard_manager(repl_id, this); + if (control_replicated && (owner_space == runtime->address_space)) + { +#ifdef DEBUG_LEGION + assert(!startup_barrier.exists()); +#endif + startup_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + pending_partition_barrier = + ApBarrier(Realm::Barrier::create_barrier(total_shards)); + // Only need shards-1 for arrivals here since it is used + // to signal from all the non-creator shards to the creator shard + creation_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + // Same thing as above for deletion barriers + deletion_ready_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + deletion_mapping_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + deletion_execution_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + // Inline mapping barrier for synchronizing inline mappings + // across all the shards + inline_mapping_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + // External resource barrier for synchronizing attach/detach ops + external_resource_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + // Fence barriers need arrivals from everyone + mapping_fence_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + trace_recording_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + summary_fence_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + execution_fence_barrier = + ApBarrier(Realm::Barrier::create_barrier(total_shards)); + attach_broadcast_barrier = + ApBarrier(Realm::Barrier::create_barrier(1)); + attach_reduce_barrier = + ApBarrier(Realm::Barrier::create_barrier(total_shards)); + dependent_partition_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + semantic_attach_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards)); + if (runtime->program_order_execution) + inorder_barrier = + ApBarrier(Realm::Barrier::create_barrier(total_shards)); + // callback barrier can't be made until we know how many + // unique address spaces we'll actually have so see + // ShardManager::launch +#ifdef DEBUG_LEGION_COLLECTIVES + collective_check_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards, + CollectiveCheckReduction::REDOP, + &CollectiveCheckReduction::IDENTITY, + sizeof(CollectiveCheckReduction::IDENTITY))); + close_check_barrier = + RtBarrier(Realm::Barrier::create_barrier(total_shards, + CloseCheckReduction::REDOP, + &CloseCheckReduction::IDENTITY, + sizeof(CloseCheckReduction::IDENTITY))); +#endif + } +#ifdef DEBUG_LEGION + else if (control_replicated) + assert(startup_barrier.exists()); +#endif + } + + //-------------------------------------------------------------------------- + ShardManager::ShardManager(const ShardManager &rhs) + : runtime(NULL), repl_id(0), owner_space(0), total_shards(0), + original_task(NULL), control_replicated(false), top_level_task(false) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ShardManager::~ShardManager(void) + //-------------------------------------------------------------------------- + { + // We can delete our shard tasks + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + delete (*it); + local_shards.clear(); + // Finally unregister ourselves with the runtime + const bool owner_manager = (owner_space == runtime->address_space); + runtime->unregister_shard_manager(repl_id, owner_manager); + if (owner_manager) + { + if (control_replicated) + { + startup_barrier.destroy_barrier(); + pending_partition_barrier.destroy_barrier(); + creation_barrier.destroy_barrier(); + deletion_ready_barrier.destroy_barrier(); + deletion_mapping_barrier.destroy_barrier(); + deletion_execution_barrier.destroy_barrier(); + inline_mapping_barrier.destroy_barrier(); + external_resource_barrier.destroy_barrier(); + mapping_fence_barrier.destroy_barrier(); + trace_recording_barrier.destroy_barrier(); + summary_fence_barrier.destroy_barrier(); + execution_fence_barrier.destroy_barrier(); + attach_broadcast_barrier.destroy_barrier(); + attach_reduce_barrier.destroy_barrier(); + dependent_partition_barrier.destroy_barrier(); + semantic_attach_barrier.destroy_barrier(); + if (inorder_barrier.exists()) + inorder_barrier.destroy_barrier(); + callback_barrier.destroy_barrier(); +#ifdef DEBUG_LEGION_COLLECTIVES + collective_check_barrier.destroy_barrier(); + close_check_barrier.destroy_barrier(); +#endif + } + // Send messages to all the remote spaces to remove the manager + std::set sent_spaces; + for (unsigned idx = 0; idx < address_spaces->size(); idx++) + { + AddressSpaceID target = (*address_spaces)[idx]; + if (sent_spaces.find(target) != sent_spaces.end()) + continue; + if (target == runtime->address_space) + continue; + Serializer rez; + { + RezCheck z(rez); + rez.serialize(repl_id); + } + runtime->send_replicate_delete(target, rez); + sent_spaces.insert(target); + } + } + if ((address_spaces != NULL) && address_spaces->remove_reference()) + delete address_spaces; + if (local_future_result != NULL) + free(local_future_result); + } + + //-------------------------------------------------------------------------- + ShardManager& ShardManager::operator=(const ShardManager &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ShardManager::set_shard_mapping(const std::vector &mapping) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(mapping.size() == total_shards); +#endif + shard_mapping = mapping; + } + + //-------------------------------------------------------------------------- + void ShardManager::set_address_spaces( + const std::vector &spaces) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(address_spaces == NULL); +#endif + address_spaces = new ShardMapping(spaces); + address_spaces->add_reference(); + } + + //-------------------------------------------------------------------------- + void ShardManager::create_callback_barrier(size_t arrival_count) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!callback_barrier.exists()); + assert(owner_space == runtime->address_space); + assert(arrival_count == runtime->total_address_spaces); +#endif + callback_barrier = + RtBarrier(Realm::Barrier::create_barrier(arrival_count)); + } + + //-------------------------------------------------------------------------- + ShardTask* ShardManager::create_shard(ShardID id, Processor target) + //-------------------------------------------------------------------------- + { + ShardTask *shard = new ShardTask(runtime, this, id, target); + local_shards.push_back(shard); + return shard; + } + + //-------------------------------------------------------------------------- + void ShardManager::extract_event_preconditions( + const std::deque &instances) + //-------------------------------------------------------------------------- + { + // Iterate through all the shards and have them extract + // their event preconditions + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + (*it)->extract_event_preconditions(instances); + } + + //-------------------------------------------------------------------------- + void ShardManager::launch(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!local_shards.empty()); + assert(address_spaces == NULL); +#endif + address_spaces = new ShardMapping(); + address_spaces->add_reference(); + address_spaces->resize(local_shards.size()); + // Sort the shards into their target address space + std::map > shard_groups; + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + const AddressSpaceID target = + runtime->find_address_space((*it)->target_proc); + shard_groups[target].push_back(*it); +#ifdef DEBUG_LEGION + assert((*it)->shard_id < address_spaces->size()); +#endif + (*address_spaces)[(*it)->shard_id] = target; + } + local_shards.clear(); + // Compute the unique shard spaces and make callback barrier + // which has as many arrivers as unique shard spaces + callback_barrier = + RtBarrier(Realm::Barrier::create_barrier(shard_groups.size())); + // Now either send the shards to the remote nodes or record them locally + for (std::map >::const_iterator + it = shard_groups.begin(); it != shard_groups.end(); it++) + { + if (it->first != runtime->address_space) + { + distribute_shards(it->first, it->second); + // Clean up the shards that are now sent remotely + for (unsigned idx = 0; idx < it->second.size(); idx++) + delete it->second[idx]; + } + else + local_shards = it->second; + } + if (!local_shards.empty()) + { + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + launch_shard(*it); + } + } + + //-------------------------------------------------------------------------- + void ShardManager::distribute_shards(AddressSpaceID target, + const std::vector &shards) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(address_spaces != NULL); +#endif + Serializer rez; + { + RezCheck z(rez); + rez.serialize(repl_id); + rez.serialize(total_shards); + rez.serialize(control_replicated); + rez.serialize(top_level_task); + rez.serialize(startup_barrier); + address_spaces->pack_mapping(rez); + if (control_replicated) + { +#ifdef DEBUG_LEGION + assert(pending_partition_barrier.exists()); + assert(creation_barrier.exists()); + assert(deletion_ready_barrier.exists()); + assert(deletion_mapping_barrier.exists()); + assert(deletion_execution_barrier.exists()); + assert(inline_mapping_barrier.exists()); + assert(external_resource_barrier.exists()); + assert(mapping_fence_barrier.exists()); + assert(trace_recording_barrier.exists()); + assert(summary_fence_barrier.exists()); + assert(execution_fence_barrier.exists()); + assert(attach_broadcast_barrier.exists()); + assert(attach_reduce_barrier.exists()); + assert(dependent_partition_barrier.exists()); + assert(semantic_attach_barrier.exists()); + assert(callback_barrier.exists()); + assert(shard_mapping.size() == total_shards); +#endif + rez.serialize(pending_partition_barrier); + rez.serialize(creation_barrier); + rez.serialize(deletion_ready_barrier); + rez.serialize(deletion_mapping_barrier); + rez.serialize(deletion_execution_barrier); + rez.serialize(inline_mapping_barrier); + rez.serialize(external_resource_barrier); + rez.serialize(mapping_fence_barrier); + rez.serialize(trace_recording_barrier); + rez.serialize(summary_fence_barrier); + rez.serialize(execution_fence_barrier); + rez.serialize(attach_broadcast_barrier); + rez.serialize(attach_reduce_barrier); + rez.serialize(dependent_partition_barrier); + rez.serialize(semantic_attach_barrier); + rez.serialize(inorder_barrier); + rez.serialize(callback_barrier); +#ifdef DEBUG_LEGION_COLLECTIVES + assert(collective_check_barrier.exists()); + rez.serialize(collective_check_barrier); + assert(close_check_barrier.exists()); + rez.serialize(close_check_barrier); +#endif + for (std::vector::const_iterator it = + shard_mapping.begin(); it != shard_mapping.end(); it++) + rez.serialize(*it); + } + rez.serialize(shards.size()); + for (std::vector::const_iterator it = + shards.begin(); it != shards.end(); it++) + { + rez.serialize((*it)->shard_id); + rez.serialize((*it)->target_proc); + (*it)->pack_task(rez, target); + } + } + runtime->send_replicate_launch(target, rez); + // Update the remote constituents count + remote_constituents++; + } + + //-------------------------------------------------------------------------- + void ShardManager::unpack_shards_and_launch(Deserializer &derez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(owner_space != runtime->address_space); + assert(local_shards.empty()); + assert(address_spaces == NULL); +#endif + address_spaces = new ShardMapping(); + address_spaces->add_reference(); + address_spaces->unpack_mapping(derez); + if (control_replicated) + { + derez.deserialize(pending_partition_barrier); + derez.deserialize(creation_barrier); + derez.deserialize(deletion_ready_barrier); + derez.deserialize(deletion_mapping_barrier); + derez.deserialize(deletion_execution_barrier); + derez.deserialize(inline_mapping_barrier); + derez.deserialize(external_resource_barrier); + derez.deserialize(mapping_fence_barrier); + derez.deserialize(trace_recording_barrier); + derez.deserialize(summary_fence_barrier); + derez.deserialize(execution_fence_barrier); + derez.deserialize(attach_broadcast_barrier); + derez.deserialize(attach_reduce_barrier); + derez.deserialize(dependent_partition_barrier); + derez.deserialize(semantic_attach_barrier); + derez.deserialize(inorder_barrier); + derez.deserialize(callback_barrier); +#ifdef DEBUG_LEGION_COLLECTIVES + derez.deserialize(collective_check_barrier); + derez.deserialize(close_check_barrier); +#endif + shard_mapping.resize(total_shards); + for (unsigned idx = 0; idx < total_shards; idx++) + derez.deserialize(shard_mapping[idx]); + } + size_t num_shards; + derez.deserialize(num_shards); + local_shards.resize(num_shards); + for (unsigned idx = 0; idx < num_shards; idx++) + { + ShardID shard_id; + derez.deserialize(shard_id); + Processor target; + derez.deserialize(target); + ShardTask *shard = new ShardTask(runtime, this, shard_id, target); + std::set ready_preconditions; + shard->unpack_task(derez, target, ready_preconditions); + local_shards[idx] = shard; + if (!ready_preconditions.empty()) + launch_shard(shard, Runtime::merge_events(ready_preconditions)); + else + launch_shard(shard); + } + } + + //-------------------------------------------------------------------------- + void ShardManager::launch_shard(ShardTask *task, RtEvent precondition) const + //-------------------------------------------------------------------------- + { + ShardManagerLaunchArgs args(task); + runtime->issue_runtime_meta_task(args, LG_LATENCY_WORK_PRIORITY, + precondition); + } + + //-------------------------------------------------------------------------- + void ShardManager::complete_startup_initialization(void) const + //-------------------------------------------------------------------------- + { + // Do our arrival + Runtime::phase_barrier_arrive(startup_barrier, 1/*count*/); + // Then wait for everyone else to be ready + startup_barrier.wait(); + } + + //-------------------------------------------------------------------------- + bool ShardManager::is_total_sharding(void) + //-------------------------------------------------------------------------- + { + AutoLock m_lock(manager_lock); + if (unique_shard_spaces.empty()) + for (unsigned shard = 0; shard < total_shards; shard++) + unique_shard_spaces.insert((*address_spaces)[shard]); + return (unique_shard_spaces.size() == runtime->total_address_spaces); + } + + //-------------------------------------------------------------------------- + void ShardManager::handle_post_mapped(bool local, RtEvent precondition) + //-------------------------------------------------------------------------- + { + bool notify = false; + { + AutoLock m_lock(manager_lock); + if (precondition.exists()) + mapping_preconditions.insert(precondition); + if (local) + { + local_mapping_complete++; +#ifdef DEBUG_LEGION + assert(local_mapping_complete <= local_shards.size()); +#endif + } + else + { + remote_mapping_complete++; +#ifdef DEBUG_LEGION + assert(remote_mapping_complete <= remote_constituents); +#endif + } + notify = (local_mapping_complete == local_shards.size()) && + (remote_mapping_complete == remote_constituents); + } + if (notify) + { + RtEvent mapped_precondition; + if (!mapping_preconditions.empty()) + mapped_precondition = Runtime::merge_events(mapping_preconditions); + if (original_task == NULL) + { + Serializer rez; + rez.serialize(repl_id); + rez.serialize(mapped_precondition); + runtime->send_replicate_post_mapped(owner_space, rez); + } + else + original_task->handle_post_mapped(false/*deferral*/, + mapped_precondition); + } + } + + //-------------------------------------------------------------------------- + void ShardManager::handle_post_execution(const void *res, size_t res_size, + bool owned, bool local) + //-------------------------------------------------------------------------- + { + bool notify = false; + bool future_claimed = false; + { + AutoLock m_lock(manager_lock); + if (local) + { + local_execution_complete++; +#ifdef DEBUG_LEGION + assert(local_execution_complete <= local_shards.size()); +#endif + } + else + { + remote_execution_complete++; +#ifdef DEBUG_LEGION + assert(remote_execution_complete <= remote_constituents); +#endif + } + notify = (local_execution_complete == local_shards.size()) && + (remote_execution_complete == remote_constituents); + // See if we need to save the future or compare it + if (!local_future_set) + { + local_future_size = res_size; + if (!owned) + { + local_future_result = malloc(local_future_size); + memcpy(local_future_result, res, local_future_size); + } + else + { + local_future_result = const_cast(res); // take ownership + future_claimed = true; + } + local_future_set = true; + } +#ifdef DEBUG_LEGION + // In debug mode we'll do a comparison to see if the futures + // are bit-wise the same or not and issue a warning if not + else if ((local_future_size != res_size) || ((local_future_size > 0) && + (strncmp((const char*)res, (const char*)local_future_result, + local_future_size) != 0))) + REPORT_LEGION_WARNING(LEGION_WARNING_MISMATCHED_REPLICATED_FUTURES, + "WARNING: futures returned from control " + "replicated task %s have different bitwise " + "values!", local_shards[0]->get_task_name()) +#endif + } + if (notify) + { + if (original_task == NULL) + { + Serializer rez; + rez.serialize(repl_id); + rez.serialize(local_future_size); + if (local_future_size > 0) + rez.serialize(local_future_result, local_future_size); + runtime->send_replicate_post_execution(owner_space, rez); + } + else + { + original_task->handle_future(local_future_result, + local_future_size, true/*owned*/); + local_future_result = NULL; + local_future_size = 0; + original_task->complete_execution(); + } + } + // if we own it and don't use it we need to free it + if (owned && !future_claimed) + free(const_cast(res)); + } + + //-------------------------------------------------------------------------- + void ShardManager::trigger_task_complete(bool local) + //-------------------------------------------------------------------------- + { + bool notify = false; + { + AutoLock m_lock(manager_lock); + if (local) + { + trigger_local_complete++; +#ifdef DEBUG_LEGION + assert(trigger_local_complete <= local_shards.size()); +#endif + } + else + { + trigger_remote_complete++; +#ifdef DEBUG_LEGION + assert(trigger_remote_complete <= remote_constituents); +#endif + } + notify = (trigger_local_complete == local_shards.size()) && + (trigger_remote_complete == remote_constituents); + } + if (notify) + { + if (original_task == NULL) + { + Serializer rez; + rez.serialize(repl_id); + runtime->send_replicate_trigger_complete(owner_space, rez); + } + else + { +#ifdef DEBUG_LEGION + assert(!local_shards.empty()); +#endif + // For one of the shards we either need to return resources up + // the tree or report leaks and duplicates of resources. + // All the shards have the same set so we only have to do this + // for one of the shards. + std::set applied; + if (original_task->is_top_level_task()) + local_shards[0]->report_leaks_and_duplicates(applied); + else + local_shards[0]->return_resources( + original_task->get_context(), applied); + // We'll just wait for now since there's no good way to + // force this to be propagated back otherwise + if (!applied.empty()) + { + const RtEvent wait_on = Runtime::merge_events(applied); + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + } + original_task->trigger_children_complete(); + } + } + } + + //-------------------------------------------------------------------------- + void ShardManager::trigger_task_commit(bool local) + //-------------------------------------------------------------------------- + { + bool notify = false; + { + AutoLock m_lock(manager_lock); + if (local) + { + trigger_local_commit++; +#ifdef DEBUG_LEGION + assert(trigger_local_commit <= local_shards.size()); +#endif + } + else + { + trigger_remote_commit++; +#ifdef DEBUG_LEGION + assert(trigger_remote_commit <= remote_constituents); +#endif + } + notify = (trigger_local_commit == local_shards.size()) && + (trigger_remote_commit == remote_constituents); + } + if (notify) + { + if (original_task == NULL) + { + Serializer rez; + rez.serialize(repl_id); + runtime->send_replicate_trigger_commit(owner_space, rez); + } + else + original_task->trigger_children_committed(); + } + } + + //-------------------------------------------------------------------------- + void ShardManager::send_collective_message(ShardID target, Serializer &rez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(target < address_spaces->size()); +#endif + AddressSpaceID target_space = (*address_spaces)[target]; + // Check to see if this is a local shard + if (target_space == runtime->address_space) + { + Deserializer derez(rez.get_buffer(), rez.get_used_bytes()); + // Have to unpack the preample we already know + ReplicationID local_repl; + derez.deserialize(local_repl); + handle_collective_message(derez); + } + else + runtime->send_control_replicate_collective_message(target_space, rez); + } + + //-------------------------------------------------------------------------- + void ShardManager::handle_collective_message(Deserializer &derez) + //-------------------------------------------------------------------------- + { + // Figure out which shard we are going to + ShardID target; + derez.deserialize(target); + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + if ((*it)->shard_id == target) + { + (*it)->handle_collective_message(derez); + return; + } + } + // Should never get here + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardManager::send_future_map_request(ShardID target, Serializer &rez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(target < address_spaces->size()); +#endif + AddressSpaceID target_space = (*address_spaces)[target]; + // Check to see if this is a local shard + if (target_space == runtime->address_space) + { + Deserializer derez(rez.get_buffer(), rez.get_used_bytes()); + // Have to unpack the preample we already know + ReplicationID local_repl; + derez.deserialize(local_repl); + handle_future_map_request(derez); + } + else + runtime->send_control_replicate_future_map_request(target_space, rez); + } + + //-------------------------------------------------------------------------- + void ShardManager::handle_future_map_request(Deserializer &derez) + //-------------------------------------------------------------------------- + { + // Figure out which shard we are going to + ShardID target; + derez.deserialize(target); + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + if ((*it)->shard_id == target) + { + (*it)->handle_future_map_request(derez); + return; + } + } + // Should never get here + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardManager::send_equivalence_set_request(ShardID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(target < address_spaces->size()); +#endif + AddressSpaceID target_space = (*address_spaces)[target]; + // Check to see if this is a local shard + if (target_space == runtime->address_space) + { + Deserializer derez(rez.get_buffer(), rez.get_used_bytes()); + // Have to unpack the preample we already know + ReplicationID local_repl; + derez.deserialize(local_repl); + handle_equivalence_set_request(derez); + } + else + runtime->send_control_replicate_equivalence_set_request(target_space, + rez); + } + + //-------------------------------------------------------------------------- + void ShardManager::handle_equivalence_set_request(Deserializer &derez) + //-------------------------------------------------------------------------- + { + // Figure out which shard we are going to + ShardID target; + derez.deserialize(target); + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + if ((*it)->shard_id == target) + { + (*it)->handle_equivalence_set_request(derez); + return; + } + } + // Should never get here + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardManager::send_intra_space_dependence(ShardID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(target < address_spaces->size()); +#endif + AddressSpaceID target_space = (*address_spaces)[target]; + // Check to see if this is a local shard + if (target_space == runtime->address_space) + { + Deserializer derez(rez.get_buffer(), rez.get_used_bytes()); + // Have to unpack the preample we already know + ReplicationID local_repl; + derez.deserialize(local_repl); + handle_intra_space_dependence(derez); + } + else + runtime->send_control_replicate_intra_space_dependence(target_space, + rez); + } + + //-------------------------------------------------------------------------- + void ShardManager::handle_intra_space_dependence(Deserializer &derez) + //-------------------------------------------------------------------------- + { + // Figure out which shard we are going to + ShardID target; + derez.deserialize(target); + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + if ((*it)->shard_id == target) + { + (*it)->handle_intra_space_dependence(derez); + return; + } + } + // Should never get here + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardManager::broadcast_resource_update(ShardTask *source, + Serializer &rez, std::set &applied_events) + //-------------------------------------------------------------------------- + { + std::vector shard_spaces; + { + AutoLock m_lock(manager_lock); + if (unique_shard_spaces.empty()) + for (unsigned shard = 0; shard < total_shards; shard++) + unique_shard_spaces.insert((*address_spaces)[shard]); + shard_spaces.insert(shard_spaces.end(), + unique_shard_spaces.begin(), unique_shard_spaces.end()); + } + // First pack it out and send it out to any remote nodes + if (shard_spaces.size() > 1) + { + // Find the start index + int start_idx = -1; + for (unsigned idx = 0; idx < shard_spaces.size(); idx++) + { + if (shard_spaces[idx] != runtime->address_space) + continue; + start_idx = idx; + break; + } +#ifdef DEBUG_LEGION + assert(start_idx >= 0); +#endif + std::vector locals; + std::vector targets; + for (int idx = 0; idx < runtime->legion_collective_radix; idx++) + { + unsigned next = idx + 1; + if (next >= shard_spaces.size()) + break; + locals.push_back(next); + // Convert from relative to actual address space + const unsigned next_index = (start_idx + next) % shard_spaces.size(); + targets.push_back(shard_spaces[next_index]); + } + for (unsigned idx = 0; idx < locals.size(); idx++) + { + RtEvent next_done = Runtime::create_rt_user_event(); + Serializer rez2; + rez2.serialize(repl_id); + rez2.serialize(start_idx); + rez2.serialize(locals[idx]); + rez2.serialize(rez.get_used_bytes()); + rez2.serialize(rez.get_buffer(), rez.get_used_bytes()); + rez2.serialize(next_done); + runtime->send_control_replicate_resource_update(targets[idx], rez2); + applied_events.insert(next_done); + } + } + // Then send it to any other local shards + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + // Skip the source since that's where it came from + if ((*it) == source) + continue; + Deserializer derez(rez.get_buffer(), rez.get_used_bytes()); + (*it)->handle_resource_update(derez, applied_events); + } + } + + //-------------------------------------------------------------------------- + void ShardManager::handle_resource_update(Deserializer &derez) + //-------------------------------------------------------------------------- + { + unsigned start_idx, local_idx; + derez.deserialize(start_idx); + derez.deserialize(local_idx); + size_t message_size; + derez.deserialize(message_size); + const void *message = derez.get_current_pointer(); + derez.advance_pointer(message_size); + RtUserEvent done_event; + derez.deserialize(done_event); + // Send out any remote updates first + std::vector shard_spaces; + { + AutoLock m_lock(manager_lock); + if (unique_shard_spaces.empty()) + for (unsigned shard = 0; shard < total_shards; shard++) + unique_shard_spaces.insert((*address_spaces)[shard]); + shard_spaces.insert(shard_spaces.end(), + unique_shard_spaces.begin(), unique_shard_spaces.end()); + } + // First pack it out and send it out to any remote nodes + std::vector locals; + std::vector targets; + const unsigned start = local_idx * runtime->legion_collective_radix + 1; + for (int idx = 0; idx < runtime->legion_collective_radix; idx++) + { + unsigned next = start + idx; + if (next >= shard_spaces.size()) + break; + locals.push_back(next); + // Convert from relative to actual address space + const unsigned next_index = (start_idx + next) % shard_spaces.size(); + targets.push_back(shard_spaces[next_index]); + } + std::set remote_handled; + if (!targets.empty()) + { + for (unsigned idx = 0; idx < targets.size(); idx++) + { + RtEvent next_done = Runtime::create_rt_user_event(); + Serializer rez; + rez.serialize(repl_id); + rez.serialize(start_idx); + rez.serialize(locals[idx]); + rez.serialize(message_size); + rez.serialize(message, message_size); + rez.serialize(next_done); + runtime->send_control_replicate_resource_update(targets[idx], rez); + remote_handled.insert(next_done); + } + } + // Handle it on all our local shards + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + Deserializer derez2(message, message_size); + (*it)->handle_resource_update(derez2, remote_handled); + } + if (!remote_handled.empty()) + Runtime::trigger_event(done_event, + Runtime::merge_events(remote_handled)); + else + Runtime::trigger_event(done_event); + } + + //-------------------------------------------------------------------------- + void ShardManager::send_trace_event_request( + ShardedPhysicalTemplate *physical_template, ShardID shard_source, + AddressSpaceID template_source, size_t template_index, ApEvent event, + AddressSpaceID event_space, RtUserEvent done_event) + //-------------------------------------------------------------------------- + { + // See whether we are on the right node to handle this request, if not + // then forward the request onto the proper node + if (event_space != runtime->address_space) + { +#ifdef DEBUG_LEGION + assert(template_source == runtime->address_space); +#endif + // Check to see if we have a shard on that address space, if not + // then we know that this event can't have come from there + bool found = false; + for (unsigned idx = 0; idx < address_spaces->size(); idx++) + { + if ((*address_spaces)[idx] != event_space) + continue; + found = true; + break; + } + if (found) + { + Serializer rez; + { + RezCheck z(rez); + rez.serialize(repl_id); + rez.serialize(physical_template); + rez.serialize(template_index); + rez.serialize(shard_source); + rez.serialize(event); + rez.serialize(done_event); + } + runtime->send_control_replicate_trace_event_request(event_space, rez); + } + else + send_trace_event_response(physical_template, template_source, + event, ApBarrier::NO_AP_BARRIER, done_event); + } + else + { + // Ask each of our local shards to check for the event in the template + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + const ApBarrier result = + (*it)->handle_find_trace_shard_event(template_index, + event, shard_source); + // If we found it then we are done + if (result.exists()) + { + send_trace_event_response(physical_template, template_source, + event, result, done_event); + return; + } + } + // If we make it here then we didn't find it so return the result + send_trace_event_response(physical_template, template_source, + event, ApBarrier::NO_AP_BARRIER, done_event); + } + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_trace_event_request( + Deserializer &derez, Runtime *runtime, AddressSpaceID source) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardedPhysicalTemplate *physical_template; + derez.deserialize(physical_template); + size_t template_index; + derez.deserialize(template_index); + ShardID shard_source; + derez.deserialize(shard_source); + ApEvent event; + derez.deserialize(event); + RtUserEvent done_event; + derez.deserialize(done_event); + + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->send_trace_event_request(physical_template, shard_source, source, + template_index, event, runtime->address_space, done_event); + } + + //-------------------------------------------------------------------------- + void ShardManager::send_trace_event_response( + ShardedPhysicalTemplate *physical_template, AddressSpaceID temp_source, + ApEvent event, ApBarrier result, RtUserEvent done_event) + //-------------------------------------------------------------------------- + { + if (temp_source != runtime->address_space) + { + // Not local so send the response message + Serializer rez; + { + RezCheck z(rez); + rez.serialize(physical_template); + rez.serialize(event); + rez.serialize(result); + rez.serialize(done_event); + } + runtime->send_control_replicate_trace_event_response(temp_source, rez); + } + else // This is local so handle it here + { + physical_template->record_trace_shard_event(event, result); + Runtime::trigger_event(done_event); + } + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_trace_event_response( + Deserializer &derez) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + ShardedPhysicalTemplate *physical_template; + derez.deserialize(physical_template); + ApEvent event; + derez.deserialize(event); + ApBarrier result; + derez.deserialize(result); + RtUserEvent done_event; + derez.deserialize(done_event); + + physical_template->record_trace_shard_event(event, result); + Runtime::trigger_event(done_event); + } + + //-------------------------------------------------------------------------- + void ShardManager::send_trace_update(ShardID target, Serializer &rez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(target < address_spaces->size()); +#endif + AddressSpaceID target_space = (*address_spaces)[target]; + // Check to see if this is a local shard + if (target_space == runtime->address_space) + { + Deserializer derez(rez.get_buffer(), rez.get_used_bytes()); + // Have to unpack the preample we already know + ReplicationID local_repl; + derez.deserialize(local_repl); + handle_trace_update(derez, target_space); + } + else + runtime->send_control_replicate_trace_update(target_space, rez); + } + + //-------------------------------------------------------------------------- + void ShardManager::handle_trace_update(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + // Figure out which shard we are going to + ShardID target; + derez.deserialize(target); + for (std::vector::const_iterator it = + local_shards.begin(); it != local_shards.end(); it++) + { + if ((*it)->shard_id == target) + { + (*it)->handle_trace_update(derez, source); + return; + } + } + // Should never get here + assert(false); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_launch(const void *args) + //-------------------------------------------------------------------------- + { + const ShardManagerLaunchArgs *largs = (const ShardManagerLaunchArgs*)args; + largs->shard->launch_shard(); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_delete(const void *args) + //-------------------------------------------------------------------------- + { + const ShardManagerDeleteArgs *dargs = (const ShardManagerDeleteArgs*)args; + if (dargs->manager->remove_reference()) + delete dargs->manager; + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_launch(Deserializer &derez, + Runtime *runtime, AddressSpaceID source) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + ReplicationID repl_id; + derez.deserialize(repl_id); + size_t total_shards; + derez.deserialize(total_shards); + bool control_repl; + derez.deserialize(control_repl); + bool top_level_task; + derez.deserialize(top_level_task); + RtBarrier startup_barrier; + derez.deserialize(startup_barrier); + ShardManager *manager = + new ShardManager(runtime, repl_id, control_repl, top_level_task, + total_shards, source, NULL/*original*/, startup_barrier); + manager->unpack_shards_and_launch(derez); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_delete( + Deserializer &derez, Runtime *runtime) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + delete manager; + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_post_mapped( + Deserializer &derez, Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + RtEvent precondition; + derez.deserialize(precondition); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->handle_post_mapped(false/*local*/, precondition); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_post_execution( + Deserializer &derez, Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + size_t future_result_size; + derez.deserialize(future_result_size); + const void *future_result = derez.get_current_pointer(); + if (future_result_size > 0) + derez.advance_pointer(future_result_size); + manager->handle_post_execution(future_result, future_result_size, + false/*owned*/, false/*local*/); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_trigger_complete( + Deserializer &derez, Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->trigger_task_complete(false/*local*/); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_trigger_commit( + Deserializer &derez, Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->trigger_task_commit(false/*local*/); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_collective_message(Deserializer &derez, + Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->handle_collective_message(derez); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_future_map_request(Deserializer &derez, + Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->handle_future_map_request(derez); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_trace_update(Deserializer &derez, + Runtime *runtime, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->handle_trace_update(derez, source); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_top_view_request(Deserializer &derez, + Runtime *runtime, AddressSpaceID request_source) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + ReplicationID repl_id; + derez.deserialize(repl_id); + DistributedID manager_did; + derez.deserialize(manager_did); + AddressSpaceID source; + derez.deserialize(source); + ReplicateContext *request_context; + derez.deserialize(request_context); + + RtEvent ready; + PhysicalManager *physical_manager = + runtime->find_or_request_instance_manager(manager_did, ready); + ShardManager *manager = runtime->find_shard_manager(repl_id); + if (!ready.has_triggered()) + ready.wait(); + manager->create_instance_top_view(physical_manager, source, + request_context, request_source, true/*handle now*/); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_top_view_response(Deserializer &derez, + Runtime *runtime) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + DistributedID manager_did, view_did; + derez.deserialize(manager_did); + derez.deserialize(view_did); + ReplicateContext *request_context; + derez.deserialize(request_context); + + RtEvent manager_ready, view_ready; + PhysicalManager *manager = + runtime->find_or_request_instance_manager(manager_did, manager_ready); + InstanceView *view = static_cast( + runtime->find_or_request_logical_view(view_did, view_ready)); + if (!manager_ready.has_triggered()) + manager_ready.wait(); + if (!view_ready.has_triggered()) + view_ready.wait(); + request_context->record_replicate_instance_top_view(manager, view); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_eq_request(Deserializer &derez, + Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->handle_equivalence_set_request(derez); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_intra_space_dependence( + Deserializer &derez, Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->handle_intra_space_dependence(derez); + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardManager::handle_resource_update(Deserializer &derez, + Runtime *runtime) + //-------------------------------------------------------------------------- + { + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *manager = runtime->find_shard_manager(repl_id); + manager->handle_resource_update(derez); + } + + //-------------------------------------------------------------------------- + ShardingFunction* ShardManager::find_sharding_function(ShardingID sid) + //-------------------------------------------------------------------------- + { + // Check to see if it is in the cache + { + AutoLock m_lock(manager_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + sharding_functions.find(sid); + if (finder != sharding_functions.end()) + return finder->second; + } + // Get the functor from the runtime + ShardingFunctor *functor = runtime->find_sharding_functor(sid); + // Retake the lock + AutoLock m_lock(manager_lock); + // See if we lost the race + std::map::const_iterator finder = + sharding_functions.find(sid); + if (finder != sharding_functions.end()) + return finder->second; + ShardingFunction *result = + new ShardingFunction(functor, runtime->forest, sid, total_shards); + // Save the result for the future + sharding_functions[sid] = result; + return result; + } + + //-------------------------------------------------------------------------- + void ShardManager::create_instance_top_view(PhysicalManager *manager, + AddressSpaceID source, ReplicateContext *request_context, + AddressSpaceID request_source, bool handle_now/*= false*/) + //-------------------------------------------------------------------------- + { + // Easy case if we are not control replicated + if (!control_replicated) + { + InstanceView *result = + request_context->create_replicate_instance_top_view(manager, source); + request_context->record_replicate_instance_top_view(manager, result); + return; + } + // If we're on the owner node of the manager just handle it here + if (handle_now || (manager->owner_space == runtime->address_space)) + { +#ifdef DEBUG_LEGION + assert(!local_shards.empty()); +#endif + // Distribute manager requests across local shards + const unsigned index = manager->did % local_shards.size(); + InstanceView *result = + local_shards[index]->create_instance_top_view(manager, source); + // Now we have to tell the request context about the result + if (request_source != runtime->address_space) + { + Serializer rez; + { + RezCheck z(rez); + rez.serialize(manager->did); + rez.serialize(result->did); + rez.serialize(request_context); + } + runtime->send_control_replicate_top_view_response(request_source,rez); + } + else + request_context->record_replicate_instance_top_view(manager, result); + } + else + { + // Check to see if we already have a manager on the owner node + // if so we can just send a message there and handle it + // If not, we round robin the distributed ID across the shards to + // find the shard to handle the request and send it there + AddressSpaceID target; + { + AutoLock m_lock(manager_lock); + if (unique_shard_spaces.empty()) + for (unsigned shard = 0; shard < total_shards; shard++) + unique_shard_spaces.insert((*address_spaces)[shard]); + if (unique_shard_spaces.find(manager->owner_space) == + unique_shard_spaces.end()) + { + // Round-robin accross the shards + const unsigned index = manager->did % total_shards; + target = (*address_spaces)[index]; + } + else + target = manager->owner_space; + } + if (target != runtime->address_space) + { + // Now we can send the message to the target + Serializer rez; + { + RezCheck z(rez); + rez.serialize(repl_id); + rez.serialize(manager->did); + rez.serialize(source); + rez.serialize(request_context); + } + runtime->send_control_replicate_top_view_request(target, rez); + } + else + create_instance_top_view(manager, source, request_context, + request_source, true/*handle now*/); + } + } + + //-------------------------------------------------------------------------- + void ShardManager::perform_global_registration_callbacks( + Realm::DSOReferenceImplementation *dso, RtEvent local_done, + RtEvent global_done, std::set &preconditions) + //-------------------------------------------------------------------------- + { + // See if we're the first one to handle this DSO + const std::pair + key(dso->dso_name, dso->symbol_name); + { + AutoLock m_lock(manager_lock); + // Check to see if we've already handled this + std::set >::const_iterator finder = + unique_registration_callbacks.find(key); + if (finder != unique_registration_callbacks.end()) + return; + unique_registration_callbacks.insert(key); + if (unique_shard_spaces.empty()) + for (unsigned shard = 0; shard < total_shards; shard++) + unique_shard_spaces.insert((*address_spaces)[shard]); + } + // We're the first one so handle it + if (!is_total_sharding()) + { + std::set local_preconditions; + AddressSpaceID space = 0; + for (std::set::const_iterator it = + unique_shard_spaces.begin(); it != + unique_shard_spaces.end(); it++, space++) + { + if ((*it) == runtime->address_space) + break; + } +#ifdef DEBUG_LEGION + assert(space < unique_shard_spaces.size()); +#endif + for ( ; space < runtime->total_address_spaces; + space += unique_shard_spaces.size()) + { + if (unique_shard_spaces.find(space) != unique_shard_spaces.end()) + continue; + runtime->send_registration_callback(space, dso, global_done, + local_preconditions); + } + if (!local_preconditions.empty()) + { + local_preconditions.insert(local_done); + Runtime::phase_barrier_arrive(callback_barrier, 1/*count*/, + Runtime::merge_events(local_preconditions)); + } + else + Runtime::phase_barrier_arrive(callback_barrier, + 1/*count*/, local_done); + } + else // there will be a callback on every node anyway + Runtime::phase_barrier_arrive(callback_barrier,1/*count*/,local_done); + preconditions.insert(callback_barrier); + Runtime::advance_barrier(callback_barrier); + if (!callback_barrier.exists()) + REPORT_LEGION_FATAL(LEGION_FATAL_UNIMPLEMENTED_FEATURE, + "Need support for refreshing exhausted callback phase " + "barrier generations.") + } + + //-------------------------------------------------------------------------- + bool ShardManager::perform_semantic_attach(void) + //-------------------------------------------------------------------------- + { + if (local_shards.size() == 1) + return true; + AutoLock m_lock(manager_lock); +#ifdef DEBUG_LEGION + assert(semantic_attach_counter < local_shards.size()); +#endif + if (++semantic_attach_counter == local_shards.size()) + { + semantic_attach_counter = 0; + return true; + } + else + return false; + } + + ///////////////////////////////////////////////////////////// + // Shard Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardCollective::ShardCollective(CollectiveIndexLocation loc, + ReplicateContext *ctx) + : manager(ctx->shard_manager), context(ctx), + local_shard(ctx->owner_shard->shard_id), + collective_index(ctx->get_next_collective_index(loc)) + //-------------------------------------------------------------------------- + { + context->add_reference(); + } + + //-------------------------------------------------------------------------- + ShardCollective::ShardCollective(ReplicateContext *ctx, CollectiveID id) + : manager(ctx->shard_manager), context(ctx), + local_shard(ctx->owner_shard->shard_id), collective_index(id) + //-------------------------------------------------------------------------- + { + context->add_reference(); + } + + //-------------------------------------------------------------------------- + ShardCollective::~ShardCollective(void) + //-------------------------------------------------------------------------- + { + // Unregister this with the context + context->unregister_collective(this); + if (context->remove_reference()) + delete context; + } + + //-------------------------------------------------------------------------- + int ShardCollective::convert_to_index(ShardID id, ShardID origin) const + //-------------------------------------------------------------------------- + { + // shift everything so that the target shard is at index 0 + const int result = + ((id + (manager->total_shards - origin)) % manager->total_shards); + return result; + } + + //-------------------------------------------------------------------------- + ShardID ShardCollective::convert_to_shard(int index, ShardID origin) const + //-------------------------------------------------------------------------- + { + // Add target then take the modulus + const ShardID result = (index + origin) % manager->total_shards; + return result; + } + + ///////////////////////////////////////////////////////////// + // Gather Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + BroadcastCollective::BroadcastCollective(CollectiveIndexLocation loc, + ReplicateContext *ctx, ShardID o) + : ShardCollective(loc, ctx), origin(o), + shard_collective_radix(ctx->get_shard_collective_radix()) + //-------------------------------------------------------------------------- + { + if (local_shard != origin) + done_event = Runtime::create_rt_user_event(); + } + + //-------------------------------------------------------------------------- + BroadcastCollective::BroadcastCollective(ReplicateContext *ctx, + CollectiveID id, ShardID o) + : ShardCollective(ctx, id), origin(o), + shard_collective_radix(ctx->get_shard_collective_radix()) + //-------------------------------------------------------------------------- + { + if (local_shard != origin) + done_event = Runtime::create_rt_user_event(); + } + + //-------------------------------------------------------------------------- + BroadcastCollective::~BroadcastCollective(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + void BroadcastCollective::perform_collective_async(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(local_shard == origin); +#endif + // Register this with the context + context->register_collective(this); + send_messages(); + } + + //-------------------------------------------------------------------------- + RtEvent BroadcastCollective::perform_collective_wait(bool block/*=true*/) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(local_shard != origin); +#endif + // Register this with the context + context->register_collective(this); + if (!done_event.has_triggered()) + { + if (block) + done_event.wait(); + else + return done_event; + } + return RtEvent::NO_RT_EVENT; + } + + //-------------------------------------------------------------------------- + void BroadcastCollective::handle_collective_message(Deserializer &derez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(local_shard != origin); +#endif + // No need for the lock since this is only written to once + unpack_collective(derez); + // Send our messages + send_messages(); + // Then trigger our event to indicate that we are ready + Runtime::trigger_event(done_event); + } + + //-------------------------------------------------------------------------- + RtEvent BroadcastCollective::get_done_event(void) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(local_shard != origin); +#endif + return done_event; + } + + //-------------------------------------------------------------------------- + void BroadcastCollective::send_messages(void) const + //-------------------------------------------------------------------------- + { + const int local_index = convert_to_index(local_shard, origin); + for (int idx = 1; idx <= shard_collective_radix; idx++) + { + const int target_index = local_index * shard_collective_radix + idx; + if (target_index >= int(manager->total_shards)) + break; + ShardID target = convert_to_shard(target_index, origin); + Serializer rez; + { + rez.serialize(manager->repl_id); + rez.serialize(target); + rez.serialize(collective_index); + pack_collective(rez); + } + manager->send_collective_message(target, rez); + } + } + + ///////////////////////////////////////////////////////////// + // Gather Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + GatherCollective::GatherCollective(CollectiveIndexLocation loc, + ReplicateContext *ctx, ShardID t) + : ShardCollective(loc, ctx), target(t), + shard_collective_radix(ctx->get_shard_collective_radix()), + expected_notifications(compute_expected_notifications()), + received_notifications(0) + //-------------------------------------------------------------------------- + { + if (expected_notifications > 1) + done_event = Runtime::create_rt_user_event(); + } + + //-------------------------------------------------------------------------- + GatherCollective::~GatherCollective(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + if (done_event.exists()) + assert(done_event.has_triggered()); +#endif + } + + //-------------------------------------------------------------------------- + void GatherCollective::perform_collective_async(void) + //-------------------------------------------------------------------------- + { + // Register this with the context + context->register_collective(this); + bool done = false; + { + AutoLock c_lock(collective_lock); +#ifdef DEBUG_LEGION + assert(received_notifications < expected_notifications); +#endif + done = (++received_notifications == expected_notifications); + } + if (done) + { + if (local_shard != target) + send_message(); + if (done_event.exists()) + Runtime::trigger_event(done_event); + } + } + + //-------------------------------------------------------------------------- + RtEvent GatherCollective::perform_collective_wait(bool block/*=true*/) + //-------------------------------------------------------------------------- + { + if (done_event.exists() && !done_event.has_triggered()) + { + if (block) + done_event.wait(); + else + return done_event; + } + return RtEvent::NO_RT_EVENT; + } + + //-------------------------------------------------------------------------- + void GatherCollective::handle_collective_message(Deserializer &derez) + //-------------------------------------------------------------------------- + { + bool done = false; + { + // Hold the lock while doing these operations + AutoLock c_lock(collective_lock); + // Unpack the result + unpack_collective(derez); + #ifdef DEBUG_LEGION + assert(received_notifications < expected_notifications); +#endif + done = (++received_notifications == expected_notifications); + } + if (done) + { + if (local_shard != target) + send_message(); + if (done_event.exists()) + Runtime::trigger_event(done_event); + } + } + + //-------------------------------------------------------------------------- + void GatherCollective::elide_collective(void) + //-------------------------------------------------------------------------- + { + if (done_event.exists()) + Runtime::trigger_event(done_event); + } + + //-------------------------------------------------------------------------- + void GatherCollective::send_message(void) + //-------------------------------------------------------------------------- + { + // Convert to our local index + const int local_index = convert_to_index(local_shard, target); +#ifdef DEBUG_LEGION + assert(local_index > 0); // should never be here for zero +#endif + // Subtract by 1 and then divide to get the target (truncate) + const int target_index = (local_index - 1) / shard_collective_radix; + // Then convert back to the target + ShardID next = convert_to_shard(target_index, target); + Serializer rez; + { + rez.serialize(manager->repl_id); + rez.serialize(next); + rez.serialize(collective_index); + AutoLock c_lock(collective_lock,1,false/*exclusive*/); + pack_collective(rez); + } + manager->send_collective_message(next, rez); + } + + //-------------------------------------------------------------------------- + int GatherCollective::compute_expected_notifications(void) const + //-------------------------------------------------------------------------- + { + int result = 1; // always have one arriver for ourself + const int index = convert_to_index(local_shard, target); + for (int idx = 1; idx <= shard_collective_radix; idx++) + { + const int source_index = index * shard_collective_radix + idx; + if (source_index >= int(manager->total_shards)) + break; + result++; + } + return result; + } + + ///////////////////////////////////////////////////////////// + // All Gather Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + template + AllGatherCollective::AllGatherCollective( + CollectiveIndexLocation loc, ReplicateContext *ctx) + : ShardCollective(loc, ctx), + shard_collective_radix(ctx->get_shard_collective_radix()), + shard_collective_log_radix(ctx->get_shard_collective_log_radix()), + shard_collective_stages(ctx->get_shard_collective_stages()), + shard_collective_participating_shards( + ctx->get_shard_collective_participating_shards()), + shard_collective_last_radix(ctx->get_shard_collective_last_radix()), + participating(int(local_shard) < shard_collective_participating_shards), + reorder_stages(NULL), pending_send_ready_stages(0) +#ifdef DEBUG_LEGION + , done_triggered(false) +#endif + //-------------------------------------------------------------------------- + { + initialize_collective(); + } + + //-------------------------------------------------------------------------- + template + AllGatherCollective::AllGatherCollective(ReplicateContext *ctx, + CollectiveID id) + : ShardCollective(ctx, id), + shard_collective_radix(ctx->get_shard_collective_radix()), + shard_collective_log_radix(ctx->get_shard_collective_log_radix()), + shard_collective_stages(ctx->get_shard_collective_stages()), + shard_collective_participating_shards( + ctx->get_shard_collective_participating_shards()), + shard_collective_last_radix(ctx->get_shard_collective_last_radix()), + participating(int(local_shard) < shard_collective_participating_shards), + reorder_stages(NULL), pending_send_ready_stages(0) +#ifdef DEBUG_LEGION + , done_triggered(false) +#endif + //-------------------------------------------------------------------------- + { + initialize_collective(); + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::initialize_collective(void) + //-------------------------------------------------------------------------- + { + if (manager->total_shards > 1) + { + // We already have our contributions for each stage so + // we can set the inditial participants to 1 + if (participating) + { +#ifdef DEBUG_LEGION + assert(shard_collective_stages > 0); +#endif + sent_stages.resize(shard_collective_stages, false); + stage_notifications.resize(shard_collective_stages, 1); + // Stage 0 always starts with 0 notifications since we'll + // explictcly arrive on it + stage_notifications[0] = 0; + } + done_event = Runtime::create_rt_user_event(); + } + } + + //-------------------------------------------------------------------------- + template + AllGatherCollective::~AllGatherCollective(void) + //-------------------------------------------------------------------------- + { + if (reorder_stages != NULL) + { +#ifdef DEBUG_LEGION + assert(reorder_stages->empty()); +#endif + delete reorder_stages; + } +#ifdef DEBUG_LEGION + if (participating) + { + // We should have sent all our stages before being deleted + for (unsigned idx = 0; idx < sent_stages.size(); idx++) + assert(sent_stages[idx]); + } + if (participating) + assert(done_triggered); + assert(done_event.has_triggered()); +#endif + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::perform_collective_sync(void) + //-------------------------------------------------------------------------- + { + perform_collective_async(); + perform_collective_wait(); + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::perform_collective_async(void) + //-------------------------------------------------------------------------- + { + // Register this with the context + context->register_collective(this); + if (manager->total_shards <= 1) + return; + // See if we are a participating shard or not + if (participating) + { + // We are a participating shard + // See if we are waiting for an initial notification + // if not we can just send our message now + if ((int(manager->total_shards) == + shard_collective_participating_shards) || + (local_shard >= (manager->total_shards - + shard_collective_participating_shards))) + { + const bool all_stages_done = initiate_collective(); + if (all_stages_done) + complete_exchange(); + } + } + else + { + // We are not a participating shard + // so we just have to send notification to one shard + send_remainder_stage(); + } + } + + //-------------------------------------------------------------------------- + template + RtEvent AllGatherCollective::perform_collective_wait( + bool block/*=true*/) + //-------------------------------------------------------------------------- + { + if (manager->total_shards <= 1) + return RtEvent::NO_RT_EVENT; + if (!done_event.has_triggered()) + { + if (block) + done_event.wait(); + else + return done_event; + } + return RtEvent::NO_RT_EVENT; + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::handle_collective_message( + Deserializer &derez) + //-------------------------------------------------------------------------- + { + int stage; + derez.deserialize(stage); +#ifdef DEBUG_LEGION + assert(participating || (stage == -1)); +#endif + unpack_stage(stage, derez); + bool all_stages_done = false; + if (stage == -1) + { + if (!participating) + all_stages_done = true; + else // we can now initiate the collective + all_stages_done = initiate_collective(); + } + else + all_stages_done = send_ready_stages(); + if (all_stages_done) + complete_exchange(); + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::elide_collective(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + // make it look like we sent all the stages + for (unsigned idx = 0; idx < sent_stages.size(); idx++) + sent_stages[idx] = true; + assert(!done_triggered); + assert(!done_event.has_triggered()); +#endif + // Trigger the user event + Runtime::trigger_event(done_event); +#ifdef DEBUG_LEGION + done_triggered = true; +#endif + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::construct_message(ShardID target, + int stage, Serializer &rez) + //-------------------------------------------------------------------------- + { + rez.serialize(manager->repl_id); + rez.serialize(target); + rez.serialize(collective_index); + rez.serialize(stage); + AutoLock c_lock(collective_lock, 1, false/*exclusive*/); + pack_collective_stage(rez, stage); + } + + //-------------------------------------------------------------------------- + template + bool AllGatherCollective::initiate_collective(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(participating); // should only get this for participating shards +#endif + { + AutoLock c_lock(collective_lock); +#ifdef DEBUG_LEGION + assert(!sent_stages.empty()); + assert(!sent_stages[0]); // stage 0 shouldn't be sent yet + assert(!stage_notifications.empty()); + if (shard_collective_stages == 1) + assert(stage_notifications[0] < shard_collective_last_radix); + else + assert(stage_notifications[0] < shard_collective_radix); +#endif + stage_notifications[0]++; + // Increment our guard to prevent deletion of the collective + // object while we are still traversing + pending_send_ready_stages++; + } + return send_ready_stages(0/*start stage*/); + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::send_remainder_stage(void) + //-------------------------------------------------------------------------- + { + if (participating) + { + // Send back to the shards that are not participating + ShardID target = local_shard + shard_collective_participating_shards; +#ifdef DEBUG_LEGION + assert(target < manager->total_shards); +#endif + Serializer rez; + construct_message(target, -1/*stage*/, rez); + manager->send_collective_message(target, rez); + } + else + { + // Send to a node that is participating + ShardID target = local_shard % shard_collective_participating_shards; + Serializer rez; + construct_message(target, -1/*stage*/, rez); + manager->send_collective_message(target, rez); + } + } + + //-------------------------------------------------------------------------- + template + bool AllGatherCollective::send_ready_stages(const int start_stage) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(participating); +#endif + // Iterate through the stages and send any that are ready + // Remember that stages have to be done in order + bool sent_previous_stage = false; + for (int stage = start_stage; stage < shard_collective_stages; stage++) + { + { + AutoLock c_lock(collective_lock); + if (sent_previous_stage) + { +#ifdef DEBUG_LEGION + assert(!sent_stages[stage-1]); +#endif + sent_stages[stage-1] = true; + sent_previous_stage = false; + } + // If this stage has already been sent then we can keep going + if (sent_stages[stage]) + continue; +#ifdef DEBUG_LEGION + assert(pending_send_ready_stages > 0); +#endif + // Check to see if we're sending this stage + // We need all the notifications from the previous stage before + // we can send this stage + if (stage > 0) + { + // We can't have multiple threads doing sends at the same time + // so make sure that only the last one is going through doing work + // but stage 0 is because it is always sent by the initiator so + // don't check this until we're past the first stage + if ((stage_notifications[stage-1] < shard_collective_radix) || + (pending_send_ready_stages > 1)) + { + // Remove our guard before exiting early + pending_send_ready_stages--; + return false; + } + else if (INORDER && (reorder_stages != NULL)) + { + // Check to see if we have any unhandled messages for + // the previous stage that we need to handle before sending + std::map > >::iterator + finder = reorder_stages->find(stage-1); + if (finder != reorder_stages->end()) + { + // Perform the handling for the buffered messages now + for (std::vector >::const_iterator it = + finder->second.begin(); it != finder->second.end(); it++) + { + Deserializer derez(it->first, it->second); + unpack_collective_stage(derez, finder->first); + free(it->first); + } + reorder_stages->erase(finder); + } + } + } + // If we get here then we can send the stage + } + // Now we can do the send + if (stage == (shard_collective_stages-1)) + { + for (int r = 1; r < shard_collective_last_radix; r++) + { + const ShardID target = local_shard ^ + (r << (stage * shard_collective_log_radix)); +#ifdef DEBUG_LEGION + assert(int(target) < shard_collective_participating_shards); +#endif + Serializer rez; + construct_message(target, stage, rez); + manager->send_collective_message(target, rez); + } + } + else + { + for (int r = 1; r < shard_collective_radix; r++) + { + const ShardID target = local_shard ^ + (r << (stage * shard_collective_log_radix)); +#ifdef DEBUG_LEGION + assert(int(target) < shard_collective_participating_shards); +#endif + Serializer rez; + construct_message(target, stage, rez); + manager->send_collective_message(target, rez); + } + } + sent_previous_stage = true; + } + // If we make it here, then we sent the last stage, check to see + // if we've seen all the notifications for it + AutoLock c_lock(collective_lock); + if (sent_previous_stage) + { +#ifdef DEBUG_LEGION + assert(!sent_stages[shard_collective_stages-1]); +#endif + sent_stages[shard_collective_stages-1] = true; + } + // Remove our pending guard and then check to see if we are done +#ifdef DEBUG_LEGION + assert(pending_send_ready_stages > 0); +#endif + if (((--pending_send_ready_stages) == 0) && + (stage_notifications.back() == shard_collective_last_radix)) + { +#ifdef DEBUG_LEGION + assert(!done_triggered); + done_triggered = true; +#endif + return true; + } + else + return false; + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::unpack_stage(int stage, + Deserializer &derez) + //-------------------------------------------------------------------------- + { + AutoLock c_lock(collective_lock); + // Do the unpack first while holding the lock + if (INORDER && (stage >= 0)) + { + // Check to see if we can handle this message now or whether we + // need to buffer it for the future because we have not finished + // sending the current stage yet or not + if (!sent_stages[stage]) + { + // Buffer this message until the stage is sent as well + const size_t buffer_size = derez.get_remaining_bytes(); + void *buffer = malloc(buffer_size); + memcpy(buffer, derez.get_current_pointer(), buffer_size); + derez.advance_pointer(buffer_size); + if (reorder_stages == NULL) + reorder_stages = + new std::map > >(); + (*reorder_stages)[stage].push_back( + std::pair(buffer, buffer_size)); + } + else + unpack_collective_stage(derez, stage); + } + else // Just do the unpack here immediately + unpack_collective_stage(derez, stage); + if (stage >= 0) + { +#ifdef DEBUG_LEGION + assert(stage < int(stage_notifications.size())); + if (stage < (shard_collective_stages-1)) + assert(stage_notifications[stage] < shard_collective_radix); + else + assert(stage_notifications[stage] < shard_collective_last_radix); +#endif + stage_notifications[stage]++; + // Increment our guard to prevent deletion of the collective + // object while we are still traversing + pending_send_ready_stages++; + } + } + + //-------------------------------------------------------------------------- + template + void AllGatherCollective::complete_exchange(void) + //-------------------------------------------------------------------------- + { + if ((reorder_stages != NULL) && !reorder_stages->empty()) + { +#ifdef DEBUG_LEGION + assert(reorder_stages->size() == 1); +#endif + std::map > >::iterator + remaining = reorder_stages->begin(); + for (std::vector >::const_iterator it = + remaining->second.begin(); it != remaining->second.end(); it++) + { + Deserializer derez(it->first, it->second); + unpack_collective_stage(derez, remaining->first); + free(it->first); + } + reorder_stages->erase(remaining); + } + // See if we have to send a message back to a non-participating shard + if ((int(manager->total_shards) > shard_collective_participating_shards) + && (int(local_shard) < int(manager->total_shards - + shard_collective_participating_shards))) + send_remainder_stage(); + // Only after we send this message can we mark that we're done + Runtime::trigger_event(done_event); + } + + ///////////////////////////////////////////////////////////// + // All Reduce Op Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + AllReduceOpCollective::AllReduceOpCollective(CollectiveIndexLocation loc, + ReplicateContext *ctx, const ReductionOp *op) + : AllGatherCollective(loc, ctx), redop(op), current_stage(-1), + value(malloc(op->sizeof_rhs)) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + AllReduceOpCollective::AllReduceOpCollective(ReplicateContext *ctx, + CollectiveID id, const ReductionOp* op) + : AllGatherCollective(ctx, id), redop(op), current_stage(-1), + value(malloc(op->sizeof_rhs)) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + AllReduceOpCollective::~AllReduceOpCollective(void) + //-------------------------------------------------------------------------- + { + while (!future_values.empty()) + { + std::map >::iterator next = + future_values.begin(); + for (std::vector::iterator it = + next->second.begin(); it != next->second.end(); it++) + free(*it); + future_values.erase(next); + } + free(value); + } + + //-------------------------------------------------------------------------- + void AllReduceOpCollective::pack_collective_stage(Serializer &rez,int stage) + //-------------------------------------------------------------------------- + { + // The first time we pack a stage we merge any values that we had + // unpacked earlier as they are needed for sending this stage for + // the first time. + if (stage != current_stage) + { + if (!future_values.empty()) + { + std::map >::iterator next = + future_values.begin(); + if (next->first == current_stage) + { + for (std::vector::const_iterator it = + next->second.begin(); it != next->second.end(); it++) + { + redop->fold(value, *it, 1/*count*/, true/*exclusive*/); + free(*it); + } + future_values.erase(next); + } + } + current_stage = stage; + } + rez.serialize(value, redop->sizeof_rhs); + } + + //-------------------------------------------------------------------------- + void AllReduceOpCollective::unpack_collective_stage( + Deserializer &derez, int stage) + //-------------------------------------------------------------------------- + { + // We never eagerly do reductions as they can arrive out of order + // and we can't apply them too early or we'll get duplicate + // applications of reductions + void *next = malloc(redop->sizeof_rhs); + derez.deserialize(next, redop->sizeof_rhs); + future_values[stage].push_back(next); + } + + //-------------------------------------------------------------------------- + RtEvent AllReduceOpCollective::async_reduce(const void *input) + //-------------------------------------------------------------------------- + { + memcpy(value, input, redop->sizeof_rhs); + perform_collective_async(); + return perform_collective_wait(false/*block*/); + } + + //-------------------------------------------------------------------------- + void AllReduceOpCollective::sync_result(void *result) + //-------------------------------------------------------------------------- + { + perform_collective_wait(true/*block*/); + // Need to avoid races here so we have to always recompute the last stage + memcpy(result, value, redop->sizeof_rhs); + if (!future_values.empty()) + { +#ifdef DEBUG_LEGION + // Should be at most one stage left + assert(future_values.size() == 1); +#endif + const std::map >::const_iterator last = + future_values.begin(); + if (last->first == -1) + { + // Special case for the last stage which already includes our + // value so just do the overwrite +#ifdef DEBUG_LEGION + assert(last->second.size() == 1); +#endif + memcpy(result, last->second.front(), redop->sizeof_rhs); + } + else + { + // Do the reduction here + for (std::vector::const_iterator it = + last->second.begin(); it != last->second.end(); it++) + redop->fold(result, *it, 1/*count*/, true/*exclusive*/); + } + } + } + + ///////////////////////////////////////////////////////////// + // All Reduce Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + template + AllReduceCollective::AllReduceCollective(CollectiveIndexLocation loc, + ReplicateContext *ctx) + : AllGatherCollective(loc, ctx), current_stage(-1) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + template + AllReduceCollective::AllReduceCollective(ReplicateContext *ctx, + CollectiveID id) + : AllGatherCollective(ctx, id), current_stage(-1) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + template + AllReduceCollective::~AllReduceCollective(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + template + void AllReduceCollective::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + // The first time we pack a stage we merge any values that we had + // unpacked earlier as they are needed for sending this stage for + // the first time. + if (stage != current_stage) + { + if (!future_values.empty()) + { + typename std::map >::iterator + next = future_values.begin(); + if (next->first == current_stage) + { + for (typename std::vector::const_iterator it = + next->second.begin(); it != next->second.end(); it++) + REDOP::template fold(value, *it); + future_values.erase(next); + } + } + current_stage = stage; + } + rez.serialize(value); + } + + //-------------------------------------------------------------------------- + template + void AllReduceCollective::unpack_collective_stage( + Deserializer &derez, int stage) + //-------------------------------------------------------------------------- + { + // We never eagerly do reductions as they can arrive out of order + // and we can't apply them too early or we'll get duplicate + // applications of reductions + typename REDOP::RHS next; + derez.deserialize(next); + future_values[stage].push_back(next); + } + + //-------------------------------------------------------------------------- + template + void AllReduceCollective::async_all_reduce(typename REDOP::RHS val) + //-------------------------------------------------------------------------- + { + value = val; + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + template + RtEvent AllReduceCollective::wait_all_reduce(bool block) + //-------------------------------------------------------------------------- + { + return perform_collective_wait(block); + } + + //-------------------------------------------------------------------------- + template + typename REDOP::RHS AllReduceCollective::sync_all_reduce( + typename REDOP::RHS val) + //-------------------------------------------------------------------------- + { + async_all_reduce(val); + return get_result(); + } + + //-------------------------------------------------------------------------- + template + typename REDOP::RHS AllReduceCollective::get_result(void) + //-------------------------------------------------------------------------- + { + // Wait for the results to be ready + wait_all_reduce(true); + // Need to avoid races here so we have to always recompute the last stage + typename REDOP::RHS result = value; + if (!future_values.empty()) + { +#ifdef DEBUG_LEGION + // Should be at most one stage left + assert(future_values.size() == 1); +#endif + const typename std::map >:: + const_iterator last = future_values.begin(); + if (last->first == -1) + { + // Special case for the last stage which already includes our + // value so just do the overwrite +#ifdef DEBUG_LEGION + assert(last->second.size() == 1); +#endif + result = last->second.front(); + } + else + { + // Do the reduction here + for (typename std::vector::const_iterator it = + last->second.begin(); it != last->second.end(); it++) + REDOP::template fold(result, *it); + } + } + return result; + } + + ///////////////////////////////////////////////////////////// + // Barrier Exchange Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + template + BarrierExchangeCollective::BarrierExchangeCollective( + ReplicateContext *ctx, size_t win_size, + typename std::vector &bars, CollectiveIndexLocation loc) + : AllGatherCollective(loc, ctx), window_size(win_size), barriers(bars) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + template + BarrierExchangeCollective::BarrierExchangeCollective( + const BarrierExchangeCollective &rhs) + : AllGatherCollective(rhs), window_size(0), barriers(rhs.barriers) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + template + BarrierExchangeCollective::~BarrierExchangeCollective(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + template + BarrierExchangeCollective& BarrierExchangeCollective::operator=( + const BarrierExchangeCollective &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + template + void BarrierExchangeCollective::exchange_barriers_async(void) + //-------------------------------------------------------------------------- + { + // First make our local barriers and put them in the data structure + { + AutoLock c_lock(collective_lock); + for (unsigned index = local_shard; + index < window_size; index += manager->total_shards) + { +#ifdef DEBUG_LEGION + assert(local_barriers.find(index) == local_barriers.end()); +#endif + local_barriers[index] = + BAR(Realm::Barrier::create_barrier(manager->total_shards)); + } + } + // Now we can start the exchange from this shard + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + template + void BarrierExchangeCollective::wait_for_barrier_exchange(void) + //-------------------------------------------------------------------------- + { + // Wait for everything to be done + perform_collective_wait(); +#ifdef DEBUG_LEGION + assert(local_barriers.size() == window_size); +#endif + // Fill in the barrier vector with the barriers we've got from everyone + barriers.resize(window_size); + for (typename std::map::const_iterator it = + local_barriers.begin(); it != local_barriers.end(); it++) + { +#ifdef DEBUG_LEGION + assert(it->first < window_size); +#endif + barriers[it->first] = it->second; + } + } + + //-------------------------------------------------------------------------- + template + void BarrierExchangeCollective::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(window_size); + rez.serialize(local_barriers.size()); + for (typename std::map::const_iterator it = + local_barriers.begin(); it != local_barriers.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + template + void BarrierExchangeCollective::unpack_collective_stage( + Deserializer &derez, int stage) + //-------------------------------------------------------------------------- + { + size_t other_window_size; + derez.deserialize(other_window_size); + if (other_window_size != window_size) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Context configurations for control replicated " + "task %s were assigned different maximum window sizes " + "of %zd and %zd by the mapper which is illegal.", + context->owner_task->get_task_name(), window_size, + other_window_size) + size_t num_bars; + derez.deserialize(num_bars); + for (unsigned idx = 0; idx < num_bars; idx++) + { + unsigned index; + derez.deserialize(index); + derez.deserialize(local_barriers[index]); + } + } + + // Explicit instantiation of our two kinds of barriers + template class BarrierExchangeCollective; + template class BarrierExchangeCollective; + + ///////////////////////////////////////////////////////////// + // Buffer Broadcast + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + void BufferBroadcast::broadcast(void *b, size_t s, bool copy) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(buffer == NULL); +#endif + if (copy) + { + size = s; + buffer = malloc(size); + memcpy(buffer, b, size); + own = true; + } + else + { + buffer = b; + size = s; + own = false; + } + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + const void* BufferBroadcast::get_buffer(size_t &s, bool wait) + //-------------------------------------------------------------------------- + { + if (wait) + perform_collective_wait(); + s = size; + return buffer; + } + + //-------------------------------------------------------------------------- + void BufferBroadcast::pack_collective(Serializer &rez) const + //-------------------------------------------------------------------------- + { + rez.serialize(size); + if (size > 0) + rez.serialize(buffer, size); + } + + //-------------------------------------------------------------------------- + void BufferBroadcast::unpack_collective(Deserializer &derez) + //-------------------------------------------------------------------------- + { + derez.deserialize(size); + if (size > 0) + { +#ifdef DEBUG_LEGION + assert(buffer == NULL); +#endif + buffer = malloc(size); + derez.deserialize(buffer, size); + own = true; + } + } + + ///////////////////////////////////////////////////////////// + // Shard Sync Tree + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardSyncTree::ShardSyncTree(ReplicateContext *ctx, ShardID origin, + CollectiveIndexLocation loc) + : BroadcastCollective(loc, ctx, origin), + is_origin(origin == ctx->owner_shard->shard_id) + //-------------------------------------------------------------------------- + { + if (is_origin) + { + // All we need to do is the broadcast and then wait for + // everything to be done + perform_collective_async(); + // Now wait for the result to be ready + if (!done_preconditions.empty()) + { + RtEvent ready = Runtime::merge_events(done_preconditions); + ready.wait(); + } + } + } + + //-------------------------------------------------------------------------- + ShardSyncTree::~ShardSyncTree(void) + //-------------------------------------------------------------------------- + { + if (!is_origin) + { + // Perform the collective wait + perform_collective_wait(); + // Trigger our done event when all the preconditions are ready +#ifdef DEBUG_LEGION + assert(done_event.exists()); +#endif + if (!done_preconditions.empty()) + Runtime::trigger_event(done_event, + Runtime::merge_events(done_preconditions)); + else + Runtime::trigger_event(done_event); + } + } + + //-------------------------------------------------------------------------- + void ShardSyncTree::pack_collective(Serializer &rez) const + //-------------------------------------------------------------------------- + { + RtUserEvent next = Runtime::create_rt_user_event(); + rez.serialize(next); + done_preconditions.insert(next); + } + + //-------------------------------------------------------------------------- + void ShardSyncTree::unpack_collective(Deserializer &derez) + //-------------------------------------------------------------------------- + { + derez.deserialize(done_event); + } + + ///////////////////////////////////////////////////////////// + // Shard Event Tree + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardEventTree::ShardEventTree(ReplicateContext *ctx, ShardID origin, + CollectiveID id) + : BroadcastCollective(ctx, id, origin), + is_origin(origin == ctx->owner_shard->shard_id) + //-------------------------------------------------------------------------- + { + if (!is_origin) + { + local_event = Runtime::create_rt_user_event(); + trigger_event = local_event; + } + } + + //-------------------------------------------------------------------------- + ShardEventTree::~ShardEventTree(void) + //-------------------------------------------------------------------------- + { + if (finished_event.exists() && !finished_event.has_triggered()) + finished_event.wait(); + } + + //-------------------------------------------------------------------------- + void ShardEventTree::signal_tree(RtEvent precondition) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(is_origin); + assert(!trigger_event.exists()); +#endif + trigger_event = precondition; + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + RtEvent ShardEventTree::get_local_event(void) + //-------------------------------------------------------------------------- + { + finished_event = perform_collective_wait(false/*block*/); + return local_event; + } + + //-------------------------------------------------------------------------- + void ShardEventTree::pack_collective(Serializer &rez) const + //-------------------------------------------------------------------------- + { + rez.serialize(trigger_event); + } + + //-------------------------------------------------------------------------- + void ShardEventTree::unpack_collective(Deserializer &derez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(local_event.exists()); +#endif + RtEvent precondition; + derez.deserialize(precondition); + Runtime::trigger_event(local_event, precondition); + } + + ///////////////////////////////////////////////////////////// + // Cross Product Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + CrossProductCollective::CrossProductCollective(ReplicateContext *ctx, + CollectiveIndexLocation loc) + : AllGatherCollective(loc, ctx) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + CrossProductCollective::CrossProductCollective( + const CrossProductCollective &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + CrossProductCollective::~CrossProductCollective(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + CrossProductCollective& CrossProductCollective::operator=( + const CrossProductCollective &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void CrossProductCollective::exchange_partitions( + std::map &handles) + //-------------------------------------------------------------------------- + { + // Need the lock in case we are unpacking other things here + { + AutoLock c_lock(collective_lock); + // Only put the non-empty partitions into our local set + for (std::map::const_iterator it = + handles.begin(); it != handles.end(); it++) + { + if (!it->second.exists()) + continue; + non_empty_handles.insert(*it); + } + } + // Now we do the exchange + perform_collective_sync(); + // When we wake up we should have all the handles and no need the lock + // to access them +#ifdef DEBUG_LEGION + assert(handles.size() == non_empty_handles.size()); +#endif + handles = non_empty_handles; + } + + //-------------------------------------------------------------------------- + void CrossProductCollective::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(non_empty_handles.size()); + for (std::map::const_iterator it = + non_empty_handles.begin(); it != non_empty_handles.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + void CrossProductCollective::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + size_t num_handles; + derez.deserialize(num_handles); + for (unsigned idx = 0; idx < num_handles; idx++) + { + IndexSpace handle; + derez.deserialize(handle); + derez.deserialize(non_empty_handles[handle]); + } + } + + ///////////////////////////////////////////////////////////// + // Sharding Gather Collective + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardingGatherCollective::ShardingGatherCollective(ReplicateContext *ctx, + ShardID target, CollectiveIndexLocation loc) + : GatherCollective(loc, ctx, target) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardingGatherCollective::ShardingGatherCollective( + const ShardingGatherCollective &rhs) + : GatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ShardingGatherCollective::~ShardingGatherCollective(void) + //-------------------------------------------------------------------------- + { + // Make sure that we wait in case we still have messages to pass on + perform_collective_wait(); + } + + //-------------------------------------------------------------------------- + ShardingGatherCollective& ShardingGatherCollective::operator=( + const ShardingGatherCollective &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ShardingGatherCollective::pack_collective(Serializer &rez) const + //-------------------------------------------------------------------------- + { + rez.serialize(results.size()); + for (std::map::const_iterator it = + results.begin(); it != results.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + void ShardingGatherCollective::unpack_collective(Deserializer &derez) + //-------------------------------------------------------------------------- + { + size_t num_results; + derez.deserialize(num_results); + for (unsigned idx = 0; idx < num_results; idx++) + { + ShardID shard; + derez.deserialize(shard); + derez.deserialize(results[shard]); + } + } + + //-------------------------------------------------------------------------- + void ShardingGatherCollective::contribute(ShardingID value) + //-------------------------------------------------------------------------- + { + { + AutoLock c_lock(collective_lock); +#ifdef DEBUG_LEGION + assert(results.find(local_shard) == results.end()); +#endif + results[local_shard] = value; + } + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + bool ShardingGatherCollective::validate(ShardingID value) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(is_target()); +#endif + // Wait for the results + perform_collective_wait(); + for (std::map::const_iterator it = + results.begin(); it != results.end(); it++) + { + if (it->second != value) + return false; + } + return true; + } + + ///////////////////////////////////////////////////////////// + // Indirect Record Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + IndirectRecordExchange::IndirectRecordExchange(ReplicateContext *ctx, + CollectiveIndexLocation loc) + : AllGatherCollective(loc, ctx) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + IndirectRecordExchange::IndirectRecordExchange( + const IndirectRecordExchange &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + IndirectRecordExchange::~IndirectRecordExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + IndirectRecordExchange& IndirectRecordExchange::operator=( + const IndirectRecordExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void IndirectRecordExchange::exchange_records( + LegionVector::aligned &local_records) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(records.empty()); +#endif + for (LegionVector::aligned::const_iterator it = + local_records.begin(); it != local_records.end(); it++) + { + const IndirectKey key(it->inst, it->ready_event, it->domain); + records[key] = it->fields; + } + perform_collective_sync(); + local_records.resize(records.size()); + unsigned index = 0; + for (LegionMap::aligned::const_iterator it = + records.begin(); it != records.end(); it++, index++) + { + IndirectRecord &record = local_records[index]; + record.inst = it->first.inst; + record.ready_event = it->first.ready_event; + record.domain = it->first.domain; + record.fields = it->second; + } + } + + //-------------------------------------------------------------------------- + void IndirectRecordExchange::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(records.size()); + for (LegionMap::aligned::const_iterator it = + records.begin(); it != records.end(); it++) + { + rez.serialize(it->first.inst); + rez.serialize(it->first.ready_event); + rez.serialize(it->first.domain); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + void IndirectRecordExchange::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + size_t num_records; + derez.deserialize(num_records); + for (unsigned idx = 0; idx < num_records; idx++) + { + IndirectKey key; + derez.deserialize(key.inst); + derez.deserialize(key.ready_event); + derez.deserialize(key.domain); + LegionMap::aligned::iterator finder = + records.find(key); + if (finder != records.end()) + { + FieldMask mask; + derez.deserialize(mask); + finder->second |= mask; + } + else + derez.deserialize(records[key]); + } + } + + ///////////////////////////////////////////////////////////// + // Field Descriptor Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + FieldDescriptorExchange::FieldDescriptorExchange(ReplicateContext *ctx, + CollectiveIndexLocation loc) + : AllGatherCollective(loc, ctx) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + FieldDescriptorExchange::FieldDescriptorExchange( + const FieldDescriptorExchange &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + FieldDescriptorExchange::~FieldDescriptorExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + FieldDescriptorExchange& FieldDescriptorExchange::operator=( + const FieldDescriptorExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + ApEvent FieldDescriptorExchange::exchange_descriptors(ApEvent ready_event, + const std::vector &descs) + //-------------------------------------------------------------------------- + { + { + AutoLock c_lock(collective_lock); + ready_events.insert(ready_event); + descriptors.insert(descriptors.end(), descs.begin(), descs.end()); + if (participating) + { + remote_to_trigger.resize(shard_collective_stages + 1); + local_preconditions.resize(shard_collective_stages + 1); + } + else + { + remote_to_trigger.resize(1); + local_preconditions.resize(1); + } + } + perform_collective_sync(); + return Runtime::merge_events(NULL, ready_events); + } + + //-------------------------------------------------------------------------- + ApEvent FieldDescriptorExchange::exchange_completion(ApEvent complete) + //-------------------------------------------------------------------------- + { + if (participating) + { + // Might have a precondition from a remainder shard + if (!local_preconditions[0].empty()) + { +#ifdef DEBUG_LEGION + assert(local_preconditions[0].size() == 1); +#endif + complete = Runtime::merge_events(NULL, complete, + *(local_preconditions[0].begin())); + } + const std::set &to_trigger = remote_to_trigger[0]; + for (std::set::const_iterator it = + to_trigger.begin(); it != to_trigger.end(); it++) + Runtime::trigger_event(NULL, *it, complete); + const ApEvent done = + Runtime::merge_events(NULL, local_preconditions.back()); + // If we have a remainder shard then we need to signal them too + if (!remote_to_trigger[shard_collective_stages].empty()) + { +#ifdef DEBUG_LEGION + assert(remote_to_trigger[shard_collective_stages].size() == 1); +#endif + Runtime::trigger_event(NULL, + *(remote_to_trigger[shard_collective_stages].begin()), done); + } + return done; + } + else + { + // Not participating so we should have exactly one thing to + // trigger and one precondition for being done +#ifdef DEBUG_LEGION + assert(remote_to_trigger[0].size() == 1); + assert(local_preconditions[0].size() == 1); +#endif + Runtime::trigger_event(NULL, *(remote_to_trigger[0].begin()), complete); + return *(local_preconditions[0].begin()); + } + } + + //-------------------------------------------------------------------------- + void FieldDescriptorExchange::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + // Always make a stage precondition and send it back + ApUserEvent stage_complete = Runtime::create_ap_user_event(NULL); + rez.serialize(stage_complete); + if (stage == -1) + { +#ifdef DEBUG_LEGION + assert(!local_preconditions.empty()); + assert(local_preconditions[0].empty()); +#endif + // Always save this as a precondition for later + local_preconditions[0].insert(stage_complete); + } + else + { +#ifdef DEBUG_LEGION + assert(participating); + assert(stage < shard_collective_stages); +#endif + std::set &preconditions = + local_preconditions[shard_collective_stages - stage]; + preconditions.insert(stage_complete); + // See if we've sent all our messages in which case we can + // trigger all the remote user events for any previous stages + if (((stage == (shard_collective_stages-1)) && + (int(preconditions.size()) == shard_collective_last_radix)) || + ((stage < (shard_collective_stages-1)) && + (int(preconditions.size()) == shard_collective_radix))) + { + const std::set &to_trigger = + remote_to_trigger[(stage > 0) ? (stage-1) : shard_collective_stages]; + // Check for empty which can happen with stage 0 if there + // are no remainders + if (!to_trigger.empty()) + { + const ApEvent stage_pre = Runtime::merge_events(NULL,preconditions); + for (std::set::const_iterator it = + to_trigger.begin(); it != to_trigger.end(); it++) + Runtime::trigger_event(NULL, *it, stage_pre); + } + } + } + rez.serialize(ready_events.size()); + for (std::set::const_iterator it = ready_events.begin(); + it != ready_events.end(); it++) + rez.serialize(*it); + rez.serialize(descriptors.size()); + for (std::vector::const_iterator it = + descriptors.begin(); it != descriptors.end(); it++) + rez.serialize(*it); + } + + //-------------------------------------------------------------------------- + void FieldDescriptorExchange::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + ApUserEvent remote_complete; + derez.deserialize(remote_complete); + if (stage == -1) + { +#ifdef DEBUG_LEGION + assert(!remote_to_trigger.empty()); +#endif + if (participating) + { +#ifdef DEBUG_LEGION + assert(remote_to_trigger[shard_collective_stages].empty()); +#endif + remote_to_trigger[shard_collective_stages].insert(remote_complete); + } + else + { +#ifdef DEBUG_LEGION + assert(remote_to_trigger[0].empty()); +#endif + remote_to_trigger[0].insert(remote_complete); + } + } + else + { +#ifdef DEBUG_LEGION + assert(participating); + assert(stage < int(remote_to_trigger.size())); +#endif + remote_to_trigger[stage].insert(remote_complete); + } + size_t num_events; + derez.deserialize(num_events); + for (unsigned idx = 0; idx < num_events; idx++) + { + ApEvent ready; + derez.deserialize(ready); + ready_events.insert(ready); + } + unsigned offset = descriptors.size(); + size_t num_descriptors; + derez.deserialize(num_descriptors); + descriptors.resize(offset + num_descriptors); + for (unsigned idx = 0; idx < num_descriptors; idx++) + derez.deserialize(descriptors[offset + idx]); + } + + ///////////////////////////////////////////////////////////// + // Field Descriptor Gather + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + FieldDescriptorGather::FieldDescriptorGather(ReplicateContext *ctx, + ShardID target, CollectiveIndexLocation loc) + : GatherCollective(loc, ctx, target), used(false) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + FieldDescriptorGather::FieldDescriptorGather( + const FieldDescriptorGather &rhs) + : GatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + FieldDescriptorGather::~FieldDescriptorGather(void) + //-------------------------------------------------------------------------- + { + // Make sure that we wait in case we still have messages to pass on + if (used) + perform_collective_wait(); +#ifdef DEBUG_LEGION + assert(!complete_event.exists() || complete_event.has_triggered()); +#endif + } + + //-------------------------------------------------------------------------- + FieldDescriptorGather& FieldDescriptorGather::operator=( + const FieldDescriptorGather &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void FieldDescriptorGather::pack_collective(Serializer &rez) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(complete_event.exists()); +#endif + // Trigger any remote complete events we have dependent on our event + if (!remote_complete_events.empty()) + { + for (std::set::const_iterator it = + remote_complete_events.begin(); it != + remote_complete_events.end(); it++) + Runtime::trigger_event(NULL, *it, complete_event); + } + rez.serialize(complete_event); + rez.serialize(ready_events.size()); + for (std::set::const_iterator it = ready_events.begin(); + it != ready_events.end(); it++) + rez.serialize(*it); + rez.serialize(descriptors.size()); + for (std::vector::const_iterator it = + descriptors.begin(); it != descriptors.end(); it++) + rez.serialize(*it); + } + + //-------------------------------------------------------------------------- + void FieldDescriptorGather::unpack_collective(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ApUserEvent remote_complete; + derez.deserialize(remote_complete); + remote_complete_events.insert(remote_complete); + size_t num_events; + derez.deserialize(num_events); + for (unsigned idx = 0; idx < num_events; idx++) + { + ApEvent ready; + derez.deserialize(ready); + ready_events.insert(ready); + } + unsigned offset = descriptors.size(); + size_t num_descriptors; + derez.deserialize(num_descriptors); + descriptors.resize(offset + num_descriptors); + for (unsigned idx = 0; idx < num_descriptors; idx++) + derez.deserialize(descriptors[offset + idx]); + } + + //-------------------------------------------------------------------------- + void FieldDescriptorGather::contribute(ApEvent ready_event, + const std::vector &descs) + //-------------------------------------------------------------------------- + { + used = true; + { + AutoLock c_lock(collective_lock); + ready_events.insert(ready_event); + descriptors.insert(descriptors.end(), descs.begin(), descs.end()); + // If we're not the owner make our complete event +#ifdef DEBUG_LEGION + assert(!complete_event.exists()); +#endif + if (!is_target()) + complete_event = Runtime::create_ap_user_event(NULL); + } + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + const std::vector& + FieldDescriptorGather::get_full_descriptors(ApEvent &ready) + //-------------------------------------------------------------------------- + { + perform_collective_wait(); + ready = Runtime::merge_events(NULL, ready_events); + return descriptors; + } + + //-------------------------------------------------------------------------- + void FieldDescriptorGather::notify_remote_complete(ApEvent precondition) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(is_target()); +#endif + if (!remote_complete_events.empty()) + { + for (std::set::const_iterator it = + remote_complete_events.begin(); it != + remote_complete_events.end(); it++) + Runtime::trigger_event(NULL, *it, precondition); + } + } + + //-------------------------------------------------------------------------- + ApEvent FieldDescriptorGather::get_complete_event(void) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!is_target()); + assert(complete_event.exists()); +#endif + return complete_event; + } + + ///////////////////////////////////////////////////////////// + // Future Broadcast + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + FutureBroadcast::FutureBroadcast(ReplicateContext *ctx, CollectiveID id, + ShardID source, FutureImpl *i) + : BroadcastCollective(ctx, id, source), impl(i) + //-------------------------------------------------------------------------- + { + if (source == ctx->owner_shard->shard_id) + ready = impl->subscribe_internal(); + } + + //-------------------------------------------------------------------------- + FutureBroadcast::FutureBroadcast(const FutureBroadcast &rhs) + : BroadcastCollective(rhs), impl(rhs.impl) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + FutureBroadcast::~FutureBroadcast(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + FutureBroadcast& FutureBroadcast::operator=(const FutureBroadcast &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void FutureBroadcast::pack_collective(Serializer &rez) const + //-------------------------------------------------------------------------- + { + const size_t result_size = impl->get_untyped_size(true/*internal*/); + rez.serialize(result_size); + if (result_size > 0) + rez.serialize(impl->get_untyped_result(true, NULL, true), result_size); + } + + //-------------------------------------------------------------------------- + void FutureBroadcast::unpack_collective(Deserializer &derez) + //-------------------------------------------------------------------------- + { + size_t result_size; + derez.deserialize(result_size); + if (result_size > 0) + { + const void *ptr = derez.get_current_pointer(); + impl->set_result(ptr, result_size, false/*owned*/); + derez.advance_pointer(result_size); + } + else + impl->set_result(NULL, 0, false/*owned*/); + } + + //-------------------------------------------------------------------------- + void FutureBroadcast::broadcast_future(void) + //-------------------------------------------------------------------------- + { + if (ready.exists() && !ready.has_triggered()) + ready.wait(); + perform_collective_async(); + } + + ///////////////////////////////////////////////////////////// + // Future Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + FutureExchange::FutureExchange(ReplicateContext *ctx, size_t size, + CollectiveIndexLocation loc) + : AllGatherCollective(loc, ctx), future_size(size) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + FutureExchange::FutureExchange(const FutureExchange &rhs) + : AllGatherCollective(rhs), future_size(0) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + FutureExchange::~FutureExchange(void) + //-------------------------------------------------------------------------- + { + // Delete all the futures except our local shard one since we know + // that we don't actually own that memory + for (std::map::const_iterator it = results.begin(); + it != results.end(); it++) + free(it->second); + } + + //-------------------------------------------------------------------------- + FutureExchange& FutureExchange::operator=(const FutureExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void FutureExchange::pack_collective_stage(Serializer &rez, int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(results.size()); + for (std::map::const_iterator it = results.begin(); + it != results.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second, future_size); + } + } + + //-------------------------------------------------------------------------- + void FutureExchange::unpack_collective_stage(Deserializer &derez, int stage) + //-------------------------------------------------------------------------- + { + size_t num_results; + derez.deserialize(num_results); + for (unsigned idx = 0; idx < num_results; idx++) + { + ShardID shard; + derez.deserialize(shard); + if (results.find(shard) != results.end()) + { + derez.advance_pointer(future_size); + continue; + } + void *buffer = malloc(future_size); + derez.deserialize(buffer, future_size); + results[shard] = buffer; + } + } + + //-------------------------------------------------------------------------- + RtEvent FutureExchange::exchange_futures(void *value) + //-------------------------------------------------------------------------- + { + { + AutoLock c_lock(collective_lock); +#ifdef DEBUG_LEGION + assert(results.find(local_shard) == results.end()); +#endif + results[local_shard] = value; + } + perform_collective_async(); + return perform_collective_wait(false/*block*/); + } + + //-------------------------------------------------------------------------- + void FutureExchange::reduce_futures(ReplIndexTask *target) + //-------------------------------------------------------------------------- + { + // Now we apply the shard results in order to ensure that we get + // the same bitwise order across all the shards + // No need for the lock anymore since we know we're done + for (std::map::const_iterator it = results.begin(); + it != results.end(); it++) + target->fold_reduction_future(it->second, future_size, + false/*owner*/, true/*exclusive*/); + } + + //-------------------------------------------------------------------------- + void FutureExchange::reduce_futures(const ReductionOp *redop, + void *result_buffer) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(future_size == redop->sizeof_rhs); +#endif + redop->init(result_buffer, 1/*count*/); + // Now we apply the shard results in order to ensure that we get + // the same bitwise order across all the shards + // No need for the lock anymore since we know we're done + for (std::map::const_iterator it = results.begin(); + it != results.end(); it++) + redop->fold(result_buffer, it->second, 1/*count*/, true/*exclusive*/); + } + + ///////////////////////////////////////////////////////////// + // Future Name Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + FutureNameExchange::FutureNameExchange(ReplicateContext *ctx, + CollectiveID id, ReplFutureMapImpl *m, ReferenceMutator *mut) + : AllGatherCollective(ctx, id), future_map(m), mutator(mut) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + FutureNameExchange::FutureNameExchange(const FutureNameExchange &rhs) + : AllGatherCollective(rhs), future_map(rhs.future_map), + mutator(rhs.mutator) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + FutureNameExchange::~FutureNameExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + FutureNameExchange& FutureNameExchange::operator=( + const FutureNameExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void FutureNameExchange::pack_collective_stage(Serializer &rez, int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(results.size()); + for (std::map::const_iterator it = + results.begin(); it != results.end(); it++) + { + rez.serialize(it->first); + if (it->second.impl != NULL) + rez.serialize(it->second.impl->did); + else + rez.serialize(0); + } + } + + //-------------------------------------------------------------------------- + void FutureNameExchange::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + size_t num_futures; + derez.deserialize(num_futures); + for (unsigned idx = 0; idx < num_futures; idx++) + { + DomainPoint point; + derez.deserialize(point); + DistributedID did; + derez.deserialize(did); + if (did > 0) + { + FutureImpl *impl = + context->runtime->find_or_create_future(did, mutator, + future_map->op, future_map->op_gen, +#ifdef LEGION_SPY + future_map->op_uid, +#endif + future_map->op_depth); + // Add the reference ourselves so we can capture the effects + impl->add_base_gc_ref(FUTURE_HANDLE_REF, mutator); + results[point] = Future(impl, false/*need referece*/); + } + else + results[point] = Future(); + } + } + + //-------------------------------------------------------------------------- + void FutureNameExchange::exchange_future_names( + std::map &futures) + //-------------------------------------------------------------------------- + { + { + AutoLock c_lock(collective_lock); + results.insert(futures.begin(), futures.end()); + } + perform_collective_sync(); + futures = results; + } + + ///////////////////////////////////////////////////////////// + // Must Epoch Processor Broadcast + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + MustEpochMappingBroadcast::MustEpochMappingBroadcast( + ReplicateContext *ctx, ShardID origin, CollectiveID collective_id) + : BroadcastCollective(ctx, collective_id, origin) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + MustEpochMappingBroadcast::MustEpochMappingBroadcast( + const MustEpochMappingBroadcast &rhs) + : BroadcastCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + MustEpochMappingBroadcast::~MustEpochMappingBroadcast(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(local_done_event.exists()); +#endif + if (!done_events.empty()) + Runtime::trigger_event(local_done_event, + Runtime::merge_events(done_events)); + else + Runtime::trigger_event(local_done_event); + // This should only happen on the owner node + if (!held_references.empty()) + { + // Wait for all the other shards to be done + local_done_event.wait(); + // Now we can remove our held references + for (std::set::const_iterator it = + held_references.begin(); it != held_references.end(); it++) + if ((*it)->remove_base_valid_ref(REPLICATION_REF)) + delete (*it); + } + } + + //-------------------------------------------------------------------------- + MustEpochMappingBroadcast& MustEpochMappingBroadcast::operator=( + const MustEpochMappingBroadcast &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void MustEpochMappingBroadcast::pack_collective(Serializer &rez) const + //-------------------------------------------------------------------------- + { + RtUserEvent next_done = Runtime::create_rt_user_event(); + done_events.insert(next_done); + rez.serialize(next_done); + rez.serialize(processors.size()); + for (unsigned idx = 0; idx < processors.size(); idx++) + rez.serialize(processors[idx]); + rez.serialize(instances.size()); + for (unsigned idx = 0; idx < instances.size(); idx++) + { + const std::vector &dids = instances[idx]; + rez.serialize(dids.size()); + for (std::vector::const_iterator it = + dids.begin(); it != dids.end(); it++) + rez.serialize(*it); + } + } + + //-------------------------------------------------------------------------- + void MustEpochMappingBroadcast::unpack_collective(Deserializer &derez) + //-------------------------------------------------------------------------- + { + derez.deserialize(local_done_event); + size_t num_procs; + derez.deserialize(num_procs); + processors.resize(num_procs); + for (unsigned idx = 0; idx < num_procs; idx++) + derez.deserialize(processors[idx]); + size_t num_constraints; + derez.deserialize(num_constraints); + instances.resize(num_constraints); + for (unsigned idx1 = 0; idx1 < num_constraints; idx1++) + { + size_t num_dids; + derez.deserialize(num_dids); + std::vector &dids = instances[idx1]; + dids.resize(num_dids); + for (unsigned idx2 = 0; idx2 < num_dids; idx2++) + derez.deserialize(dids[idx2]); + } + } + + //-------------------------------------------------------------------------- + void MustEpochMappingBroadcast::broadcast( + const std::vector &processor_mapping, + const std::vector > &mappings) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!local_done_event.exists()); +#endif + local_done_event = Runtime::create_rt_user_event(); + processors = processor_mapping; + instances.resize(mappings.size()); + // Add valid references to all the physical instances that we will + // hold until all the must epoch operations are done with the exchange + WrapperReferenceMutator mutator(done_events); + for (unsigned idx1 = 0; idx1 < mappings.size(); idx1++) + { + std::vector &dids = instances[idx1]; + dids.resize(mappings[idx1].size()); + for (unsigned idx2 = 0; idx2 < dids.size(); idx2++) + { + const Mapping::PhysicalInstance &inst = mappings[idx1][idx2]; + PhysicalManager *manager = inst.impl->as_instance_manager(); + dids[idx2] = manager->did; + if (held_references.find(manager) != held_references.end()) + continue; + manager->add_base_valid_ref(REPLICATION_REF, &mutator); + held_references.insert(manager); + } + } + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + void MustEpochMappingBroadcast::receive_results( + std::vector &processor_mapping, + const std::vector &constraint_indexes, + std::vector > &mappings, + std::map &acquired) + //-------------------------------------------------------------------------- + { + perform_collective_wait(); + // Just grab all the processors since we still need them + processor_mapping = processors; + // We are a little smarter with the mappings since we know exactly + // which ones we are actually going to need for our local points + std::set ready_events; + Runtime *runtime = manager->runtime; + for (std::vector::const_iterator it = + constraint_indexes.begin(); it != constraint_indexes.end(); it++) + { +#ifdef DEBUG_LEGION + assert((*it) < instances.size()); + assert((*it) < mappings.size()); +#endif + const std::vector &dids = instances[*it]; + std::vector &mapping = mappings[*it]; + mapping.resize(dids.size()); + for (unsigned idx = 0; idx < dids.size(); idx++) + { + RtEvent ready; + mapping[idx].impl = + runtime->find_or_request_instance_manager(dids[idx], ready); + if (!ready.has_triggered()) + ready_events.insert(ready); + } + } + // Have to wait for the ready events to trigger before we can add + // our references safely + if (!ready_events.empty()) + { + RtEvent ready = Runtime::merge_events(ready_events); + if (!ready.has_triggered()) + ready.wait(); + } + // Lastly we need to put acquire references on any of local instances + WrapperReferenceMutator mutator(done_events); + for (unsigned idx = 0; idx < constraint_indexes.size(); idx++) + { + const unsigned constraint_index = constraint_indexes[idx]; + const std::vector &mapping = + mappings[constraint_index]; + // Also grab an acquired reference to these instances + for (std::vector::const_iterator it = + mapping.begin(); it != mapping.end(); it++) + { + PhysicalManager *manager = it->impl->as_instance_manager(); + // If we already had a reference to this instance + // then we don't need to add any additional ones + if (acquired.find(manager) != acquired.end()) + continue; + manager->add_base_resource_ref(INSTANCE_MAPPER_REF); + manager->add_base_valid_ref(MAPPING_ACQUIRE_REF, &mutator); + acquired[manager] = 1/*count*/; + } + } + } + + ///////////////////////////////////////////////////////////// + // Must Epoch Mapping Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + MustEpochMappingExchange::MustEpochMappingExchange(ReplicateContext *ctx, + CollectiveID collective_id) + : AllGatherCollective(ctx, collective_id) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + MustEpochMappingExchange::MustEpochMappingExchange( + const MustEpochMappingExchange &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + MustEpochMappingExchange::~MustEpochMappingExchange(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(local_done_event.exists()); // better have one of these +#endif + Runtime::trigger_event(local_done_event); + // See if we need to wait for others to be done before we can + // remove our valid references + if (!done_events.empty()) + { + RtEvent done = Runtime::merge_events(done_events); + if (!done.has_triggered()) + done.wait(); + } + // Now we can remove our held references + for (std::set::const_iterator it = + held_references.begin(); it != held_references.end(); it++) + if ((*it)->remove_base_valid_ref(REPLICATION_REF)) + delete (*it); + } + + //-------------------------------------------------------------------------- + MustEpochMappingExchange& MustEpochMappingExchange::operator=( + const MustEpochMappingExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void MustEpochMappingExchange::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(processors.size()); + for (std::map::const_iterator it = + processors.begin(); it != processors.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + rez.serialize(constraints.size()); + for (std::map::const_iterator it = + constraints.begin(); it != constraints.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second.instances.size()); + for (unsigned idx = 0; idx < it->second.instances.size(); idx++) + rez.serialize(it->second.instances[idx]); + rez.serialize(it->second.origin_shard); + rez.serialize(it->second.weight); + } + rez.serialize(done_events.size()); + for (std::set::const_iterator it = + done_events.begin(); it != done_events.end(); it++) + rez.serialize(*it); + } + + //-------------------------------------------------------------------------- + void MustEpochMappingExchange::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + size_t num_procs; + derez.deserialize(num_procs); + for (unsigned idx = 0; idx < num_procs; idx++) + { + DomainPoint point; + derez.deserialize(point); + derez.deserialize(processors[point]); + } + size_t num_mappings; + derez.deserialize(num_mappings); + for (unsigned idx1 = 0; idx1 < num_mappings; idx1++) + { + unsigned constraint_index; + derez.deserialize(constraint_index); + std::map::iterator + finder = constraints.find(constraint_index); + if (finder == constraints.end()) + { + // Can unpack directly since we're first + ConstraintInfo &info = constraints[constraint_index]; + size_t num_dids; + derez.deserialize(num_dids); + info.instances.resize(num_dids); + for (unsigned idx2 = 0; idx2 < num_dids; idx2++) + derez.deserialize(info.instances[idx2]); + derez.deserialize(info.origin_shard); + derez.deserialize(info.weight); + } + else + { + // Unpack into a temporary + ConstraintInfo info; + size_t num_dids; + derez.deserialize(num_dids); + info.instances.resize(num_dids); + for (unsigned idx2 = 0; idx2 < num_dids; idx2++) + derez.deserialize(info.instances[idx2]); + derez.deserialize(info.origin_shard); + derez.deserialize(info.weight); + // Only keep the result if we have a larger weight + // or we have the same weight and a smaller shard + if ((info.weight > finder->second.weight) || + ((info.weight == finder->second.weight) && + (info.origin_shard < finder->second.origin_shard))) + finder->second = info; + } + } + size_t num_done; + derez.deserialize(num_done); + for (unsigned idx = 0; idx < num_done; idx++) + { + RtEvent done_event; + derez.deserialize(done_event); + done_events.insert(done_event); + } + } + + //-------------------------------------------------------------------------- + void MustEpochMappingExchange::exchange_must_epoch_mappings( + ShardID shard_id, size_t total_shards, size_t total_constraints, + const std::vector &local_tasks, + const std::vector &all_tasks, + std::vector &processor_mapping, + const std::vector &constraint_indexes, + std::vector > &mappings, + const std::vector &mapping_weights, + std::map &acquired) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(local_tasks.size() == processor_mapping.size()); + assert(constraint_indexes.size() == mappings.size()); +#endif + // Add valid references to all the physical instances that we will + // hold until all the must epoch operations are done with the exchange + WrapperReferenceMutator mutator(done_events); + for (unsigned idx = 0; idx < mappings.size(); idx++) + { + for (std::vector::const_iterator it = + mappings[idx].begin(); it != mappings[idx].end(); it++) + { + PhysicalManager *manager = it->impl->as_instance_manager(); + if (held_references.find(manager) != held_references.end()) + continue; + manager->add_base_valid_ref(REPLICATION_REF, &mutator); + held_references.insert(manager); + } + } +#ifdef DEBUG_LEGION + assert(!local_done_event.exists()); +#endif + local_done_event = Runtime::create_rt_user_event(); + // Then we can add our instances to the set and do the exchange + { + AutoLock c_lock(collective_lock); + for (unsigned idx = 0; idx < local_tasks.size(); idx++) + { + const Task *task = local_tasks[idx]; +#ifdef DEBUG_LEGION + assert(processors.find(task->index_point) == processors.end()); +#endif + processors[task->index_point] = processor_mapping[idx]; + } + for (unsigned idx1 = 0; idx1 < mappings.size(); idx1++) + { + const unsigned constraint_index = constraint_indexes[idx1]; +#ifdef DEBUG_LEGION + assert(constraint_index < total_constraints); +#endif + std::map::iterator + finder = constraints.find(constraint_index); + // Only add it if it doesn't exist or it has a lower weight + // or it has the same weight and is a lower shard + if ((finder == constraints.end()) || + (mapping_weights[idx1] > finder->second.weight) || + ((mapping_weights[idx1] == finder->second.weight) && + (shard_id < finder->second.origin_shard))) + { + ConstraintInfo &info = constraints[constraint_index]; + info.instances.resize(mappings[idx1].size()); + for (unsigned idx2 = 0; idx2 < mappings[idx1].size(); idx2++) + info.instances[idx2] = mappings[idx1][idx2].impl->did; + info.origin_shard = shard_id; + info.weight = mapping_weights[idx1]; + } + } + // Also update the local done events + done_events.insert(local_done_event); + } + perform_collective_sync(); + // Start fetching the all the mapping results to get them in flight + mappings.clear(); + mappings.resize(total_constraints); + std::set ready_events; + Runtime *runtime = manager->runtime; + // We only need to get the results for local constraints as we + // know that we aren't going to care about any of the rest + for (unsigned idx1 = 0; idx1 < constraint_indexes.size(); idx1++) + { + const unsigned constraint_index = constraint_indexes[idx1]; + const std::vector &dids = + constraints[constraint_index].instances; + std::vector &mapping = + mappings[constraint_index]; + mapping.resize(dids.size()); + for (unsigned idx2 = 0; idx2 < dids.size(); idx2++) + { + RtEvent ready; + mapping[idx2].impl = + runtime->find_or_request_instance_manager(dids[idx2], ready); + if (!ready.has_triggered()) + ready_events.insert(ready); + } + } + // Update the processor mapping + processor_mapping.resize(all_tasks.size()); + for (unsigned idx = 0; idx < all_tasks.size(); idx++) + { + const Task *task = all_tasks[idx]; + std::map::const_iterator finder = + processors.find(task->index_point); +#ifdef DEBUG_LEGION + assert(finder != processors.end()); +#endif + processor_mapping[idx] = finder->second; + } + // Wait for all the instances to be ready + if (!ready_events.empty()) + { + RtEvent ready = Runtime::merge_events(ready_events); + if (!ready.has_triggered()) + ready.wait(); + } + // Lastly we need to put acquire references on any of local instances + for (unsigned idx = 0; idx < constraint_indexes.size(); idx++) + { + const unsigned constraint_index = constraint_indexes[idx]; + const std::vector &mapping = + mappings[constraint_index]; + // Also grab an acquired reference to these instances + for (std::vector::const_iterator it = + mapping.begin(); it != mapping.end(); it++) + { + PhysicalManager *manager = it->impl->as_instance_manager(); + // If we already had a reference to this instance + // then we don't need to add any additional ones + if (acquired.find(manager) != acquired.end()) + continue; + manager->add_base_resource_ref(INSTANCE_MAPPER_REF); + manager->add_base_valid_ref(MAPPING_ACQUIRE_REF, &mutator); + acquired[manager] = 1/*count*/; + } + } + } + + ///////////////////////////////////////////////////////////// + // Must Epoch Dependence Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + MustEpochDependenceExchange::MustEpochDependenceExchange( + ReplicateContext *ctx, CollectiveIndexLocation loc) + : AllGatherCollective(loc, ctx) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + MustEpochDependenceExchange::MustEpochDependenceExchange( + const MustEpochDependenceExchange &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + MustEpochDependenceExchange::~MustEpochDependenceExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + MustEpochDependenceExchange& MustEpochDependenceExchange::operator=( + const MustEpochDependenceExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void MustEpochDependenceExchange::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(mapping_dependences.size()); + for (std::map::const_iterator it = + mapping_dependences.begin(); it != mapping_dependences.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + void MustEpochDependenceExchange::unpack_collective_stage( + Deserializer &derez, int stage) + //-------------------------------------------------------------------------- + { + size_t num_deps; + derez.deserialize(num_deps); + for (unsigned idx = 0; idx < num_deps; idx++) + { + DomainPoint point; + derez.deserialize(point); + derez.deserialize(mapping_dependences[point]); + } + } + + //-------------------------------------------------------------------------- + void MustEpochDependenceExchange::exchange_must_epoch_dependences( + std::map &mapped_events) + //-------------------------------------------------------------------------- + { + { + AutoLock c_lock(collective_lock); + for (std::map::const_iterator it = + mapped_events.begin(); it != mapped_events.end(); it++) + mapping_dependences.insert(*it); + } + perform_collective_sync(); + // No need to hold the lock after the collective is complete + mapped_events.swap(mapping_dependences); + } + + ///////////////////////////////////////////////////////////// + // Must Epoch Completion Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + MustEpochCompletionExchange::MustEpochCompletionExchange( + ReplicateContext *ctx, CollectiveIndexLocation loc) + : AllGatherCollective(loc, ctx) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + MustEpochCompletionExchange::MustEpochCompletionExchange( + const MustEpochCompletionExchange &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + MustEpochCompletionExchange::~MustEpochCompletionExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + MustEpochCompletionExchange& MustEpochCompletionExchange::operator=( + const MustEpochCompletionExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void MustEpochCompletionExchange::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(tasks_mapped.size()); + for (std::set::const_iterator it = + tasks_mapped.begin(); it != tasks_mapped.end(); it++) + rez.serialize(*it); + rez.serialize(tasks_complete.size()); + for (std::set::const_iterator it = + tasks_complete.begin(); it != tasks_complete.end(); it++) + rez.serialize(*it); + } + + //-------------------------------------------------------------------------- + void MustEpochCompletionExchange::unpack_collective_stage( + Deserializer &derez, int stage) + //-------------------------------------------------------------------------- + { + size_t num_mapped; + derez.deserialize(num_mapped); + for (unsigned idx = 0; idx < num_mapped; idx++) + { + RtEvent mapped; + derez.deserialize(mapped); + tasks_mapped.insert(mapped); + } + size_t num_complete; + derez.deserialize(num_complete); + for (unsigned idx = 0; idx < num_complete; idx++) + { + ApEvent complete; + derez.deserialize(complete); + tasks_complete.insert(complete); + } + } + + //-------------------------------------------------------------------------- + void MustEpochCompletionExchange::exchange_must_epoch_completion( + RtEvent mapped, ApEvent complete, + std::set &all_mapped, + std::set &all_complete) + //-------------------------------------------------------------------------- + { + { + AutoLock c_lock(collective_lock); + tasks_mapped.insert(mapped); + tasks_complete.insert(complete); + } + perform_collective_sync(); + // No need to hold the lock after the collective is complete + all_mapped.swap(tasks_mapped); + all_complete.swap(tasks_complete); + } + + ///////////////////////////////////////////////////////////// + // Sharded Mapping Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardedMappingExchange::ShardedMappingExchange(CollectiveIndexLocation loc, + ReplicateContext *ctx, ShardID sid, bool check_map) + : AllGatherCollective(loc, ctx), shard_id(sid), check_mappings(check_map) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardedMappingExchange::ShardedMappingExchange( + const ShardedMappingExchange &i) + : AllGatherCollective(i), shard_id(0), check_mappings(false) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ShardedMappingExchange::~ShardedMappingExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardedMappingExchange& ShardedMappingExchange::operator=( + const ShardedMappingExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ShardedMappingExchange::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(mappings.size()); + for (std::map::aligned>:: + const_iterator mit = mappings.begin(); + mit != mappings.end(); mit++) + { + rez.serialize(mit->first); + rez.serialize(mit->second.size()); + for (LegionMap::aligned::const_iterator it = + mit->second.begin(); it != mit->second.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + rez.serialize(global_views.size()); + for (LegionMap::aligned::const_iterator it = + global_views.begin(); it != global_views.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + void ShardedMappingExchange::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + size_t num_mappings; + derez.deserialize(num_mappings); + for (unsigned idx1 = 0; idx1 < num_mappings; idx1++) + { + DistributedID did; + derez.deserialize(did); + size_t num_shards; + derez.deserialize(num_shards); + LegionMap::aligned &inst_map = mappings[did]; + for (unsigned idx2 = 0; idx2 < num_shards; idx2++) + { + ShardID sid; + derez.deserialize(sid); + LegionMap::aligned::iterator finder = + inst_map.find(sid); + if (finder != inst_map.end()) + { + FieldMask mask; + derez.deserialize(mask); + finder->second |= mask; + } + else + derez.deserialize(inst_map[sid]); + } + } + size_t num_views; + derez.deserialize(num_views); + for (unsigned idx = 0; idx < num_views; idx++) + { + DistributedID did; + derez.deserialize(did); + LegionMap::aligned::iterator finder = + global_views.find(did); + if (finder != global_views.end()) + { + FieldMask mask; + derez.deserialize(mask); + finder->second |= mask; + } + else + derez.deserialize(global_views[did]); + } + } + + //-------------------------------------------------------------------------- + void ShardedMappingExchange::initiate_exchange( + const InstanceSet &local_mappings, + const std::vector &local_views) + //-------------------------------------------------------------------------- + { + { + AutoLock c_lock(collective_lock); + // Populate the data structure with instance names + for (unsigned idx = 0; idx < local_mappings.size(); idx++) + { + const InstanceRef &mapping = local_mappings[idx]; + const FieldMask &mask = mapping.get_valid_fields(); + if (check_mappings) + { + const DistributedID did = mapping.get_manager()->did; + LegionMap::aligned &inst_map = mappings[did]; + LegionMap::aligned::iterator finder = + inst_map.find(shard_id); + if (finder == inst_map.end()) + inst_map[shard_id] = mask; + else + finder->second |= mask; + } + const DistributedID view_did = local_views[idx]->did; + LegionMap::aligned::iterator finder = + global_views.find(view_did); + if (finder == global_views.end()) + global_views[view_did] = mask; + else + finder->second |= mask; + } + } + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + void ShardedMappingExchange::complete_exchange(Operation *op, + ShardedView *sharded_view, + const InstanceSet &local_mappings, + std::set &applied_events) + //-------------------------------------------------------------------------- + { + perform_collective_wait(); + if (sharded_view != NULL) + sharded_view->initialize(global_views, local_mappings, applied_events); + if (check_mappings) + { +#ifdef DEBUG_LEGION + assert(op != NULL); +#endif + // Check to see if our mappings interfere with any others + for (unsigned idx = 0; idx < local_mappings.size(); idx++) + { + const InstanceRef &mapping = local_mappings[idx]; + const DistributedID did = mapping.get_manager()->did; + const FieldMask &mask = mapping.get_valid_fields(); + const std::map::aligned>::const_iterator + finder = mappings.find(did); +#ifdef DEBUG_LEGION + // We should have at least our own + assert(finder != mappings.end()); +#endif + for (LegionMap::aligned::const_iterator it = + finder->second.begin(); it != finder->second.end(); it++) + { + // We can skip ourself + if (it->first == shard_id) + continue; + const FieldMask overlap = mask & it->second; + if (!overlap) + continue; + // This is the error condition + TaskContext *ctx = op->get_context(); + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "%s in control replicated contexts must " + "map to different instances for the same field. Inline " + "mapping in shard %d conflicts with mapping in shard %d " + "of control replciated task %s (UID %lld)", + op->get_logging_name(), shard_id, it->first, + ctx->get_task_name(), ctx->get_unique_id()) + } + } + } + } + + ///////////////////////////////////////////////////////////// + // Template Index Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + TemplateIndexExchange::TemplateIndexExchange(ReplicateContext *ctx, + CollectiveID id) + : AllGatherCollective(ctx, id), current_stage(-1) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + TemplateIndexExchange::TemplateIndexExchange( + const TemplateIndexExchange &rhs) + : AllGatherCollective(rhs), current_stage(-1) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + TemplateIndexExchange::~TemplateIndexExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + TemplateIndexExchange& TemplateIndexExchange::operator=( + const TemplateIndexExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void TemplateIndexExchange::pack_collective_stage(Serializer &rez,int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(index_counts.size()); + for (std::map::const_iterator it = + index_counts.begin(); it != index_counts.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + void TemplateIndexExchange::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + // If we are not a participating stage then we already contributed our + // data into the output so we clear ourself to avoid double counting + if ((stage == -1) && !participating) + index_counts.clear(); + size_t num_counts; + derez.deserialize(num_counts); + for (unsigned idx = 0; idx < num_counts; idx++) + { + int index; + derez.deserialize(index); + unsigned count; + derez.deserialize(count); + std::map::iterator finder = index_counts.find(index); + if (finder == index_counts.end()) + index_counts[index] = count; + else + finder->second += count; + } + } + + //-------------------------------------------------------------------------- + void TemplateIndexExchange::initiate_exchange( + const std::vector &indexes) + //-------------------------------------------------------------------------- + { + for (std::vector::const_iterator it = indexes.begin(); + it != indexes.end(); it++) + index_counts[*it] = 1; + perform_collective_async(); + } + + //-------------------------------------------------------------------------- + void TemplateIndexExchange::complete_exchange( + std::map &result_counts) + //-------------------------------------------------------------------------- + { + perform_collective_wait(true/*block*/); + result_counts.swap(index_counts); + } + + ///////////////////////////////////////////////////////////// + // Unordered Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + UnorderedExchange::UnorderedExchange(ReplicateContext *ctx, + CollectiveIndexLocation loc) + : AllGatherCollective(loc, ctx), current_stage(-1) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + UnorderedExchange::UnorderedExchange(const UnorderedExchange &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + UnorderedExchange::~UnorderedExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + UnorderedExchange& UnorderedExchange::operator=( + const UnorderedExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + template + void UnorderedExchange::update_future_counts(const int stage, + std::map > &future_counts, + std::map &counts) + //-------------------------------------------------------------------------- + { + typename std::map >::iterator next = + future_counts.find(stage-1); + if (next != future_counts.end()) + { + for (typename std::map::const_iterator it = + next->second.begin(); it != next->second.end(); it++) + { + typename std::map::iterator finder = + counts.find(it->first); + if (finder == counts.end()) + counts.insert(*it); + else + finder->second += it->second; + } + future_counts.erase(next); + } + } + + //-------------------------------------------------------------------------- + template + void UnorderedExchange::pack_counts(Serializer &rez, + const std::map &counts) + //-------------------------------------------------------------------------- + { + rez.serialize(counts.size()); + for (typename std::map::const_iterator it = + counts.begin(); it != counts.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + template + void UnorderedExchange::unpack_counts(const int stage, Deserializer &derez, + std::map &counts) + //-------------------------------------------------------------------------- + { + size_t num_counts; + derez.deserialize(num_counts); + if (num_counts == 0) + return; + for (unsigned idx = 0; idx < num_counts; idx++) + { + T key; + derez.deserialize(key); + unsigned count; + derez.deserialize(count); + typename std::map::iterator finder = counts.find(key); + if (finder == counts.end()) + counts[key] = count; + else + finder->second += count; + } + } + + //-------------------------------------------------------------------------- + template + void UnorderedExchange::initialize_counts(const std::map &ops, + std::map &counts) + //-------------------------------------------------------------------------- + { + for (typename std::map::const_iterator it = + ops.begin(); it != ops.end(); it++) + counts[it->first] = 1; + } + + //-------------------------------------------------------------------------- + template + void UnorderedExchange::find_ready_ops(const size_t total_shards, + const std::map &final_counts, + const std::map &ops, std::vector &ready_ops) + //-------------------------------------------------------------------------- + { + for (typename std::map::const_iterator it = + final_counts.begin(); it != final_counts.end(); it++) + { +#ifdef DEBUG_LEGION + assert(it->second <= total_shards); +#endif + if (it->second == total_shards) + { + typename std::map::const_iterator finder = ops.find(it->first); +#ifdef DEBUG_LEGION + assert(finder != ops.end()); +#endif + ready_ops.push_back(finder->second); + } + } + } + + //-------------------------------------------------------------------------- + void UnorderedExchange::pack_collective_stage(Serializer &rez, int stage) + //-------------------------------------------------------------------------- + { + pack_counts(rez, index_space_counts); + pack_counts(rez, index_partition_counts); + pack_counts(rez, field_space_counts); + pack_counts(rez, field_counts); + pack_counts(rez, logical_region_counts); + pack_counts(rez, detach_counts); + } + + //-------------------------------------------------------------------------- + void UnorderedExchange::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + // If we are not a participating stage then we already contributed our + // data into the output so we clear ourself to avoid double counting + if ((stage == -1) && !participating) + { + index_space_counts.clear(); + index_partition_counts.clear(); + field_space_counts.clear(); + field_counts.clear(); + logical_region_counts.clear(); + detach_counts.clear(); + } + unpack_counts(stage, derez, index_space_counts); + unpack_counts(stage, derez, index_partition_counts); + unpack_counts(stage, derez, field_space_counts); + unpack_counts(stage, derez, field_counts); + unpack_counts(stage, derez, logical_region_counts); + unpack_counts(stage, derez, detach_counts); + } + + //-------------------------------------------------------------------------- + bool UnorderedExchange::exchange_unordered_ops( + const std::list &unordered_ops, + std::vector &ready_ops) + //-------------------------------------------------------------------------- + { + // Sort our operations + if (!unordered_ops.empty()) + { + for (std::list::const_iterator it = + unordered_ops.begin(); it != unordered_ops.end(); it++) + { + switch ((*it)->get_operation_kind()) + { + case Operation::DELETION_OP_KIND: + { +#ifdef DEBUG_LEGION + ReplDeletionOp *op = dynamic_cast(*it); + assert(op != NULL); +#else + ReplDeletionOp *op = static_cast(*it); +#endif + op->record_unordered_kind(index_space_deletions, + index_partition_deletions, field_space_deletions, + field_deletions, logical_region_deletions); + break; + } + case Operation::DETACH_OP_KIND: + { +#ifdef DEBUG_LEGION + ReplDetachOp *op = dynamic_cast(*it); + assert(op != NULL); +#else + ReplDetachOp *op = static_cast(*it); +#endif + op->record_unordered_kind(detachments); + break; + } + default: // Unimplemented operation kind + assert(false); + } + } + // Set the initial counts to one for all our unordered ops + initialize_counts(index_space_deletions, index_space_counts); + initialize_counts(index_partition_deletions, index_partition_counts); + initialize_counts(field_space_deletions, field_space_counts); + initialize_counts(field_deletions, field_counts); + initialize_counts(logical_region_deletions, logical_region_counts); + initialize_counts(detachments, detach_counts); + } + // Perform the exchange + perform_collective_sync(); + // Now look and see which operations have keys for all shards + // Only need to do this if we have ops, if we didn't have ops then + // it's impossible for anyone else to have them all too + if (!unordered_ops.empty()) + { + const size_t total_shards = manager->total_shards; + find_ready_ops(total_shards, index_space_counts, + index_space_deletions, ready_ops); + find_ready_ops(total_shards, index_partition_counts, + index_partition_deletions, ready_ops); + find_ready_ops(total_shards, field_space_counts, + field_space_deletions, ready_ops); + find_ready_ops(total_shards, field_counts, + field_deletions, ready_ops); + find_ready_ops(total_shards, logical_region_counts, + logical_region_deletions, ready_ops); + find_ready_ops(total_shards, detach_counts, + detachments, ready_ops); + } + // Return true if anybody anywhere had a non-zero count + return (!index_space_counts.empty() || !index_partition_counts.empty() || + !field_space_counts.empty() || !field_counts.empty() || + !logical_region_counts.empty() || !detach_counts.empty()); + } + + ///////////////////////////////////////////////////////////// + // Consensus Match Base + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ConsensusMatchBase::ConsensusMatchBase(ReplicateContext *ctx, + CollectiveIndexLocation loc) + : AllGatherCollective(ctx, loc) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ConsensusMatchBase::ConsensusMatchBase(const ConsensusMatchBase &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ConsensusMatchBase::~ConsensusMatchBase(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + /*static*/ void ConsensusMatchBase::handle_consensus_match(const void *args) + //-------------------------------------------------------------------------- + { + const ConsensusMatchArgs *margs = (const ConsensusMatchArgs*)args; + margs->base->complete_exchange(); + delete margs->base; + } + + ///////////////////////////////////////////////////////////// + // Consensus Match Exchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + template + ConsensusMatchExchange::ConsensusMatchExchange(ReplicateContext *ctx, + CollectiveIndexLocation loc, Future f, void *out, ApUserEvent trig) + : ConsensusMatchBase(ctx, loc), to_complete(f), + output(static_cast(out)), to_trigger(trig) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + template + ConsensusMatchExchange::ConsensusMatchExchange( + const ConsensusMatchExchange &rhs) + : ConsensusMatchBase(rhs), to_complete(rhs.to_complete), + output(rhs.output), to_trigger(rhs.to_trigger) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + template + ConsensusMatchExchange::~ConsensusMatchExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + template + ConsensusMatchExchange& ConsensusMatchExchange::operator=( + const ConsensusMatchExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + template + void ConsensusMatchExchange::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(element_counts.size()); + for (typename std::map::const_iterator it = + element_counts.begin(); it != element_counts.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + template + void ConsensusMatchExchange::unpack_collective_stage( + Deserializer &derez, int stage) + //-------------------------------------------------------------------------- + { + size_t num_elements; + derez.deserialize(num_elements); + for (unsigned idx = 0; idx < num_elements; idx++) + { + T element; + derez.deserialize(element); + typename std::map::iterator finder = + element_counts.find(element); + if (finder != element_counts.end()) + { + size_t count; + derez.deserialize(count); + finder->second += count; + } + else + derez.deserialize(element_counts[element]); + } + } + + //-------------------------------------------------------------------------- + template + bool ConsensusMatchExchange::match_elements_async(const void *input, + size_t num_elements) + //-------------------------------------------------------------------------- + { + const T *inputs = static_cast(input); + for (unsigned idx = 0; idx < num_elements; idx++) + element_counts[inputs[idx]] = 1; +#ifdef DEBUG_LEGION + max_elements = num_elements; +#endif + perform_collective_async(); + const RtEvent precondition = perform_collective_wait(false/*block*/); + if (precondition.exists() && !precondition.has_triggered()) + { + ConsensusMatchArgs args(this, context->get_unique_id()); + context->runtime->issue_runtime_meta_task(args, + LG_LATENCY_DEFERRED_PRIORITY, precondition); + return false; + } + else + { + complete_exchange(); + return true; + } + } + + //-------------------------------------------------------------------------- + template + void ConsensusMatchExchange::complete_exchange(void) + //-------------------------------------------------------------------------- + { + const size_t total_shards = manager->total_shards; + size_t next_index = 0; + for (typename std::map::const_iterator it = + element_counts.begin(); it != element_counts.end(); it++) + { +#ifdef DEBUG_LEGION + assert(it->second <= total_shards); +#endif + if (it->second < total_shards) + continue; +#ifdef DEBUG_LEGION + assert(next_index < max_elements); +#endif + output[next_index++] = it->first; + } + // A little bit of help from the replicate context to complete the future + context->help_complete_future(to_complete, &next_index, + sizeof(next_index), false/*own*/); + Runtime::trigger_event(NULL, to_trigger); + } + + template class ConsensusMatchExchange; + template class ConsensusMatchExchange; + template class ConsensusMatchExchange; + template class ConsensusMatchExchange; + + ///////////////////////////////////////////////////////////// + // VerifyReplicableExchange + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + VerifyReplicableExchange::VerifyReplicableExchange( + CollectiveIndexLocation loc, ReplicateContext *ctx) + : AllGatherCollective(loc, ctx) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + VerifyReplicableExchange::VerifyReplicableExchange( + const VerifyReplicableExchange &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + VerifyReplicableExchange::~VerifyReplicableExchange(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + VerifyReplicableExchange& VerifyReplicableExchange::operator=( + const VerifyReplicableExchange &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void VerifyReplicableExchange::pack_collective_stage(Serializer &rez, + int stage) + //-------------------------------------------------------------------------- + { + rez.serialize(unique_hashes.size()); + for (ShardHashes::const_iterator it = unique_hashes.begin(); + it != unique_hashes.end(); it++) + { + rez.serialize(it->first.first); + rez.serialize(it->first.second); + rez.serialize(it->second); + } + } + + //-------------------------------------------------------------------------- + void VerifyReplicableExchange::unpack_collective_stage(Deserializer &derez, + int stage) + //-------------------------------------------------------------------------- + { + size_t num_hashes; + derez.deserialize(num_hashes); + for (unsigned idx = 0; idx < num_hashes; idx++) + { + std::pair key; + derez.deserialize(key.first); + derez.deserialize(key.second); + ShardHashes::iterator finder = unique_hashes.find(key); + if (finder != unique_hashes.end()) + { + ShardID sid; + derez.deserialize(sid); + if (sid < finder->second) + finder->second = sid; + } + else + derez.deserialize(unique_hashes[key]); + } + } + + //-------------------------------------------------------------------------- + const VerifyReplicableExchange::ShardHashes& + VerifyReplicableExchange::exchange(uint64_t hash[2]) + //-------------------------------------------------------------------------- + { + const std::pair key(hash[0],hash[1]); + unique_hashes[key] = local_shard; + perform_collective_sync(); + return unique_hashes; + } + + ///////////////////////////////////////////////////////////// + // Slow Barrier + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + SlowBarrier::SlowBarrier(ReplicateContext *ctx, CollectiveID id) + : AllGatherCollective(ctx, id) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + SlowBarrier::SlowBarrier(const SlowBarrier &rhs) + : AllGatherCollective(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + SlowBarrier::~SlowBarrier(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + SlowBarrier& SlowBarrier::operator=(const SlowBarrier &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + }; // namespace Internal +}; // namespace Legion + diff --git a/runtime/legion/legion_replication.h b/runtime/legion/legion_replication.h new file mode 100644 index 0000000000..279f0b27ec --- /dev/null +++ b/runtime/legion/legion_replication.h @@ -0,0 +1,2225 @@ +/* Copyright 2020 Stanford University, NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __LEGION_REPLICATION_H__ +#define __LEGION_REPLICATION_H__ + +#include "legion/legion_ops.h" +#include "legion/legion_tasks.h" +#include "legion/legion_trace.h" + +namespace Legion { + namespace Internal { + +#ifdef DEBUG_LEGION_COLLECTIVES + /** + * \class CollectiveCheckReduction + * A small helper reduction for use with checking that + * Legion collectives are properly aligned across all shards + */ + class CollectiveCheckReduction { + public: + typedef long RHS; + typedef long LHS; + static const long IDENTITY; + static const long identity; + static const long BAD; + static const ReductionOpID REDOP; + + template static void apply(LHS &lhs, RHS rhs); + template static void fold(RHS &rhs1, RHS rhs2); + }; + + /** + * \class CloseCheckReduction + * Another helper reduction for comparing the phase barriers + * used by close operations which should be ordered + */ + class CloseCheckReduction { + public: + struct CloseCheckValue { + public: + CloseCheckValue(void); + CloseCheckValue(const LogicalUser &user, RtBarrier barrier, + RegionTreeNode *node, bool read_only); + public: + bool operator==(const CloseCheckValue &rhs) const; + public: + unsigned operation_index; + unsigned region_requirement_index; + RtBarrier barrier; + LogicalRegion region; + LogicalPartition partition; + bool is_region; + bool read_only; + }; + public: + typedef CloseCheckValue RHS; + typedef CloseCheckValue LHS; + static const CloseCheckValue IDENTITY; + static const CloseCheckValue identity; + static const ReductionOpID REDOP; + + template static void apply(LHS &lhs, RHS rhs); + template static void fold(RHS &rhs1, RHS rhs2); + }; +#endif + + /** + * \class ShardCollective + * The shard collective is the base class for performing + * collective operations between shards + */ + class ShardCollective { + public: + ShardCollective(CollectiveIndexLocation loc, ReplicateContext *ctx); + ShardCollective(ReplicateContext *ctx, CollectiveID id); + virtual ~ShardCollective(void); + public: + virtual void handle_collective_message(Deserializer &derez) = 0; + protected: + int convert_to_index(ShardID id, ShardID origin) const; + ShardID convert_to_shard(int index, ShardID origin) const; + public: + ShardManager *const manager; + ReplicateContext *const context; + const ShardID local_shard; + const CollectiveID collective_index; + protected: + mutable LocalLock collective_lock; + }; + + /** + * \class BroadcastCollective + * This shard collective has equivalent functionality to + * MPI Broadcast in that it will transmit some data on one + * shard to all the other shards. + */ + class BroadcastCollective : public ShardCollective { + public: + BroadcastCollective(CollectiveIndexLocation loc, + ReplicateContext *ctx, ShardID origin); + BroadcastCollective(ReplicateContext *ctx, + CollectiveID id, ShardID origin); + virtual ~BroadcastCollective(void); + public: + // We guarantee that these methods will be called atomically + virtual void pack_collective(Serializer &rez) const = 0; + virtual void unpack_collective(Deserializer &derez) = 0; + public: + void perform_collective_async(void); + RtEvent perform_collective_wait(bool block = true); + virtual void handle_collective_message(Deserializer &derez); + public: + RtEvent get_done_event(void) const; + protected: + void send_messages(void) const; + public: + const ShardID origin; + const int shard_collective_radix; + private: + RtUserEvent done_event; // valid on all shards except origin + }; + + /** + * \class GatherCollective + * This shard collective has equivalent functionality to + * MPI Gather in that it will ensure that data from all + * the shards are reduced down to a single shard. + */ + class GatherCollective : public ShardCollective { + public: + GatherCollective(CollectiveIndexLocation loc, + ReplicateContext *ctx, ShardID target); + virtual ~GatherCollective(void); + public: + // We guarantee that these methods will be called atomically + virtual void pack_collective(Serializer &rez) const = 0; + virtual void unpack_collective(Deserializer &derez) = 0; + public: + void perform_collective_async(void); + // Make sure to call this in the destructor of anything not the target + RtEvent perform_collective_wait(bool block = true); + virtual void handle_collective_message(Deserializer &derez); + inline bool is_target(void) const { return (target == local_shard); } + // Use this method in case we don't actually end up using the collective + void elide_collective(void); + protected: + void send_message(void); + int compute_expected_notifications(void) const; + public: + const ShardID target; + const int shard_collective_radix; + const int expected_notifications; + private: + RtUserEvent done_event; // only valid on owner shard + int received_notifications; + }; + + /** + * \class AllGatherCollective + * This shard collective has equivalent functionality to + * MPI All Gather in that it will ensure that all shards + * see the value data from all other shards. + */ + template + class AllGatherCollective : public ShardCollective { + public: + // Inorder says whether we need to see messages for stages inorder, + // e.g. do we need to see all stage 0 messages before stage 1 + AllGatherCollective(CollectiveIndexLocation loc, ReplicateContext *ctx); + AllGatherCollective(ReplicateContext *ctx, CollectiveID id); + virtual ~AllGatherCollective(void); + public: + // We guarantee that these methods will be called atomically + virtual void pack_collective_stage(Serializer &rez, int stage) = 0; + virtual void unpack_collective_stage(Deserializer &derez, int stage) = 0; + public: + void perform_collective_sync(void); + void perform_collective_async(void); + RtEvent perform_collective_wait(bool block = true); + virtual void handle_collective_message(Deserializer &derez); + // Use this method in case we don't actually end up using the collective + void elide_collective(void); + protected: + void initialize_collective(void); + void construct_message(ShardID target, int stage, Serializer &rez); + bool initiate_collective(void); + void send_remainder_stage(void); + bool send_ready_stages(const int start_stage=1); + void unpack_stage(int stage, Deserializer &derez); + void complete_exchange(void); + public: + const int shard_collective_radix; + const int shard_collective_log_radix; + const int shard_collective_stages; + const int shard_collective_participating_shards; + const int shard_collective_last_radix; + const bool participating; + private: + RtUserEvent done_event; + std::vector stage_notifications; + std::vector sent_stages; + std::map > > *reorder_stages; + // Handle a small race on deciding who gets to + // trigger the done event, only the last one of these + // will get to do the trigger to avoid any races + unsigned pending_send_ready_stages; +#ifdef DEBUG_LEGION + bool done_triggered; +#endif + }; + + /** + * \class AllReduceOpCollective + * This collective has equivalent functonality to + * MPI All Reduce in that it will take a value from each + * shard and reduce it down to a final value using a + * Realm reduction operator. + */ + class AllReduceOpCollective : public AllGatherCollective { + public: + AllReduceOpCollective(CollectiveIndexLocation loc, ReplicateContext *ctx, + const ReductionOp *redop); + AllReduceOpCollective(ReplicateContext *ctx, CollectiveID id, + const ReductionOp *redop); + virtual ~AllReduceOpCollective(void); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + RtEvent async_reduce(const void *value); + void sync_result(void *result); + public: + const ReductionOp *const redop; + protected: + int current_stage; + void *const value; + std::map > future_values; + }; + + /** + * \class AllReduceCollective + * This shard collective has equivalent functionality to + * MPI All Reduce in that it will take a value from each + * shard and reduce it down to a final value using a + * Legion reduction operator. We'll build this on top + * of the AllGatherCollective + */ + template + class AllReduceCollective : public AllGatherCollective { + public: + AllReduceCollective(CollectiveIndexLocation loc, ReplicateContext *ctx); + AllReduceCollective(ReplicateContext *ctx, CollectiveID id); + virtual ~AllReduceCollective(void); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + void async_all_reduce(typename REDOP::RHS value); + RtEvent wait_all_reduce(bool block = true); + typename REDOP::RHS sync_all_reduce(typename REDOP::RHS value); + typename REDOP::RHS get_result(void); + protected: + typename REDOP::RHS value; + int current_stage; + std::map > future_values; + }; + + /** + * \class BarrierExchangeCollective + * A class for exchanging sets of barriers between shards + */ + template + class BarrierExchangeCollective : public AllGatherCollective { + public: + BarrierExchangeCollective(ReplicateContext *ctx, size_t window_size, + typename std::vector &barriers, + CollectiveIndexLocation loc); + BarrierExchangeCollective(const BarrierExchangeCollective &rhs); + virtual ~BarrierExchangeCollective(void); + public: + BarrierExchangeCollective& operator=(const BarrierExchangeCollective &rs); + public: + void exchange_barriers_async(void); + void wait_for_barrier_exchange(void); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + protected: + const size_t window_size; + std::vector &barriers; + std::map local_barriers; + }; + + /** + * \class ValueBroadcast + * This will broadcast a value of any type that can be + * trivially serialized to all the shards. + */ + template + class ValueBroadcast : public BroadcastCollective { + public: + ValueBroadcast(ReplicateContext *ctx, CollectiveIndexLocation loc) + : BroadcastCollective(loc, ctx, ctx->owner_shard->shard_id) { } + ValueBroadcast(ReplicateContext *ctx, ShardID origin, + CollectiveIndexLocation loc) + : BroadcastCollective(loc, ctx, origin) { } + ValueBroadcast(CollectiveID id, ReplicateContext *ctx, ShardID origin) + : BroadcastCollective(ctx, id, origin) { } + ValueBroadcast(const ValueBroadcast &rhs) + : BroadcastCollective(rhs) { assert(false); } + virtual ~ValueBroadcast(void) { } + public: + ValueBroadcast& operator=(const ValueBroadcast &rhs) + { assert(false); return *this; } + inline void broadcast(const T &v) + { value = v; perform_collective_async(); } + inline T get_value(bool wait = true) + { if (wait) perform_collective_wait(); return value; } + public: + virtual void pack_collective(Serializer &rez) const + { rez.serialize(value); } + virtual void unpack_collective(Deserializer &derez) + { derez.deserialize(value); } + protected: + T value; + }; + + /** + * \class ValueExchange + * This class will exchange a value of any type that can be + * trivially serialized to all the shards + */ + template + class ValueExchange : public AllGatherCollective { + public: + ValueExchange(CollectiveIndexLocation loc, ReplicateContext *ctx) + : AllGatherCollective(loc, ctx) { } + ValueExchange(ReplicateContext *ctx, CollectiveID id) + : AllGatherCollective(ctx, id) { } + virtual ~ValueExchange(void) { } + public: + virtual void pack_collective_stage(Serializer &rez, int stage) + { + rez.serialize(values.size()); + for (typename std::set::const_iterator it = values.begin(); + it != values.end(); it++) + rez.serialize(*it); + } + virtual void unpack_collective_stage(Deserializer &derez, int stage) + { + size_t num_values; + derez.deserialize(num_values); + for (unsigned idx = 0; idx < num_values; idx++) + { + T value; + derez.deserialize(value); + values.insert(value); + } + } + public: + const std::set& exchange_values(T value) + { + values.insert(value); + perform_collective_sync(); + return values; + } + protected: + std::set values; + }; + + /** + * \class BufferBroadcast + * Broadcast out a binary buffer out to all the shards + */ + class BufferBroadcast : public BroadcastCollective { + public: + BufferBroadcast(ReplicateContext *ctx, CollectiveIndexLocation loc) + : BroadcastCollective(loc, ctx, ctx->owner_shard->shard_id), + buffer(NULL), size(0), own(false) { } + BufferBroadcast(ReplicateContext *ctx, ShardID origin, + CollectiveIndexLocation loc) + : BroadcastCollective(loc, ctx, origin), + buffer(NULL), size(0), own(false) { } + BufferBroadcast(const BufferBroadcast &rhs) + : BroadcastCollective(rhs) { assert(false); } + virtual ~BufferBroadcast(void) { if (own) free(buffer); } + public: + BufferBroadcast& operator=(const BufferBroadcast &rhs) + { assert(false); return *this; } + void broadcast(void *buffer, size_t size, bool copy = true); + const void* get_buffer(size_t &size, bool wait = true); + public: + virtual void pack_collective(Serializer &rez) const; + virtual void unpack_collective(Deserializer &derez); + protected: + void *buffer; + size_t size; + bool own; + }; + + /** + * \class ShardSyncTree + * A synchronization tree allows one shard to be notified when + * all the other shards have reached a certain point in the + * execution of the program. + */ + class ShardSyncTree : public BroadcastCollective { + public: + ShardSyncTree(ReplicateContext *ctx, ShardID origin, + CollectiveIndexLocation loc); + ShardSyncTree(const ShardSyncTree &rhs) + : BroadcastCollective(rhs), is_origin(false) + { assert(false); } + virtual ~ShardSyncTree(void); + public: + ShardSyncTree& operator=(const ShardSyncTree &rhs) + { assert(false); return *this; } + public: + virtual void pack_collective(Serializer &rez) const; + virtual void unpack_collective(Deserializer &derez); + protected: + RtUserEvent done_event; + mutable std::set done_preconditions; + const bool is_origin; + }; + + /** + * \class ShardEventTree + * This collective will construct an event broadcast tree + * so that one shard can notify all the other shards once + * an event has triggered + */ + class ShardEventTree : public BroadcastCollective { + public: + ShardEventTree(ReplicateContext *ctx, ShardID origin, + CollectiveID id); + ShardEventTree(const ShardEventTree &rhs) + : BroadcastCollective(rhs), is_origin(false) { assert(false); } + virtual ~ShardEventTree(void); + public: + ShardEventTree& operator=(const ShardEventTree &rhs) + { assert(false); return *this; } + public: + void signal_tree(RtEvent precondition); // origin + RtEvent get_local_event(void); + public: + virtual void pack_collective(Serializer &rez) const; + virtual void unpack_collective(Deserializer &derez); + protected: + RtUserEvent local_event; + RtEvent trigger_event; + RtEvent finished_event; + const bool is_origin; + }; + + /** + * \class CrossProductExchange + * A class for exchanging the names of partitions created by + * a call for making cross-product partitions + */ + class CrossProductCollective : public AllGatherCollective { + public: + CrossProductCollective(ReplicateContext *ctx, + CollectiveIndexLocation loc); + CrossProductCollective(const CrossProductCollective &rhs); + virtual ~CrossProductCollective(void); + public: + CrossProductCollective& operator=(const CrossProductCollective &rhs); + public: + void exchange_partitions(std::map &handles); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + protected: + std::map non_empty_handles; + }; + + /** + * \class ShardingGatherCollective + * A class for gathering all the names of the ShardingIDs chosen + * by different mappers to confirm that they are all the same. + * This is primarily only used in debug mode. + */ + class ShardingGatherCollective : public GatherCollective { + public: + ShardingGatherCollective(ReplicateContext *ctx, ShardID target, + CollectiveIndexLocation loc); + ShardingGatherCollective(const ShardingGatherCollective &rhs); + virtual ~ShardingGatherCollective(void); + public: + ShardingGatherCollective& operator=(const ShardingGatherCollective &rhs); + public: + virtual void pack_collective(Serializer &rez) const; + virtual void unpack_collective(Deserializer &derez); + public: + void contribute(ShardingID value); + bool validate(ShardingID value); + protected: + std::map results; + }; + + /** + * \class IndirectRecordExchange + * A class for doing an all-gather of indirect records for + * doing gather/scatter/full-indirect copy operations. + */ + class IndirectRecordExchange : public AllGatherCollective { + public: + struct IndirectKey { + public: + IndirectKey(void) { } + IndirectKey(PhysicalInstance i, ApEvent e, const Domain &d) + : inst(i), ready_event(e), domain(d) { } + public: + inline bool operator<(const IndirectKey &rhs) const + { + if (inst.id < rhs.inst.id) + return true; + if (inst.id > rhs.inst.id) + return false; + if (ready_event.id < rhs.ready_event.id) + return true; + if (ready_event.id > rhs.ready_event.id) + return false; + return (domain < rhs.domain); + } + inline bool operator==(const IndirectKey &rhs) const + { + if (inst.id != rhs.inst.id) + return false; + if (ready_event.id != rhs.ready_event.id) + return false; + return (domain == rhs.domain); + } + public: + PhysicalInstance inst; + ApEvent ready_event; + Domain domain; + }; + public: + IndirectRecordExchange(ReplicateContext *ctx, + CollectiveIndexLocation loc); + IndirectRecordExchange(const IndirectRecordExchange &rhs); + virtual ~IndirectRecordExchange(void); + public: + IndirectRecordExchange& operator=(const IndirectRecordExchange &rhs); + public: + void exchange_records(LegionVector::aligned &records); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + protected: + LegionMap::aligned records; + }; + + /** + * \class FieldDescriptorExchange + * A class for doing an all-gather of field descriptors for + * doing dependent partitioning operations. This will also build + * a butterfly tree of user events that will be used to know when + * all of the constituent shards are done with the operation they + * are collectively performing together. + */ + class FieldDescriptorExchange : public AllGatherCollective { + public: + FieldDescriptorExchange(ReplicateContext *ctx, + CollectiveIndexLocation loc); + FieldDescriptorExchange(const FieldDescriptorExchange &rhs); + virtual ~FieldDescriptorExchange(void); + public: + FieldDescriptorExchange& operator=(const FieldDescriptorExchange &rhs); + public: + ApEvent exchange_descriptors(ApEvent ready_event, + const std::vector &desc); + // Have to call this with the completion event + ApEvent exchange_completion(ApEvent complete_event); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + std::set ready_events; + std::vector descriptors; + public: + // Use these for building the butterfly network of user events for + // knowing when everything is done on all the nodes. + // This vector is of the number of stages and tracks the incoming + // set of remote complete events for a stage, in the case of a + // remainder stage it is of size 1 + std::vector > remote_to_trigger; // stages + // This vector is the number of stages+1 to capture the ready + // event for each of the different stages as well as the event + // for when the entire collective is done + mutable std::vector > local_preconditions; + }; + + /** + * \class FieldDescriptorGather + * A class for doing a gather of field descriptors to a specific + * node for doing dependent partitioning operations. This collective + * also will construct an event broadcast tree to inform all the + * constituent shards about when the operation is done with the + * instances which are being gathered. + */ + class FieldDescriptorGather : public GatherCollective { + public: + FieldDescriptorGather(ReplicateContext *ctx, ShardID target, + CollectiveIndexLocation loc); + FieldDescriptorGather(const FieldDescriptorGather &rhs); + virtual ~FieldDescriptorGather(void); + public: + FieldDescriptorGather& operator=(const FieldDescriptorGather &rhs); + public: + virtual void pack_collective(Serializer &rez) const; + virtual void unpack_collective(Deserializer &derez); + public: + void contribute(ApEvent ready_event, + const std::vector &descriptors); + const std::vector& + get_full_descriptors(ApEvent &ready); + // Owner shard only + void notify_remote_complete(ApEvent precondition); + // Non-owner shard only + ApEvent get_complete_event(void) const; + protected: + std::set ready_events; + std::vector descriptors; + std::set remote_complete_events; + ApUserEvent complete_event; + bool used; + }; + + /** + * \class FutureBroadcast + * A class for broadcasting a future result to all the shards + */ + class FutureBroadcast : public BroadcastCollective { + public: + FutureBroadcast(ReplicateContext *ctx, CollectiveID id, + ShardID source, FutureImpl *impl); + FutureBroadcast(const FutureBroadcast &rhs); + virtual ~FutureBroadcast(void); + public: + FutureBroadcast& operator=(const FutureBroadcast &rhs); + public: + virtual void pack_collective(Serializer &rez) const; + virtual void unpack_collective(Deserializer &derez); + public: + void broadcast_future(void); + protected: + FutureImpl *const impl; + RtEvent ready; + }; + + /** + * \class FutureExchange + * A class for doing an all-to-all exchange of future values + */ + class FutureExchange : public AllGatherCollective { + public: + FutureExchange(ReplicateContext *ctx, size_t future_size, + CollectiveIndexLocation loc); + FutureExchange(const FutureExchange &rhs); + virtual ~FutureExchange(void); + public: + FutureExchange& operator=(const FutureExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + // This takes ownership of the buffer + RtEvent exchange_futures(void *value); + void reduce_futures(ReplIndexTask *target); + void reduce_futures(const ReductionOp *redop, void *result_buffer); + public: + const size_t future_size; + protected: + std::map results; + }; + + /** + * \class FutureNameExchange + * A class for doing an all-to-all exchange of future names + */ + class FutureNameExchange : public AllGatherCollective { + public: + FutureNameExchange(ReplicateContext *ctx, CollectiveID id, + ReplFutureMapImpl *future_map, + ReferenceMutator *mutator); + FutureNameExchange(const FutureNameExchange &rhs); + virtual ~FutureNameExchange(void); + public: + FutureNameExchange& operator=(const FutureNameExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + void exchange_future_names(std::map &futures); + public: + ReplFutureMapImpl *const future_map; + ReferenceMutator *const mutator; + protected: + std::map results; + }; + + /** + * \class MustEpochMappingBroadcast + * A class for broadcasting the results of the mapping decisions + * for a map must epoch call on a single node + */ + class MustEpochMappingBroadcast : public BroadcastCollective { + public: + MustEpochMappingBroadcast(ReplicateContext *ctx, ShardID origin, + CollectiveID collective_id); + MustEpochMappingBroadcast(const MustEpochMappingBroadcast &rhs); + virtual ~MustEpochMappingBroadcast(void); + public: + MustEpochMappingBroadcast& operator=( + const MustEpochMappingBroadcast &rhs); + public: + virtual void pack_collective(Serializer &rez) const; + virtual void unpack_collective(Deserializer &derez); + public: + void broadcast(const std::vector &processor_mapping, + const std::vector > &mappings); + void receive_results(std::vector &processor_mapping, + const std::vector &constraint_indexes, + std::vector > &mappings, + std::map &acquired); + protected: + std::vector processors; + std::vector > instances; + protected: + RtUserEvent local_done_event; + mutable std::set done_events; + std::set held_references; + }; + + /** + * \class MustEpochMappingExchange + * A class for exchanging the mapping decisions for + * specific constraints for a must epoch launch + */ + class MustEpochMappingExchange : public AllGatherCollective { + public: + struct ConstraintInfo { + std::vector instances; + ShardID origin_shard; + int weight; + }; + public: + MustEpochMappingExchange(ReplicateContext *ctx, + CollectiveID collective_id); + MustEpochMappingExchange(const MustEpochMappingExchange &rhs); + virtual ~MustEpochMappingExchange(void); + public: + MustEpochMappingExchange& operator=(const MustEpochMappingExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + void exchange_must_epoch_mappings(ShardID shard_id, + size_t total_shards, size_t total_constraints, + const std::vector &local_tasks, + const std::vector &all_tasks, + std::vector &processor_mapping, + const std::vector &constraint_indexes, + std::vector > &mappings, + const std::vector &mapping_weights, + std::map &acquired); + protected: + std::map processors; + std::map constraints; + protected: + RtUserEvent local_done_event; + std::set done_events; + std::set held_references; + }; + + /** + * \class MustEpochDependenceExchange + * A class for exchanging the mapping dependence events for all + * the single tasks in a must epoch launch so we can know which + * order the point tasks are being mapped in. + */ + class MustEpochDependenceExchange : public AllGatherCollective { + public: + MustEpochDependenceExchange(ReplicateContext *ctx, + CollectiveIndexLocation loc); + MustEpochDependenceExchange(const MustEpochDependenceExchange &rhs); + virtual ~MustEpochDependenceExchange(void); + public: + MustEpochDependenceExchange& operator=( + const MustEpochDependenceExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + void exchange_must_epoch_dependences( + std::map &mapped_events); + protected: + std::map mapping_dependences; + }; + + /** + * \class MustEpochCompletionExchange + * A class for exchanging the local mapping and completion events + * for all the tasks in a must epoch operation + */ + class MustEpochCompletionExchange : public AllGatherCollective { + public: + MustEpochCompletionExchange(ReplicateContext *ctx, + CollectiveIndexLocation loc); + MustEpochCompletionExchange(const MustEpochCompletionExchange &rhs); + virtual ~MustEpochCompletionExchange(void); + public: + MustEpochCompletionExchange& operator=( + const MustEpochCompletionExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + void exchange_must_epoch_completion(RtEvent mapped, ApEvent complete, + std::set &tasks_mapped, + std::set &tasks_complete); + protected: + std::set tasks_mapped; + std::set tasks_complete; + }; + + /** + * \class ShardedMappingExchange + * A class for exchanging the names of instances and mapping dependence + * events for sharded mapping operations. + */ + class ShardedMappingExchange : public AllGatherCollective { + public: + ShardedMappingExchange(CollectiveIndexLocation loc, ReplicateContext *ctx, + ShardID shard_id, bool check_mappings); + ShardedMappingExchange(const ShardedMappingExchange &rhs); + virtual ~ShardedMappingExchange(void); + public: + ShardedMappingExchange& operator=(const ShardedMappingExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + void initiate_exchange(const InstanceSet &mappings, + const std::vector &views); + void complete_exchange(Operation *op, ShardedView *sharded_view, + const InstanceSet &mappings, + std::set &map_applied_events); + public: + const ShardID shard_id; + const bool check_mappings; + protected: + std::map::aligned> mappings; + LegionMap::aligned global_views; + }; + + /** + * \class TemplateIndexExchange + * A class for exchanging proposed templates for trace replay + */ + class TemplateIndexExchange : public AllGatherCollective { + public: + TemplateIndexExchange(ReplicateContext *ctx, CollectiveID id); + TemplateIndexExchange(const TemplateIndexExchange &rhs); + virtual ~TemplateIndexExchange(void); + public: + TemplateIndexExchange& operator=(const TemplateIndexExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + void initiate_exchange(const std::vector &indexes); + void complete_exchange(std::map &index_counts); + protected: + int current_stage; + std::map index_counts; + }; + + /** + * \class UnorderedExchange + * This is a class that exchanges information about unordered operations + * that are ready to execute on each shard so that we can determine which + * operations can be inserted into a task stream + */ + class UnorderedExchange : public AllGatherCollective { + public: + UnorderedExchange(ReplicateContext *ctx, CollectiveIndexLocation loc); + UnorderedExchange(const UnorderedExchange &rhs); + virtual ~UnorderedExchange(void); + public: + UnorderedExchange& operator=(const UnorderedExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + bool exchange_unordered_ops(const std::list &unordered_ops, + std::vector &ready_ops); + protected: + template + void update_future_counts(const int stage, + std::map > &future_counts, + std::map &counts); + template + void pack_counts(Serializer &rez, const std::map &counts); + template + void unpack_counts(const int stage, Deserializer &derez, + std::map &future_counts); + template + void initialize_counts(const std::map &ops, + std::map &counts); + template + void find_ready_ops(const size_t total_shards, + const std::map &final_counts, + const std::map &ops, std::vector &ready_ops); + protected: + int current_stage; + protected: + std::map index_space_counts; + std::map index_partition_counts; + std::map field_space_counts; + // Use the lowest field ID here as the key + std::map,unsigned> field_counts; + std::map logical_region_counts; + // Use the lowest field ID here as the key + std::map,unsigned> detach_counts; + protected: + std::map index_space_deletions; + std::map index_partition_deletions; + std::map field_space_deletions; + // Use the lowest field ID here as the key + std::map,ReplDeletionOp*> field_deletions; + std::map logical_region_deletions; + // Use the lowest field ID here as the key + std::map,ReplDetachOp*> detachments; + }; + + /** + * \class ConsensusMatchBase + * A base class for consensus match + */ + class ConsensusMatchBase : public AllGatherCollective { + public: + struct ConsensusMatchArgs : public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_DEFER_CONSENSUS_MATCH_TASK_ID; + public: + ConsensusMatchArgs(ConsensusMatchBase *b, UniqueID uid) + : LgTaskArgs(uid), base(b) { } + public: + ConsensusMatchBase *const base; + }; + public: + ConsensusMatchBase(ReplicateContext *ctx, CollectiveIndexLocation loc); + ConsensusMatchBase(const ConsensusMatchBase &rhs); + virtual ~ConsensusMatchBase(void); + public: + virtual void complete_exchange(void) = 0; + public: + static void handle_consensus_match(const void *args); + }; + + /** + * \class ConsensusMatchExchange + * This is collective for performing a consensus exchange between + * the shards for a collection of values. + */ + template + class ConsensusMatchExchange : ConsensusMatchBase { + public: + ConsensusMatchExchange(ReplicateContext *ctx, CollectiveIndexLocation loc, + Future to_complete, void *output, ApUserEvent to_trigger); + ConsensusMatchExchange(const ConsensusMatchExchange &rhs); + virtual ~ConsensusMatchExchange(void); + public: + ConsensusMatchExchange& operator=(const ConsensusMatchExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + bool match_elements_async(const void *input, size_t num_elements); + virtual void complete_exchange(void); + protected: + Future to_complete; + T *const output; + const ApUserEvent to_trigger; + std::map element_counts; +#ifdef DEBUG_LEGION + size_t max_elements; +#endif + }; + + /** + * \class VerifyReplicableExchange + * This class exchanges hash values of all the inputs for calls + * into control replication contexts in order to ensure that they + * all are the same. + */ + class VerifyReplicableExchange : public AllGatherCollective { + public: + VerifyReplicableExchange(CollectiveIndexLocation loc, + ReplicateContext *ctx); + VerifyReplicableExchange(const VerifyReplicableExchange &rhs); + virtual ~VerifyReplicableExchange(void); + public: + VerifyReplicableExchange& operator=(const VerifyReplicableExchange &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage); + virtual void unpack_collective_stage(Deserializer &derez, int stage); + public: + typedef std::map,ShardID> ShardHashes; + const ShardHashes& exchange(uint64_t hash[2]); + public: + ShardHashes unique_hashes; + }; + + /** + * \class SlowBarrier + * This class creates a collective that behaves like a barrier, but is + * probably slower than Realm phase barriers. It's useful for cases + * where we may not know whether we are going to perform a barrier or + * not so we grab a collective ID. We can throw away collective IDs + * for free, but in the rare case we actually do need to perform + * the barrier then this class will handle the implementation. + */ + class SlowBarrier : public AllGatherCollective { + public: + SlowBarrier(ReplicateContext *ctx, CollectiveID id); + SlowBarrier(const SlowBarrier &rhs); + virtual ~SlowBarrier(void); + public: + SlowBarrier& operator=(const SlowBarrier &rhs); + public: + virtual void pack_collective_stage(Serializer &rez, int stage) { } + virtual void unpack_collective_stage(Deserializer &derez, int stage) { } + }; + + /** + * \class ReplIndividualTask + * An individual task that is aware that it is + * being executed in a control replication context. + */ + class ReplIndividualTask : public IndividualTask { + public: + ReplIndividualTask(Runtime *rt); + ReplIndividualTask(const ReplIndividualTask &rhs); + virtual ~ReplIndividualTask(void); + public: + ReplIndividualTask& operator=(const ReplIndividualTask &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_prepipeline_stage(void); + virtual void trigger_ready(void); + virtual void replay_analysis(void); + virtual void resolve_false(bool speculated, bool launched); + public: + // Override these so we can broadcast the future result + virtual void trigger_task_complete(bool deferred = false); + public: + void initialize_replication(ReplicateContext *ctx); + void set_sharding_function(ShardingID functor,ShardingFunction *function); + protected: + ShardID owner_shard; + ShardingID sharding_functor; + ShardingFunction *sharding_function; + CollectiveID mapped_collective_id; // id for mapped event broadcast + CollectiveID future_collective_id; // id for the future broadcast + ShardEventTree *mapped_collective; + FutureBroadcast *future_collective; +#ifdef DEBUG_LEGION + public: + inline void set_sharding_collective(ShardingGatherCollective *collective) + { sharding_collective = collective; } + protected: + ShardingGatherCollective *sharding_collective; +#endif + }; + + /** + * \class ReplIndexTask + * An individual task that is aware that it is + * being executed in a control replication context. + */ + class ReplIndexTask : public IndexTask { + public: + ReplIndexTask(Runtime *rt); + ReplIndexTask(const ReplIndexTask &rhs); + virtual ~ReplIndexTask(void); + public: + ReplIndexTask& operator=(const ReplIndexTask &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_prepipeline_stage(void); + virtual void trigger_dependence_analysis(void); + virtual void trigger_ready(void); + virtual void replay_analysis(void); + public: + // Override this so we can exchange reduction results + virtual void trigger_task_complete(bool deferred = false); + // Have to override this too for doing output in the + // case that we misspeculate + virtual void resolve_false(bool speculated, bool launched); + public: + void initialize_replication(ReplicateContext *ctx); + void set_sharding_function(ShardingID functor,ShardingFunction *function); + virtual FutureMapImpl* create_future_map(TaskContext *ctx, + IndexSpace launch_space, IndexSpace shard_space); + void select_sharding_function(ReplicateContext *repl_ctx); + public: + // Methods for supporting intra-index-space mapping dependences + virtual RtEvent find_intra_space_dependence(const DomainPoint &point); + virtual void record_intra_space_dependence(const DomainPoint &point, + const DomainPoint &next, + RtEvent point_mapped); + protected: + ShardingID sharding_functor; + ShardingFunction *sharding_function; + FutureExchange *reduction_collective; + protected: + std::set > unique_intra_space_deps; +#ifdef DEBUG_LEGION + public: + inline void set_sharding_collective(ShardingGatherCollective *collective) + { sharding_collective = collective; } + protected: + ShardingGatherCollective *sharding_collective; +#endif + }; + + /** + * \class ReplMergeCloseOp + * A close operation that is aware that it is being + * executed in a control replication context. + */ + class ReplMergeCloseOp : public MergeCloseOp { + public: + ReplMergeCloseOp(Runtime *runtime); + ReplMergeCloseOp(const ReplMergeCloseOp &rhs); + virtual ~ReplMergeCloseOp(void); + public: + ReplMergeCloseOp& operator=(const ReplMergeCloseOp &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + void set_repl_close_info(RtBarrier mapped_barrier); + virtual void trigger_dependence_analysis(void); + virtual void trigger_mapping(void); + protected: + RtBarrier mapped_barrier; + }; + + /** + * \class ReplFillOp + * A copy operation that is aware that it is being + * executed in a control replication context. + */ + class ReplFillOp : public FillOp { + public: + ReplFillOp(Runtime *rt); + ReplFillOp(const ReplFillOp &rhs); + virtual ~ReplFillOp(void); + public: + ReplFillOp& operator=(const ReplFillOp &rhs); + public: + void initialize_replication(ReplicateContext *ctx); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_prepipeline_stage(void); + virtual void trigger_ready(void); + virtual void replay_analysis(void); + virtual void resolve_false(bool speculated, bool launched); + protected: + ShardingID sharding_functor; + ShardingFunction *sharding_function; + MapperManager *mapper; + public: + CollectiveID mapped_collective_id; + ShardEventTree *mapped_collective; +#ifdef DEBUG_LEGION + public: + inline void set_sharding_collective(ShardingGatherCollective *collective) + { sharding_collective = collective; } + protected: + ShardingGatherCollective *sharding_collective; +#endif + }; + + /** + * \class ReplIndexFillOp + * An index fill operation that is aware that it is + * being executed in a control replication context. + */ + class ReplIndexFillOp : public IndexFillOp { + public: + ReplIndexFillOp(Runtime *rt); + ReplIndexFillOp(const ReplIndexFillOp &rhs); + virtual ~ReplIndexFillOp(void); + public: + ReplIndexFillOp& operator=(const ReplIndexFillOp &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_prepipeline_stage(void); + virtual void trigger_dependence_analysis(void); + virtual void trigger_ready(void); + virtual void replay_analysis(void); + virtual void resolve_false(bool speculated, bool launched); + public: + void initialize_replication(ReplicateContext *ctx); + protected: + ShardingID sharding_functor; + ShardingFunction *sharding_function; + MapperManager *mapper; +#ifdef DEBUG_LEGION + public: + inline void set_sharding_collective(ShardingGatherCollective *collective) + { sharding_collective = collective; } + protected: + ShardingGatherCollective *sharding_collective; +#endif + }; + + /** + * \class ReplCopyOp + * A copy operation that is aware that it is being + * executed in a control replication context. + */ + class ReplCopyOp : public CopyOp { + public: + ReplCopyOp(Runtime *rt); + ReplCopyOp(const ReplCopyOp &rhs); + virtual ~ReplCopyOp(void); + public: + ReplCopyOp& operator=(const ReplCopyOp &rhs); + public: + void initialize_replication(ReplicateContext *ctx); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_prepipeline_stage(void); + virtual void trigger_ready(void); + virtual void replay_analysis(void); + virtual void resolve_false(bool speculated, bool launched); + protected: + ShardingID sharding_functor; + ShardingFunction *sharding_function; + public: + CollectiveID mapped_collective_id; + ShardEventTree *mapped_collective; +#ifdef DEBUG_LEGION + public: + inline void set_sharding_collective(ShardingGatherCollective *collective) + { sharding_collective = collective; } + protected: + ShardingGatherCollective *sharding_collective; +#endif + }; + + /** + * \class ReplIndexCopyOp + * An index fill operation that is aware that it is + * being executed in a control replication context. + */ + class ReplIndexCopyOp : public IndexCopyOp { + public: + ReplIndexCopyOp(Runtime *rt); + ReplIndexCopyOp(const ReplIndexCopyOp &rhs); + virtual ~ReplIndexCopyOp(void); + public: + ReplIndexCopyOp& operator=(const ReplIndexCopyOp &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_prepipeline_stage(void); + virtual void trigger_dependence_analysis(void); + virtual void trigger_ready(void); + virtual void replay_analysis(void); + virtual void resolve_false(bool speculated, bool launched); + virtual ApEvent exchange_indirect_records(const unsigned index, + const ApEvent local_done, const PhysicalTraceInfo &trace_info, + const InstanceSet &instances, const IndexSpace space, + const DomainPoint &key, + LegionVector::aligned &records, const bool sources); + public: + void initialize_replication(ReplicateContext *ctx, + std::vector &indirection_bars, + unsigned &next_indirection_index); + protected: + ShardingID sharding_functor; + ShardingFunction *sharding_function; + std::vector indirection_barriers; + std::vector src_collectives; + std::vector dst_collectives; +#ifdef DEBUG_LEGION + public: + inline void set_sharding_collective(ShardingGatherCollective *collective) + { sharding_collective = collective; } + protected: + ShardingGatherCollective *sharding_collective; +#endif + }; + + /** + * \class ReplDeletionOp + * A deletion operation that is aware that it is + * being executed in a control replication context. + */ + class ReplDeletionOp : public DeletionOp { + public: + ReplDeletionOp(Runtime *rt); + ReplDeletionOp(const ReplDeletionOp &rhs); + virtual ~ReplDeletionOp(void); + public: + ReplDeletionOp& operator=(const ReplDeletionOp &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_ready(void); + virtual void trigger_mapping(void); + virtual void trigger_complete(void); + public: + void initialize_replication(ReplicateContext *ctx, + RtBarrier &deletion_ready_barrier,RtBarrier &deletion_mapping_barrier, + RtBarrier &deletion_execution_barrier, bool is_total, bool is_first, + bool unordered = false); + // Help for handling unordered deletions + void record_unordered_kind( + std::map &index_space_deletions, + std::map &index_partition_deletions, + std::map field_space_deletions, + std::map,ReplDeletionOp*> &field_deletions, + std::map &logical_region_deletions); + protected: + RtBarrier ready_barrier; + RtBarrier mapping_barrier; + RtBarrier execution_barrier; + bool is_total_sharding; + bool is_first_local_shard; + }; + + /** + * \class ReplPendingPartitionOp + * A pending partition operation that knows that its + * being executed in a control replication context + */ + class ReplPendingPartitionOp : public PendingPartitionOp { + public: + ReplPendingPartitionOp(Runtime *rt); + ReplPendingPartitionOp(const ReplPendingPartitionOp &rhs); + virtual ~ReplPendingPartitionOp(void); + public: + ReplPendingPartitionOp& operator=(const ReplPendingPartitionOp &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_complete(void); + }; + + /** + * \class ReplDependentPartitionOp + * A dependent partitioning operation that knows that it + * is being executed in a control replication context + */ + class ReplDependentPartitionOp : public DependentPartitionOp { + public: + class ReplByFieldThunk : public ByFieldThunk { + public: + ReplByFieldThunk(ReplicateContext *ctx, + ShardID target, IndexPartition p); + public: + virtual ApEvent perform(DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances); + virtual void elide_collectives(void) + { gather_collective.elide_collective(); } + protected: + FieldDescriptorGather gather_collective; + }; + class ReplByImageThunk : public ByImageThunk { + public: +#ifdef SHARD_BY_IMAGE + ReplByImageThunk(ReplicateContext *ctx, + IndexPartition p, IndexPartition proj, + ShardID shard_id, size_t total); +#else + ReplByImageThunk(ReplicateContext *ctx, ShardID target, + IndexPartition p, IndexPartition proj, + ShardID shard_id, size_t total); +#endif + public: + virtual ApEvent perform(DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances); + virtual void elide_collectives(void) { collective.elide_collective(); } + protected: +#ifdef SHARD_BY_IMAGE + FieldDescriptorExchange collective; +#else + FieldDescriptorGather collective; +#endif + const ShardID shard_id; + const size_t total_shards; + }; + class ReplByImageRangeThunk : public ByImageRangeThunk { + public: +#ifdef SHARD_BY_IMAGE + ReplByImageRangeThunk(ReplicateContext *ctx, + IndexPartition p, IndexPartition proj, + ShardID shard_id, size_t total); +#else + ReplByImageRangeThunk(ReplicateContext *ctx, ShardID target, + IndexPartition p, IndexPartition proj, + ShardID shard_id, size_t total); +#endif + public: + virtual ApEvent perform(DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances); + virtual void elide_collectives(void) { collective.elide_collective(); } + protected: +#ifdef SHARD_BY_IMAGE + FieldDescriptorExchange collective; +#else + FieldDescriptorGather collective; +#endif + const ShardID shard_id; + const size_t total_shards; + }; + class ReplByPreimageThunk : public ByPreimageThunk { + public: + ReplByPreimageThunk(ReplicateContext *ctx, ShardID target, + IndexPartition p, IndexPartition proj); + public: + virtual ApEvent perform(DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances); + virtual void elide_collectives(void) + { gather_collective.elide_collective(); } + protected: + FieldDescriptorGather gather_collective; + }; + class ReplByPreimageRangeThunk : public ByPreimageRangeThunk { + public: + ReplByPreimageRangeThunk(ReplicateContext *ctx, ShardID target, + IndexPartition p, IndexPartition proj); + public: + virtual ApEvent perform(DependentPartitionOp *op, + RegionTreeForest *forest, ApEvent instances_ready, + const std::vector &instances); + virtual void elide_collectives(void) + { gather_collective.elide_collective(); } + protected: + FieldDescriptorGather gather_collective; + }; + // Nothing special about association for control replication + public: + ReplDependentPartitionOp(Runtime *rt); + ReplDependentPartitionOp(const ReplDependentPartitionOp &rhs); + virtual ~ReplDependentPartitionOp(void); + public: + ReplDependentPartitionOp& operator=(const ReplDependentPartitionOp &rhs); + public: + void initialize_by_field(ReplicateContext *ctx, ShardID target, + ApEvent ready_event, IndexPartition pid, + LogicalRegion handle, LogicalRegion parent, + FieldID fid, MapperID id, MappingTagID tag, + RtBarrier &dependent_partition_bar); + void initialize_by_image(ReplicateContext *ctx, +#ifndef SHARD_BY_IMAGE + ShardID target, +#endif + ApEvent ready_event, IndexPartition pid, + LogicalPartition projection, + LogicalRegion parent, FieldID fid, + MapperID id, MappingTagID tag, + ShardID shard, size_t total_shards, + RtBarrier &dependent_partition_bar); + void initialize_by_image_range(ReplicateContext *ctx, +#ifndef SHARD_BY_IMAGE + ShardID target, +#endif + ApEvent ready_event, IndexPartition pid, + LogicalPartition projection, + LogicalRegion parent, FieldID fid, + MapperID id, MappingTagID tag, + ShardID shard, size_t total_shards, + RtBarrier &dependent_partition_bar); + void initialize_by_preimage(ReplicateContext *ctx, ShardID target, + ApEvent ready_event, IndexPartition pid, + IndexPartition projection, LogicalRegion handle, + LogicalRegion parent, FieldID fid, + MapperID id, MappingTagID tag, + RtBarrier &dependent_partition_bar); + void initialize_by_preimage_range(ReplicateContext *ctx, ShardID target, + ApEvent ready_event, IndexPartition pid, + IndexPartition projection, LogicalRegion handle, + LogicalRegion parent, FieldID fid, + MapperID id, MappingTagID tag, + RtBarrier &dependent_partition_bar); + void initialize_by_association(ReplicateContext *ctx,LogicalRegion domain, + LogicalRegion domain_parent, FieldID fid, + IndexSpace range, MapperID id, MappingTagID tag, + RtBarrier &dependent_partition_bar); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + // Need to pick our sharding functor + virtual void trigger_dependence_analysis(void); + virtual void trigger_ready(void); + virtual void finalize_mapping(void); + protected: + void select_sharding_function(void); + protected: + ShardingFunction *sharding_function; + RtBarrier mapping_barrier; +#ifdef DEBUG_LEGION + public: + inline void set_sharding_collective(ShardingGatherCollective *collective) + { sharding_collective = collective; } + protected: + ShardingGatherCollective *sharding_collective; +#endif + }; + + /** + * \class ReplMustEpochOp + * A must epoch operation that is aware that it is + * being executed in a control replication context + */ + class ReplMustEpochOp : public MustEpochOp { + public: + ReplMustEpochOp(Runtime *rt); + ReplMustEpochOp(const ReplMustEpochOp &rhs); + virtual ~ReplMustEpochOp(void); + public: + ReplMustEpochOp& operator=(const ReplMustEpochOp &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual FutureMapImpl* create_future_map(TaskContext *ctx, + const Domain &domain, IndexSpace shard_space, RtUserEvent deleted); + virtual void instantiate_tasks(InnerContext *ctx, + const MustEpochLauncher &launcher); + virtual MapperManager* invoke_mapper(void); + virtual void map_and_distribute(std::set &tasks_mapped, + std::set &tasks_complete); + virtual bool has_prepipeline_stage(void) const { return true; } + virtual void trigger_prepipeline_stage(void); + virtual void trigger_commit(void); + void map_replicate_tasks(void) const; + void distribute_replicate_tasks(void) const; + public: + void initialize_replication(ReplicateContext *ctx); + Domain get_shard_domain(void) const; + protected: + ShardingID sharding_functor; + ShardingFunction *sharding_function; + CollectiveID mapping_collective_id; + bool collective_map_must_epoch_call; + MustEpochMappingBroadcast *mapping_broadcast; + MustEpochMappingExchange *mapping_exchange; + MustEpochDependenceExchange *dependence_exchange; + MustEpochCompletionExchange *completion_exchange; + std::set shard_single_tasks; +#ifdef DEBUG_LEGION + public: + inline void set_sharding_collective(ShardingGatherCollective *collective) + { sharding_collective = collective; } + protected: + ShardingGatherCollective *sharding_collective; +#endif + }; + + /** + * \class ReplTimingOp + * A timing operation that is aware that it is + * being executed in a control replication context + */ + class ReplTimingOp : public TimingOp { + public: + ReplTimingOp(Runtime *rt); + ReplTimingOp(const ReplTimingOp &rhs); + virtual ~ReplTimingOp(void); + public: + ReplTimingOp& operator=(const ReplTimingOp &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_mapping(void); + virtual void deferred_execute(void); + public: + inline void set_timing_collective(ValueBroadcast *collective) + { timing_collective = collective; } + protected: + ValueBroadcast *timing_collective; + }; + + /** + * \class ReplAllReduceOp + * An all-reduce operation that is aware that it is + * being executed in a control replication context + */ + class ReplAllReduceOp : public AllReduceOp { + public: + ReplAllReduceOp(Runtime *rt); + ReplAllReduceOp(const ReplAllReduceOp &rhs); + virtual ~ReplAllReduceOp(void); + public: + ReplAllReduceOp& operator=(const ReplAllReduceOp &rhs); + public: + void initialize_replication(ReplicateContext *ctx); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void deferred_execute(void); + protected: + void *result_buffer; + FutureExchange *exchange_collective; + AllReduceOpCollective *all_reduce_collective; + }; + + /** + * \class ReplFenceOp + * A fence operation that is aware that it is being + * executed in a control replicated context. Currently + * this only applies to mixed and execution fences. + */ + class ReplFenceOp : public FenceOp { + public: + ReplFenceOp(Runtime *rt); + ReplFenceOp(const ReplFenceOp &rhs); + virtual ~ReplFenceOp(void); + public: + ReplFenceOp& operator=(const ReplFenceOp &rhs); + public: + Future initialize_repl_fence(ReplicateContext *ctx, FenceKind kind, + bool need_future, bool track = true); + public: + virtual void activate(void); + virtual void deactivate(void); + public: + virtual void trigger_mapping(void); + protected: + RtBarrier mapping_fence_barrier; + ApBarrier execution_fence_barrier; + }; + + /** + * \class ReplMapOp + * An inline mapping operation that is aware that it is being + * executed in a control replicated context. We require that + * any inline mapping be mapped on all shards before we consider + * it mapped on any shard. The reason for this is that inline + * mappings can act like a kind of communication between shards + * where they are all reading/writing to the same logical region. + */ + class ReplMapOp : public MapOp { + public: + ReplMapOp(Runtime *rt); + ReplMapOp(const ReplMapOp &rhs); + virtual ~ReplMapOp(void); + public: + ReplMapOp& operator=(const ReplMapOp &rhs); + public: + void initialize_replication(ReplicateContext *ctx, RtBarrier &inline_bar); + RtEvent complete_inline_mapping(RtEvent mapping_applied); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual void trigger_ready(void); + virtual void trigger_mapping(void); + protected: + RtBarrier inline_barrier; + ShardedMappingExchange *exchange; + ValueBroadcast *view_did_broadcast; + ShardedView *sharded_view; + }; + + /** + * \class ReplAttachOp + * An attach operation that is aware that it is being + * executed in a control replicated context. + */ + class ReplAttachOp : public AttachOp { + public: + ReplAttachOp(Runtime *rt); + ReplAttachOp(const ReplAttachOp &rhs); + virtual ~ReplAttachOp(void); + public: + ReplAttachOp& operator=(const ReplAttachOp &rhs); + public: + void initialize_replication(ReplicateContext *ctx, + RtBarrier &resource_bar, + ApBarrier &broadcast_bar, + ApBarrier &reduce_bar); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual void trigger_ready(void); + virtual void trigger_mapping(void); + protected: + RtBarrier resource_barrier; + ApBarrier broadcast_barrier; + ApBarrier reduce_barrier; + RtUserEvent repl_mapping_applied; + InstanceRef external_instance; + ShardedMappingExchange *exchange; + ValueBroadcast *did_broadcast; + ShardedView *sharded_view; + RtEvent all_mapped_event; + bool exchange_complete; + }; + + /** + * \class ReplDetachOp + * An detach operation that is aware that it is being + * executed in a control replicated context. + */ + class ReplDetachOp : public DetachOp { + public: + ReplDetachOp(Runtime *rt); + ReplDetachOp(const ReplDetachOp &rhs); + virtual ~ReplDetachOp(void); + public: + ReplDetachOp& operator=(const ReplDetachOp &rhs); + public: + void initialize_replication(ReplicateContext *ctx, + RtBarrier &resource_bar); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual void trigger_ready(void); + virtual void trigger_mapping(void); + virtual void select_sources(const unsigned index, + const InstanceRef &target, + const InstanceSet &sources, + std::vector &ranking); + public: + // Help for unordered detachments + void record_unordered_kind( + std::map,ReplDetachOp*> &detachments); + public: + RtBarrier resource_barrier; + }; + + /** + * \class ReplTraceOp + * Base class for all replicated trace operations + */ + class ReplTraceOp : public ReplFenceOp { + public: + ReplTraceOp(Runtime *rt); + ReplTraceOp(const ReplTraceOp &rhs); + virtual ~ReplTraceOp(void); + public: + ReplTraceOp& operator=(const ReplTraceOp &rhs); + public: + virtual void execute_dependence_analysis(void); + virtual void sync_for_replayable_check(void); + virtual bool exchange_replayable(ReplicateContext *ctx, bool replayable); + virtual void elide_fences_pre_sync(void); + virtual void elide_fences_post_sync(void); + protected: + LegionTrace *local_trace; + }; + + /** + * \class ReplTraceCaptureOp + * Control replicated version of the TraceCaptureOp + */ + class ReplTraceCaptureOp : public ReplTraceOp { + public: + static const AllocationType alloc_type = TRACE_CAPTURE_OP_ALLOC; + public: + ReplTraceCaptureOp(Runtime *rt); + ReplTraceCaptureOp(const ReplTraceCaptureOp &rhs); + virtual ~ReplTraceCaptureOp(void); + public: + ReplTraceCaptureOp& operator=(const ReplTraceCaptureOp &rhs); + public: + void initialize_capture(ReplicateContext *ctx, + bool has_blocking_call, bool remove_trace_reference); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual const char* get_logging_name(void) const; + virtual OpKind get_operation_kind(void) const; + virtual void trigger_dependence_analysis(void); + virtual void trigger_mapping(void); + virtual void sync_for_replayable_check(void); + virtual bool exchange_replayable(ReplicateContext *ctx, bool replayable); + virtual void elide_fences_pre_sync(void); + virtual void elide_fences_post_sync(void); + protected: + PhysicalTemplate *current_template; + CollectiveID replayable_collective_id; + CollectiveID replay_sync_collective_id; + CollectiveID pre_elide_fences_collective_id; + CollectiveID post_elide_fences_collective_id; + bool has_blocking_call; + bool remove_trace_reference; + }; + + /** + * \class ReplTraceCompleteOp + * Control replicated version of TraceCompleteOp + */ + class ReplTraceCompleteOp : public ReplTraceOp { + public: + static const AllocationType alloc_type = TRACE_COMPLETE_OP_ALLOC; + public: + ReplTraceCompleteOp(Runtime *rt); + ReplTraceCompleteOp(const ReplTraceCompleteOp &rhs); + virtual ~ReplTraceCompleteOp(void); + public: + ReplTraceCompleteOp& operator=(const ReplTraceCompleteOp &rhs); + public: + void initialize_complete(ReplicateContext *ctx, bool has_blocking_call); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual const char* get_logging_name(void) const; + virtual OpKind get_operation_kind(void) const; + virtual void trigger_dependence_analysis(void); + virtual void trigger_mapping(void); + virtual void sync_for_replayable_check(void); + virtual bool exchange_replayable(ReplicateContext *ctx, bool replayable); + virtual void elide_fences_pre_sync(void); + virtual void elide_fences_post_sync(void); + protected: + PhysicalTemplate *current_template; + ApEvent template_completion; + CollectiveID replayable_collective_id; + CollectiveID replay_sync_collective_id; + CollectiveID pre_elide_fences_collective_id; + CollectiveID post_elide_fences_collective_id; + bool replayed; + bool has_blocking_call; + }; + + /** + * \class ReplTraceReplayOp + * Control replicated version of TraceReplayOp + */ + class ReplTraceReplayOp : public ReplTraceOp { + public: + static const AllocationType alloc_type = TRACE_REPLAY_OP_ALLOC; + public: + ReplTraceReplayOp(Runtime *rt); + ReplTraceReplayOp(const ReplTraceReplayOp &rhs); + virtual ~ReplTraceReplayOp(void); + public: + ReplTraceReplayOp& operator=(const ReplTraceReplayOp &rhs); + public: + void initialize_replay(ReplicateContext *ctx, LegionTrace *trace); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual const char* get_logging_name(void) const; + virtual OpKind get_operation_kind(void) const; + virtual void trigger_dependence_analysis(void); + virtual void pack_remote_operation(Serializer &rez, AddressSpaceID target, + std::set &applied) const; + protected: + // This a parameter that controls how many rounds of template + // selection we want shards to go through before giving up + // and doing a capture. The trick is to get all the shards to + // agree on the template. Each round will have each shard + // propose twice as many viable traces the previous round so + // we get some nice exponential back-off properties. Increase + // the number of rounds if you want them to try for longer. + static const int TRACE_SELECTION_ROUNDS = 2; + CollectiveID trace_selection_collective_ids[TRACE_SELECTION_ROUNDS]; + }; + + /** + * \class ReplTraceBeginOp + * Control replicated version of trace begin op + */ + class ReplTraceBeginOp : public ReplTraceOp { + public: + static const AllocationType alloc_type = TRACE_BEGIN_OP_ALLOC; + public: + ReplTraceBeginOp(Runtime *rt); + ReplTraceBeginOp(const ReplTraceBeginOp &rhs); + virtual ~ReplTraceBeginOp(void); + public: + ReplTraceBeginOp& operator=(const ReplTraceBeginOp &rhs); + public: + void initialize_begin(ReplicateContext *ctx, LegionTrace *trace); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual const char* get_logging_name(void) const; + virtual OpKind get_operation_kind(void) const; + }; + + /** + * \class ReplTraceSummaryOp + * Control replicated version of TraceSummaryOp + */ + class ReplTraceSummaryOp : public ReplTraceOp { + public: + static const AllocationType alloc_type = TRACE_SUMMARY_OP_ALLOC; + public: + ReplTraceSummaryOp(Runtime *rt); + ReplTraceSummaryOp(const ReplTraceSummaryOp &rhs); + virtual ~ReplTraceSummaryOp(void); + public: + ReplTraceSummaryOp& operator=(const ReplTraceSummaryOp &rhs); + public: + void initialize_summary(ReplicateContext *ctx, + ShardedPhysicalTemplate *tpl, + Operation *invalidator); + void perform_logging(void); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual const char* get_logging_name(void) const; + virtual OpKind get_operation_kind(void) const; + public: + virtual void trigger_dependence_analysis(void); + virtual void trigger_ready(void); + virtual void trigger_mapping(void); + virtual void pack_remote_operation(Serializer &rez, AddressSpaceID target, + std::set &applied) const; + protected: + PhysicalTemplate *current_template; + }; + + /** + * \class ShardMapping + * A mapping from the shard IDs to their address spaces + */ + class ShardMapping : public Collectable { + public: + ShardMapping(void); + ShardMapping(const ShardMapping &rhs); + ShardMapping(const std::vector &spaces); + ~ShardMapping(void); + public: + ShardMapping& operator=(const ShardMapping &rhs); + AddressSpaceID operator[](unsigned idx) const; + AddressSpaceID& operator[](unsigned idx); + public: + inline size_t size(void) const { return address_spaces.size(); } + inline void resize(size_t size) { address_spaces.resize(size); } + public: + void pack_mapping(Serializer &rez) const; + void unpack_mapping(Deserializer &derez); + protected: + std::vector address_spaces; + }; + + /** + * \class ShardManager + * This is a class that manages the execution of one or + * more shards for a given control replication context on + * a single node. It provides support for doing broadcasts, + * reductions, and exchanges of information between the + * variaous shard tasks. + */ + class ShardManager : public Mapper::SelectShardingFunctorInput, + public Collectable { + public: + struct ShardManagerLaunchArgs : + public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_CONTROL_REP_LAUNCH_TASK_ID; + public: + ShardManagerLaunchArgs(ShardTask *s) + : LgTaskArgs(s->get_unique_op_id()), + shard(s) { } + public: + ShardTask *const shard; + }; + struct ShardManagerDeleteArgs : + public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_CONTROL_REP_DELETE_TASK_ID; + public: + ShardManager *manager; + }; + public: + ShardManager(Runtime *rt, ReplicationID repl_id, + bool control, bool top, size_t total_shards, + AddressSpaceID owner_space, SingleTask *original = NULL, + RtBarrier startup_barrier = RtBarrier::NO_RT_BARRIER); + ShardManager(const ShardManager &rhs); + ~ShardManager(void); + public: + ShardManager& operator=(const ShardManager &rhs); + public: + inline ApBarrier get_pending_partition_barrier(void) const + { return pending_partition_barrier; } + inline RtBarrier get_creation_barrier(void) const + { return creation_barrier; } + inline RtBarrier get_deletion_ready_barrier(void) const + { return deletion_ready_barrier; } + inline RtBarrier get_deletion_mapping_barrier(void) const + { return deletion_mapping_barrier; } + inline RtBarrier get_deletion_execution_barrier(void) const + { return deletion_mapping_barrier; } + inline RtBarrier get_inline_mapping_barrier(void) const + { return inline_mapping_barrier; } + inline RtBarrier get_external_resource_barrier(void) const + { return external_resource_barrier; } + inline RtBarrier get_mapping_fence_barrier(void) const + { return mapping_fence_barrier; } + inline RtBarrier get_trace_recording_barrier(void) const + { return trace_recording_barrier; } + inline RtBarrier get_summary_fence_barrier(void) const + { return summary_fence_barrier; } + inline ApBarrier get_execution_fence_barrier(void) const + { return execution_fence_barrier; } + inline ApBarrier get_attach_broadcast_barrier(void) const + { return attach_broadcast_barrier; } + inline ApBarrier get_attach_reduce_barrier(void) const + { return attach_reduce_barrier; } + inline RtBarrier get_dependent_partition_barrier(void) const + { return dependent_partition_barrier; } + inline RtBarrier get_semantic_attach_barrier(void) const + { return semantic_attach_barrier; } + inline ApBarrier get_inorder_barrier(void) const + { return inorder_barrier; } + inline RtBarrier get_callback_barrier(void) const + { return callback_barrier; } +#ifdef DEBUG_LEGION_COLLECTIVES + inline RtBarrier get_collective_check_barrier(void) const + { return collective_check_barrier; } + inline RtBarrier get_close_check_barrier(void) const + { return close_check_barrier; } +#endif + public: + inline ShardMapping& get_mapping(void) const + { return *address_spaces; } + inline AddressSpaceID get_shard_space(ShardID sid) const + { return (*address_spaces)[sid]; } + inline bool is_first_local_shard(ShardTask *task) const + { return (local_shards[0] == task); } + inline const std::set& get_unique_shard_spaces(void) const + { return unique_shard_spaces; } + public: + void set_shard_mapping(const std::vector &shard_mapping); + void set_address_spaces(const std::vector &spaces); + void create_callback_barrier(size_t arrival_count); + ShardTask* create_shard(ShardID id, Processor target); + void extract_event_preconditions(const std::deque &insts); + void launch(void); + void distribute_shards(AddressSpaceID target, + const std::vector &shards); + void unpack_shards_and_launch(Deserializer &derez); + void launch_shard(ShardTask *task, + RtEvent precondition = RtEvent::NO_RT_EVENT) const; + void complete_startup_initialization(void) const; + // Return true if we have a shard on every address space + bool is_total_sharding(void); + public: + void handle_post_mapped(bool local, RtEvent precondition); + void handle_post_execution(const void *res, size_t res_size, + bool owned, bool local); + void trigger_task_complete(bool local); + void trigger_task_commit(bool local); + public: + void send_collective_message(ShardID target, Serializer &rez); + void handle_collective_message(Deserializer &derez); + public: + void send_future_map_request(ShardID target, Serializer &rez); + void handle_future_map_request(Deserializer &derez); + public: + void send_equivalence_set_request(ShardID target, Serializer &rez); + void handle_equivalence_set_request(Deserializer &derez); + public: + void send_intra_space_dependence(ShardID target, Serializer &rez); + void handle_intra_space_dependence(Deserializer &derez); + public: + void broadcast_resource_update(ShardTask *source, Serializer &rez, + std::set &applied_events); + void handle_resource_update(Deserializer &derez); + public: + void send_trace_event_request(ShardedPhysicalTemplate *physical_template, + ShardID shard_source, AddressSpaceID template_source, + size_t template_index, ApEvent event, + AddressSpaceID event_space, RtUserEvent done_event); + void send_trace_event_response(ShardedPhysicalTemplate *physical_template, + AddressSpaceID template_source, ApEvent event, + ApBarrier result, RtUserEvent done_event); + void send_trace_update(ShardID target, Serializer &rez); + void handle_trace_update(Deserializer &derez, AddressSpaceID source); + public: + static void handle_launch(const void *args); + static void handle_delete(const void *args); + public: + static void handle_launch(Deserializer &derez, Runtime *rt, + AddressSpaceID source); + static void handle_delete(Deserializer &derez, Runtime *rt); + static void handle_post_mapped(Deserializer &derez, Runtime *rt); + static void handle_post_execution(Deserializer &derez, Runtime *rt); + static void handle_trigger_complete(Deserializer &derez, Runtime *rt); + static void handle_trigger_commit(Deserializer &derez, Runtime *rt); + static void handle_collective_message(Deserializer &derez, Runtime *rt); + static void handle_future_map_request(Deserializer &derez, Runtime *rt); + static void handle_top_view_request(Deserializer &derez, Runtime *rt, + AddressSpaceID request_source); + static void handle_top_view_response(Deserializer &derez, Runtime *rt); + static void handle_eq_request(Deserializer &derez, Runtime *rt); + static void handle_intra_space_dependence(Deserializer &derez, + Runtime *rt); + static void handle_resource_update(Deserializer &derez, Runtime *rt); + static void handle_trace_event_request(Deserializer &derez, Runtime *rt, + AddressSpaceID request_source); + static void handle_trace_event_response(Deserializer &derez); + static void handle_trace_update(Deserializer &derez, Runtime *rt, + AddressSpaceID source); + static void handle_barrier_refresh(Deserializer &derez, Runtime *rt); + public: + ShardingFunction* find_sharding_function(ShardingID sid); + public: + void create_instance_top_view(PhysicalManager *manager, + AddressSpaceID source, + ReplicateContext *request_context, + AddressSpaceID request_source, + bool handle_now = false); + void perform_global_registration_callbacks( + Realm::DSOReferenceImplementation *dso, RtEvent local_done, + RtEvent global_done, std::set &preconditions); + bool perform_semantic_attach(void); + public: + Runtime *const runtime; + const ReplicationID repl_id; + const AddressSpaceID owner_space; + const size_t total_shards; + SingleTask *const original_task; + const bool control_replicated; + const bool top_level_task; + protected: + mutable LocalLock manager_lock; + // Inheritted from Mapper::SelectShardingFunctorInput + // std::vector shard_mapping; + ShardMapping* address_spaces; + std::vector local_shards; + protected: + // There are four kinds of signals that come back from + // the execution of the shards: + // - mapping complete + // - future result + // - task complete + // - task commit + // The owner applies these to the original task object only + // after they have occurred for all the shards + unsigned local_mapping_complete, remote_mapping_complete; + unsigned local_execution_complete, remote_execution_complete; + unsigned trigger_local_complete, trigger_remote_complete; + unsigned trigger_local_commit, trigger_remote_commit; + unsigned remote_constituents; + unsigned semantic_attach_counter; + void* local_future_result; size_t local_future_size; + bool local_future_set; + std::set mapping_preconditions; + protected: + RtBarrier startup_barrier; + ApBarrier pending_partition_barrier; + RtBarrier creation_barrier; + RtBarrier deletion_ready_barrier; + RtBarrier deletion_mapping_barrier; + RtBarrier deletion_execution_barrier; + RtBarrier inline_mapping_barrier; + RtBarrier external_resource_barrier; + RtBarrier mapping_fence_barrier; + RtBarrier trace_recording_barrier; + RtBarrier summary_fence_barrier; + ApBarrier execution_fence_barrier; + ApBarrier attach_broadcast_barrier; + ApBarrier attach_reduce_barrier; + RtBarrier dependent_partition_barrier; + RtBarrier semantic_attach_barrier; + ApBarrier inorder_barrier; + RtBarrier callback_barrier; +#ifdef DEBUG_LEGION_COLLECTIVES + RtBarrier collective_check_barrier; + RtBarrier close_check_barrier; +#endif + protected: + std::map sharding_functions; + protected: + // A unique set of address spaces on which shards exist + std::set unique_shard_spaces; + std::set > + unique_registration_callbacks; + }; + + }; // namespace Internal +}; // namespace Legion + +#endif // __LEGION_REPLICATION_H__ diff --git a/runtime/legion/legion_spy.h b/runtime/legion/legion_spy.h index 7419ee0ca8..d854ddb820 100644 --- a/runtime/legion/legion_spy.h +++ b/runtime/legion/legion_spy.h @@ -872,6 +872,24 @@ namespace Legion { #endif } + static inline void log_replication(UniqueID uid, ReplicationID repl_id, + bool control_replicated) + { + log_spy.print("Replicate Task %llu %d %d", uid, repl_id, + (control_replicated ? 1 : 0)); + } + + static inline void log_shard(ReplicationID repl_id, + ShardID sid, UniqueID uid) + { + log_spy.print("Replicate Shard %d %d %llu", repl_id, sid, uid); + } + + static inline void log_owner_shard(UniqueID uid, ShardID sid) + { + log_spy.print("Owner Shard %llu %d", uid, sid); + } + static inline void log_intra_space_dependence(UniqueID point_id, const DomainPoint &point) { diff --git a/runtime/legion/legion_tasks.cc b/runtime/legion/legion_tasks.cc index e91339adf4..22afef03c6 100644 --- a/runtime/legion/legion_tasks.cc +++ b/runtime/legion/legion_tasks.cc @@ -13,7 +13,6 @@ * limitations under the License. */ - #include "legion/region_tree.h" #include "legion/legion_tasks.h" #include "legion/legion_spy.h" @@ -23,6 +22,7 @@ #include "legion/legion_instances.h" #include "legion/legion_analysis.h" #include "legion/legion_views.h" +#include "legion/legion_replication.h" #include @@ -596,6 +596,7 @@ namespace Legion { options_selected = false; map_origin = false; request_valid_instances = false; + replicate = false; true_guard = PredEvent::NO_PRED_EVENT; false_guard = PredEvent::NO_PRED_EVENT; local_cached = false; @@ -686,6 +687,7 @@ namespace Legion { } rez.serialize(request_valid_instances); rez.serialize(execution_fence_event); + rez.serialize(replicate); rez.serialize(true_guard); rez.serialize(false_guard); rez.serialize(early_mapped_regions.size()); @@ -730,6 +732,7 @@ namespace Legion { } derez.deserialize(request_valid_instances); derez.deserialize(execution_fence_event); + derez.deserialize(replicate); derez.deserialize(true_guard); derez.deserialize(false_guard); size_t num_early; @@ -945,6 +948,31 @@ namespace Legion { target_proc = options.initial_proc; stealable = options.stealable; map_origin = options.map_locally; + replicate = options.replicate; + if (replicate && !runtime->unsafe_mapper) + { + // Reduction-only privileges and relaxed coherence modes + // are not permitted for tasks that are going to be replicated + for (unsigned idx = 0; idx < regions.size(); idx++) + { + if (IS_REDUCE(regions[idx])) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s requested to replicate task %s (UID %lld) " + "but region requirement %d has reduction privileges. " + "Tasks with reduction-only privileges are not " + "permitted to be replicated.", + mapper->get_mapper_name(), get_task_name(), + get_unique_id(), idx) + else if (!IS_EXCLUSIVE(regions[idx])) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s requested to replicate task %s (UID %lld) " + "but region requirement %d has relaxed coherence. " + "Tasks with relaxed coherence modes are not " + "permitted to be replicated.", + mapper->get_mapper_name(), get_task_name(), + get_unique_id(), idx) + } + } request_valid_instances = options.valid_instances; if (parent_priority != options.parent_priority) { @@ -1624,6 +1652,7 @@ namespace Legion { this->speculated = rhs->speculated; this->parent_task = rhs->parent_task; this->map_origin = rhs->map_origin; + this->replicate = rhs->replicate; this->sharding_space = rhs->sharding_space; this->request_valid_instances = rhs->request_valid_instances; // From TaskOp @@ -1675,8 +1704,8 @@ namespace Legion { runtime->find_projection_function(regions[idx].projection); if (function->is_invertible) assert(false); // TODO: implement dependent launches for inline - regions[idx].region = - function->project_point(this, idx, runtime, index_point); + regions[idx].region = function->project_point(this, idx, runtime, + index_domain, index_point); // Update the region requirement kind regions[idx].handle_type = LEGION_SINGULAR_PROJECTION; } @@ -1890,6 +1919,15 @@ namespace Legion { trigger_task_commit(); } + //-------------------------------------------------------------------------- + /*static*/ void TaskOp::handle_deferred_task_complete(const void *args) + //-------------------------------------------------------------------------- + { + const DeferredTaskCompleteArgs *dargs = + (const DeferredTaskCompleteArgs*)args; + dargs->task->trigger_task_complete(true/*deferred*/); + } + //-------------------------------------------------------------------------- /*static*/ void TaskOp::log_requirement(UniqueID uid, unsigned idx, const RegionRequirement &req) @@ -2074,6 +2112,7 @@ namespace Legion { first_mapping = true; execution_context = NULL; remote_trace_info = NULL; + shard_manager = NULL; leaf_cached = false; inner_cached = false; } @@ -2104,6 +2143,8 @@ namespace Legion { delete execution_context; if (remote_trace_info != NULL) delete remote_trace_info; + if ((shard_manager != NULL) && shard_manager->remove_reference()) + delete shard_manager; #ifdef DEBUG_LEGION premapped_instances.clear(); assert(!deferred_complete_mapping.exists()); @@ -2153,6 +2194,24 @@ namespace Legion { regions[idx].privilege_fields.empty(); } + //-------------------------------------------------------------------------- + void SingleTask::clone_single_from(SingleTask *rhs) + //-------------------------------------------------------------------------- + { + this->clone_task_op_from(rhs, this->target_proc, + false/*stealable*/, true/*duplicate*/); + this->virtual_mapped = rhs->virtual_mapped; + this->no_access_regions = rhs->no_access_regions; + this->target_processors = rhs->target_processors; + this->physical_instances = rhs->physical_instances; + // no need to copy the control replication map + this->selected_variant = rhs->selected_variant; + this->task_priority = rhs->task_priority; + this->shard_manager = rhs->shard_manager; + // For now don't copy anything else below here + // In the future we may need to copy the profiling requests + } + //-------------------------------------------------------------------------- void SingleTask::pack_single_task(Serializer &rez, AddressSpaceID target) //-------------------------------------------------------------------------- @@ -2163,6 +2222,7 @@ namespace Legion { if (map_origin) { rez.serialize(selected_variant); + rez.serialize(task_priority); rez.serialize(target_processors.size()); for (unsigned idx = 0; idx < target_processors.size(); idx++) rez.serialize(target_processors[idx]); @@ -2238,6 +2298,7 @@ namespace Legion { if (map_origin) { derez.deserialize(selected_variant); + derez.deserialize(task_priority); size_t num_target_processors; derez.deserialize(num_target_processors); target_processors.resize(num_target_processors); @@ -2289,6 +2350,39 @@ namespace Legion { derez.deserialize(profiling_priority); } + //-------------------------------------------------------------------------- + void SingleTask::send_remote_context(AddressSpaceID remote_instance, + RemoteTask *remote_ctx) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(remote_instance != runtime->address_space); +#endif + Serializer rez; + { + RezCheck z(rez); + rez.serialize(remote_ctx); + execution_context->pack_remote_context(rez, remote_instance); + } + runtime->send_remote_context_response(remote_instance, rez); + AutoLock o_lock(op_lock); +#ifdef DEBUG_LEGION + assert(remote_instances.find(remote_instance) == remote_instances.end()); +#endif + remote_instances[remote_instance] = remote_ctx; + } + + //-------------------------------------------------------------------------- + void SingleTask::shard_off(RtEvent mapped_precondition) + //-------------------------------------------------------------------------- + { + // Do the stuff to record that this is mapped and executed + complete_mapping(mapped_precondition); + complete_execution(); + trigger_children_complete(); + trigger_children_committed(); + } + //-------------------------------------------------------------------------- void SingleTask::trigger_mapping(void) //-------------------------------------------------------------------------- @@ -2470,10 +2564,10 @@ namespace Legion { } #endif // Prepare the output too - output.chosen_instances.resize(regions.size()); output.chosen_variant = 0; output.postmap_task = false; output.task_priority = 0; + output.postmap_task = false; } //-------------------------------------------------------------------------- @@ -2539,13 +2633,88 @@ namespace Legion { // Only one valid choice in this case, ignore everything else target_processors.push_back(this->target_proc); } + // Sort out any profiling requests that we need to perform + if (!output.task_prof_requests.empty()) + { + profiling_priority = output.profiling_priority; + // If we do any legion specific checks, make sure we ask + // Realm for the proc profiling info so that we can get + // a callback to report our profiling information + bool has_proc_request = false; + // Filter profiling requests into those for copies and the actual task + for (std::set::const_iterator it = + output.task_prof_requests.requested_measurements.begin(); it != + output.task_prof_requests.requested_measurements.end(); it++) + { + if ((*it) > Mapping::PMID_LEGION_FIRST) + { + // If we haven't seen a proc usage yet, then add it + // to the realm requests to ensure we get a callback + // for this task. We know we'll see it before this + // because the measurement IDs are in order + if (!has_proc_request) + task_profiling_requests.push_back( + (ProfilingMeasurementID)Realm::PMID_OP_PROC_USAGE); + // These are legion profiling requests and currently + // are only profiling task information + task_profiling_requests.push_back(*it); + continue; + } + switch ((Realm::ProfilingMeasurementID)*it) + { + case Realm::PMID_OP_PROC_USAGE: + has_proc_request = true; // Then fall through + case Realm::PMID_OP_STATUS: + case Realm::PMID_OP_BACKTRACE: + case Realm::PMID_OP_TIMELINE: + case Realm::PMID_PCTRS_CACHE_L1I: + case Realm::PMID_PCTRS_CACHE_L1D: + case Realm::PMID_PCTRS_CACHE_L2: + case Realm::PMID_PCTRS_CACHE_L3: + case Realm::PMID_PCTRS_IPC: + case Realm::PMID_PCTRS_TLB: + case Realm::PMID_PCTRS_BP: + { + // Just task + task_profiling_requests.push_back(*it); + break; + } + default: + { + REPORT_LEGION_WARNING(LEGION_WARNING_MAPPER_REQUESTED_PROFILING, + "Mapper %s requested a profiling " + "measurement of type %d which is not applicable to " + "task %s (UID %lld) and will be ignored.", + mapper->get_mapper_name(), *it, get_task_name(), + get_unique_id()); + } + } + } +#ifdef DEBUG_LEGION + assert(!profiling_reported.exists()); + assert(outstanding_profiling_requests == 0); +#endif + profiling_reported = Runtime::create_rt_user_event(); + // Increment the number of profiling responses here since we + // know that we're going to get one for launching the task + // No need for the lock since no outstanding physical analyses + // can be running yet + outstanding_profiling_requests = 1; + } + if (!output.copy_prof_requests.empty()) + { + filter_copy_request_kinds(mapper, + output.copy_prof_requests.requested_measurements, + copy_profiling_requests, true/*warn*/); + profiling_priority = output.profiling_priority; + if (!profiling_reported.exists()) + profiling_reported = Runtime::create_rt_user_event(); + } // See whether the mapper picked a variant or a generator VariantImpl *variant_impl = NULL; if (output.chosen_variant > 0) - { variant_impl = runtime->find_variant_impl(task_id, output.chosen_variant, true/*can fail*/); - } else // TODO: invoke a generator if one exists REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, "Invalid mapper output from invocation of '%s' on " @@ -2603,7 +2772,22 @@ namespace Legion { // visible from all the target processors std::set visible_memories; if (!runtime->unsafe_mapper) - runtime->find_visible_memories(target_proc, visible_memories); + { + if (target_processors.size() > 1) + { + // If we have multiple processor, we want the set of + // memories visible to all of them + Machine::MemoryQuery visible_query(runtime->machine); + for (std::vector::const_iterator it = + target_processors.begin(); it != target_processors.end(); it++) + visible_query.has_affinity_to(*it); + for (Machine::MemoryQuery::iterator it = visible_query.begin(); + it != visible_query.end(); it++) + visible_memories.insert(*it); + } + else + runtime->find_visible_memories(target_proc, visible_memories); + } for (unsigned idx = 0; idx < regions.size(); idx++) { // If it was early mapped then it is easy @@ -2971,6 +3155,17 @@ namespace Legion { return inner_ctx; } + //-------------------------------------------------------------------------- + void SingleTask::set_shard_manager(ShardManager *manager) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(shard_manager == NULL); +#endif + shard_manager = manager; + shard_manager->add_reference(); + } + //-------------------------------------------------------------------------- void SingleTask::validate_target_processors( const std::vector &processors) const @@ -3075,6 +3270,8 @@ namespace Legion { } if (conflict_constraint != NULL) { + if (local_mapper == NULL) + local_mapper = runtime->find_mapper(current_proc, map_id); const char *constraint_names[] = { #define CONSTRAINT_NAMES(name, desc) desc, LEGION_LAYOUT_CONSTRAINT_KINDS(CONSTRAINT_NAMES) @@ -3101,6 +3298,9 @@ namespace Legion { { // If the constraint is a no processor constraint we can ignore it if (!execution_constraints.processor_constraint.can_use(kind)) + { + if (local_mapper == NULL) + local_mapper = runtime->find_mapper(current_proc, map_id); REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, "Invalid mapper output. Mapper %s selected variant %d " "for task %s (ID %lld). However, this variant does not " @@ -3108,6 +3308,7 @@ namespace Legion { local_mapper->get_mapper_name(), impl->vid, get_task_name(), get_unique_id(), Processor::get_kind_name(kind)) + } } // Then check the colocation constraints for (std::vector::const_iterator con_it = @@ -3215,6 +3416,8 @@ namespace Legion { get_task_name(), tree_id, req.region.get_tree_id()) const InstanceSet &insts = physical_instances[*iit]; + if (local_mapper == NULL) + local_mapper = runtime->find_mapper(current_proc, map_id); for (unsigned idx = 0; idx < insts.size(); idx++) { const InstanceRef &ref = insts[idx]; @@ -3295,83 +3498,6 @@ namespace Legion { mapper->invoke_map_task(this, &input, &output); // Now we can convert the mapper output into our physical instances finalize_map_task_output(input, output, must_epoch_owner,valid_instances); - // Sort out any profiling requests that we need to perform - if (!output.task_prof_requests.empty()) - { - profiling_priority = output.profiling_priority; - // If we do any legion specific checks, make sure we ask - // Realm for the proc profiling info so that we can get - // a callback to report our profiling information - bool has_proc_request = false; - // Filter profiling requests into those for copies and the actual task - for (std::set::const_iterator it = - output.task_prof_requests.requested_measurements.begin(); it != - output.task_prof_requests.requested_measurements.end(); it++) - { - if ((*it) > Mapping::PMID_LEGION_FIRST) - { - // If we haven't seen a proc usage yet, then add it - // to the realm requests to ensure we get a callback - // for this task. We know we'll see it before this - // because the measurement IDs are in order - if (!has_proc_request) - task_profiling_requests.push_back( - (ProfilingMeasurementID)Realm::PMID_OP_PROC_USAGE); - // These are legion profiling requests and currently - // are only profiling task information - task_profiling_requests.push_back(*it); - continue; - } - switch ((Realm::ProfilingMeasurementID)*it) - { - case Realm::PMID_OP_PROC_USAGE: - has_proc_request = true; // Then fall through - case Realm::PMID_OP_STATUS: - case Realm::PMID_OP_BACKTRACE: - case Realm::PMID_OP_TIMELINE: - case Realm::PMID_PCTRS_CACHE_L1I: - case Realm::PMID_PCTRS_CACHE_L1D: - case Realm::PMID_PCTRS_CACHE_L2: - case Realm::PMID_PCTRS_CACHE_L3: - case Realm::PMID_PCTRS_IPC: - case Realm::PMID_PCTRS_TLB: - case Realm::PMID_PCTRS_BP: - { - // Just task - task_profiling_requests.push_back(*it); - break; - } - default: - { - REPORT_LEGION_WARNING(LEGION_WARNING_MAPPER_REQUESTED_PROFILING, - "Mapper %s requested a profiling " - "measurement of type %d which is not applicable to " - "task %s (UID %lld) and will be ignored.", - mapper->get_mapper_name(), *it, get_task_name(), - get_unique_id()); - } - } - } -#ifdef DEBUG_LEGION - assert(!profiling_reported.exists()); - assert(outstanding_profiling_requests == 0); -#endif - profiling_reported = Runtime::create_rt_user_event(); - // Increment the number of profiling responses here since we - // know that we're going to get one for launching the task - // No need for the lock since no outstanding physical analyses - // can be running yet - outstanding_profiling_requests = 1; - } - if (!output.copy_prof_requests.empty()) - { - filter_copy_request_kinds(mapper, - output.copy_prof_requests.requested_measurements, - copy_profiling_requests, true/*warn*/); - profiling_priority = output.profiling_priority; - if (!profiling_reported.exists()) - profiling_reported = Runtime::create_rt_user_event(); - } if (is_recording()) { @@ -3389,33 +3515,242 @@ namespace Legion { } //-------------------------------------------------------------------------- - RtEvent SingleTask::map_all_regions(ApEvent local_termination_event, - MustEpochOp *must_epoch_op, - const DeferMappingArgs *defer_args) + void SingleTask::invoke_mapper_replicated(MustEpochOp *must_epoch_owner) //-------------------------------------------------------------------------- { - DETAILED_PROFILER(runtime, MAP_ALL_REGIONS_CALL); - // Only do this the first or second time through - if ((defer_args == NULL) || (defer_args->invocation_count < 3)) + if (mapper == NULL) + mapper = runtime->find_mapper(current_proc, map_id); + if (must_epoch_owner != NULL) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s requested to replicate task %s (UID %lld) " + "which is part of a must epoch launch. Replication of " + "tasks in must epoch launches is not permitted.", + mapper->get_mapper_name(), get_task_name(), + get_unique_id()) + Mapper::MapTaskInput input; + Mapper::MapTaskOutput default_output; + Mapper::MapReplicateTaskOutput output; + // Initialize the mapping input which also does all the traversal + // down to the target nodes + std::vector valid_instances(regions.size()); + initialize_map_task_input(input, default_output, + must_epoch_owner, valid_instances); + // Now we can invoke the mapper to do the mapping + mapper->invoke_map_replicate_task(this, &input, &default_output, &output); + if (output.task_mappings.empty()) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s failed to provide any mappings for task %s " + "(UID %lld) in 'map_replicate_task' mapper call.", + mapper->get_mapper_name(), get_task_name(), + get_unique_id()) + // Quick test to see if there is only one output requested in which + // case then there is no replication + else if (output.task_mappings.size() == 1) + { + // Set replicate back to false since this is no longer replicated + replicate = false; + finalize_map_task_output(input, output.task_mappings[0], + must_epoch_owner, valid_instances); + return; + } + else { - if ((defer_args == NULL) || (defer_args->invocation_count < 2)) - { - if (request_valid_instances) +#ifdef DEBUG_LEGION + assert(shard_manager == NULL); +#endif + // First make a shard manager to handle the all the shard tasks + const size_t total_shards = output.task_mappings.size(); + const ReplicationID repl_context = runtime->get_unique_replication_id(); + if (runtime->legion_spy_enabled) + LegionSpy::log_replication(get_unique_id(), repl_context, + !output.control_replication_map.empty()); + if (!output.control_replication_map.empty()) + { + shard_manager = new ShardManager(runtime, repl_context, true/*cr*/, + is_top_level_task(), total_shards, runtime->address_space, this); + shard_manager->add_reference(); + if (output.control_replication_map.size() != total_shards) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Mapper %s specified a non-empty control replication " + "map of size %zd that does not match the requested " + "number of %zd shards for task %s (UID %lld).", + mapper->get_mapper_name(), + output.control_replication_map.size(), total_shards, + get_task_name(), get_unique_id()) + else + shard_manager->set_shard_mapping(output.control_replication_map); + if (!runtime->unsafe_mapper) { - // If the mapper wants valid instances we first need to do our - // versioning analysis and then call the mapper - if ((defer_args == NULL/*first invocation*/) || - (defer_args->invocation_count == 0)) + // Check to make sure that they all picked the same variant + // and that it is a replicable variant + VariantID chosen_variant = output.task_mappings[0].chosen_variant; + for (unsigned idx = 1; idx < total_shards; idx++) { - const RtEvent version_ready_event = - perform_versioning_analysis(false/*post mapper*/); + if (output.task_mappings[idx].chosen_variant != chosen_variant) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Invalid mapper output from invocation of '%s' " + "on mapper %s. Mapper picked different variants " + "%d and %d for task %s (UID %lld) that was " + "designated to be control replicated.", + "map_replicate_task", mapper->get_mapper_name(), + chosen_variant, + output.task_mappings[idx].chosen_variant, + get_task_name(), get_unique_id()) + } + VariantImpl *var_impl = runtime->find_variant_impl(task_id, + chosen_variant, true/*can_fail*/); + // If it's NULL we'll catch it later in the checks + if ((var_impl != NULL) && !var_impl->is_replicable()) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Invalid mapper output from invocation of '%s' on " + "mapper %s. Mapper failed to pick a replicable " + "variant for task %s (UID %lld) that was designated" + " to be control replicated.", "map_replicate_task", + mapper->get_mapper_name(), get_task_name(), + get_unique_id()) + } + } + else + { + shard_manager = new ShardManager(runtime, repl_context, false/*cr*/, + is_top_level_task(), total_shards, runtime->address_space, this); + shard_manager->add_reference(); + if (!runtime->unsafe_mapper) + { + // Currently we only support non-control replication of + // leaf task variants because there is no way to guarantee + // that the physical instances chosen by the sub-operations + // launched by the replicated tasks are not the same and we + // could end up with interfering sub-operations + for (unsigned idx = 0; idx < total_shards; idx++) + { + VariantID variant = output.task_mappings[idx].chosen_variant; + VariantImpl *var_impl = runtime->find_variant_impl(task_id, + variant, true/*can_fail*/); + // If it's NULL we'll catch it later in the checks + if ((var_impl != NULL) && !var_impl->is_leaf()) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Invalid mapper output from invocation of '%s' " + "on mapper %s. Mapper failed to pick a leaf task " + "variant for task %s (UID %lld) that was chosen " + "to be replicated. Only leaf task variants are " + "currently permitted for non-control-replicated " + "task invocations.", "map_replicate_task", + mapper->get_mapper_name(), get_task_name(), + get_unique_id()) + } + } + } + // We're going to store the needed instances locally so we can + // do the mapping when we return on behalf of all the shards + physical_instances.resize(regions.size()); + // Create the shard tasks and have them complete their mapping + for (unsigned shard_idx = 0; shard_idx < total_shards; shard_idx++) + { + Processor target = output.control_replication_map.empty() ? + output.task_mappings[shard_idx].target_procs[0] : + output.control_replication_map[shard_idx]; + ShardTask *shard = shard_manager->create_shard(shard_idx, target); + shard->clone_single_from(this); + // Shard tasks are always effectively mapped locally + shard->map_origin = true; + // Finalize the mapping output + shard->finalize_map_task_output(input,output.task_mappings[shard_idx], + must_epoch_owner, valid_instances); + // All shards can just record themselves as being done their + // mapping now, their mapping effects will actually come back + // through the shard manager + shard->complete_mapping(); + // Now record the instances that we need locally + const std::deque &shard_instances = + shard->get_physical_instances(); + for (unsigned region_idx = 0; + region_idx < regions.size(); region_idx++) + { + if (no_access_regions[region_idx] || + !regions[region_idx].region.exists()) + continue; + const InstanceSet &instances = shard_instances[region_idx]; + InstanceSet &local_instances = physical_instances[region_idx]; + const bool is_write = IS_WRITE(regions[region_idx]); + // No virtual mappings are permitted + if (instances.is_virtual_mapping()) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Invalid mapper output from invocation of '%s' on " + "mapper %s. Mapper selected a virtual mapping for " + "region %d of replicated copy %d of task %s " + "(UID %lld). Virtual mappings are not permitted " + "for replicated tasks.", "map_replicate_task", + mapper->get_mapper_name(), region_idx, shard_idx, + get_task_name(), get_unique_id()) + // For each of the shard instances + for (unsigned idx1 = 0; idx1 < instances.size(); idx1++) + { + const InstanceRef &shard_ref = instances[idx1]; + bool found = false; + for (unsigned idx2 = 0; idx2 < local_instances.size(); idx2++) + { + InstanceRef &local_ref = local_instances[idx2]; + if (shard_ref.get_manager() != local_ref.get_manager()) + continue; + // If this is a write then we need to check for + // overlapping fields to prevent common writes + if (is_write && !(local_ref.get_valid_fields() * + shard_ref.get_valid_fields())) + REPORT_LEGION_ERROR(ERROR_INVALID_MAPPER_OUTPUT, + "Invalid mapper output from invocation of '%s' " + "on mapper %s. Mapper selected the same " + "physical instance for write privilege region " + "%d of two different replicated copies of task " + "%s (UID %lld). All regions with write " + "privileges must be mapped to different " + "physical instances for replicated tasks.", + "map_replicate_task", mapper->get_mapper_name(), + region_idx, get_task_name(), get_unique_id()) + // Update the set of needed fields + local_ref.update_fields(shard_ref.get_valid_fields()); + found = true; + break; + } + if (!found) + local_instances.add_instance(shard_ref); + } + } + } + } + } + + //-------------------------------------------------------------------------- + RtEvent SingleTask::map_all_regions(ApEvent local_termination_event, + MustEpochOp *must_epoch_op, + const DeferMappingArgs *defer_args) + //-------------------------------------------------------------------------- + { + DETAILED_PROFILER(runtime, MAP_ALL_REGIONS_CALL); + // Only do this the first or second time through + if ((defer_args == NULL) || (defer_args->invocation_count < 3)) + { + if ((defer_args == NULL) || (defer_args->invocation_count < 2)) + { + if (request_valid_instances) + { + // If the mapper wants valid instances we first need to do our + // versioning analysis and then call the mapper + if ((defer_args == NULL/*first invocation*/) || + (defer_args->invocation_count == 0)) + { + const RtEvent version_ready_event = + perform_versioning_analysis(false/*post mapper*/); if (version_ready_event.exists() && !version_ready_event.has_triggered()) - return defer_perform_mapping(version_ready_event, must_epoch_op, - defer_args, 1/*invocation count*/); + return defer_perform_mapping(version_ready_event, must_epoch_op, + defer_args, 1/*invocation count*/); } // Now do the mapping call - invoke_mapper(must_epoch_op); + if (is_replicated()) + invoke_mapper_replicated(must_epoch_op); + else + invoke_mapper(must_epoch_op); } else { @@ -3424,13 +3759,16 @@ namespace Legion { if ((defer_args == NULL/*first invocation*/) || (defer_args->invocation_count == 0)) { - invoke_mapper(must_epoch_op); + if (is_replicated()) + invoke_mapper_replicated(must_epoch_op); + else + invoke_mapper(must_epoch_op); const RtEvent version_ready_event = perform_versioning_analysis(true/*post mapper*/); if (version_ready_event.exists() && !version_ready_event.has_triggered()) - return defer_perform_mapping(version_ready_event, must_epoch_op, - defer_args, 1/*invocation count*/); + return defer_perform_mapping(version_ready_event, must_epoch_op, + defer_args, 1/*invocation count*/); } } } @@ -3618,6 +3956,10 @@ namespace Legion { perform_post_mapping(trace_info); } } + // If we are replicating the task then we have to extract the conditions + // under which each of the instances will be ready to be used + if (shard_manager != NULL) + shard_manager->extract_event_preconditions(physical_instances); if (is_recording()) { const TraceInfo trace_info = (remote_trace_info == NULL) ? @@ -3834,6 +4176,13 @@ namespace Legion { assert(regions.size() == physical_instances.size()); assert(regions.size() == no_access_regions.size()); #endif + // If we have a shard manager that means we were replicated so + // we just do the launch directly from the shard manager + if ((shard_manager != NULL) && !is_shard_task()) + { + shard_manager->launch(); + return; + } // If we haven't computed our virtual mapping information // yet (e.g. because we origin mapped) then we have to // do that now @@ -3890,15 +4239,7 @@ namespace Legion { // STEP 2: Set up the task's context { if (!variant->is_leaf()) - { - InnerContext *inner_ctx = new InnerContext(runtime, this, - get_depth(), variant->is_inner(), regions, parent_req_indexes, - virtual_mapped, unique_op_id, execution_fence_event); - if (mapper == NULL) - mapper = runtime->find_mapper(current_proc, map_id); - inner_ctx->configure_context(mapper, task_priority); - execution_context = inner_ctx; - } + execution_context = initialize_inner_execution_context(variant); else execution_context = new LeafContext(runtime, this); // Add a reference to our execution context @@ -4350,6 +4691,19 @@ namespace Legion { target->handle_remote_profiling_response(derez); } + //-------------------------------------------------------------------------- + InnerContext* SingleTask::initialize_inner_execution_context(VariantImpl *v) + //-------------------------------------------------------------------------- + { + InnerContext *inner_ctx = new InnerContext(runtime, this, + get_depth(), v->is_inner(), regions, parent_req_indexes, + virtual_mapped, unique_op_id, execution_fence_event); + if (mapper == NULL) + mapper = runtime->find_mapper(current_proc, map_id); + inner_ctx->configure_context(mapper, task_priority); + return inner_ctx; + } + ///////////////////////////////////////////////////////////// // Multi Task ///////////////////////////////////////////////////////////// @@ -4451,6 +4805,10 @@ namespace Legion { Mapper::SliceTaskInput input; Mapper::SliceTaskOutput output; input.domain_is = internal_space; + if (sharding_space.exists()) + input.sharding_is = sharding_space; + else + input.sharding_is = launch_space->handle; runtime->forest->find_launch_space_domain(internal_space, input.domain); output.verify_correctness = false; if (mapper == NULL) @@ -4857,6 +5215,13 @@ namespace Legion { //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, ACTIVATE_INDIVIDUAL_CALL); + activate_individual_task(); + } + + //-------------------------------------------------------------------------- + void IndividualTask::activate_individual_task(void) + //-------------------------------------------------------------------------- + { activate_single(); predicate_false_result = NULL; predicate_false_size = 0; @@ -4876,22 +5241,15 @@ namespace Legion { //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, DEACTIVATE_INDIVIDUAL_CALL); + deactivate_individual_task(); + runtime->free_individual_task(this); + } + + //-------------------------------------------------------------------------- + void IndividualTask::deactivate_individual_task(void) + //-------------------------------------------------------------------------- + { deactivate_single(); - if (!remote_instances.empty()) - { - UniqueID local_uid = get_unique_id(); - Serializer rez; - { - RezCheck z(rez); - rez.serialize(local_uid); - } - for (std::map::const_iterator it = - remote_instances.begin(); it != remote_instances.end(); it++) - { - runtime->send_remote_context_free(it->first, rez); - } - remote_instances.clear(); - } if (predicate_false_result != NULL) { legion_free(PREDICATE_ALLOC, predicate_false_result, @@ -4904,8 +5262,7 @@ namespace Legion { predicate_false_future = Future(); privilege_paths.clear(); if (!acquired_instances.empty()) - release_acquired_instances(acquired_instances); - runtime->free_individual_task(this); + release_acquired_instances(acquired_instances); } //-------------------------------------------------------------------------- @@ -5050,10 +5407,7 @@ namespace Legion { { set_must_epoch(epoch, index, do_registration); FutureMap map = epoch->get_future_map(); -#ifdef DEBUG_LEGION - map.impl->add_valid_point(index_point); -#endif - result = map.impl->get_future(index_point); + result = map.impl->get_future(index_point, true/*internal only*/); } //-------------------------------------------------------------------------- @@ -5102,6 +5456,19 @@ namespace Legion { void IndividualTask::trigger_dependence_analysis(void) //-------------------------------------------------------------------------- { + perform_base_dependence_analysis(); + ProjectionInfo projection_info; + for (unsigned idx = 0; idx < regions.size(); idx++) + runtime->forest->perform_dependence_analysis(this, idx, regions[idx], + projection_info, + privilege_paths[idx], + map_applied_conditions); + } + + //-------------------------------------------------------------------------- + void IndividualTask::perform_base_dependence_analysis(void) + //-------------------------------------------------------------------------- + { #ifdef DEBUG_LEGION assert(memo_state != MEMO_REQ); assert(privilege_paths.size() == regions.size()); @@ -5126,14 +5493,6 @@ namespace Legion { predicate_false_future.impl->register_dependence(this); // Also have to register any dependences on our predicate register_predicate_dependence(); - ProjectionInfo projection_info; - for (unsigned idx = 0; idx < regions.size(); idx++) - { - runtime->forest->perform_dependence_analysis(this, idx, regions[idx], - projection_info, - privilege_paths[idx], - map_applied_conditions); - } } //-------------------------------------------------------------------------- @@ -5308,41 +5667,59 @@ namespace Legion { // If we succeeded in mapping and it's a leaf task // then we get to mark that we are done mapping RtEvent applied_condition; - if (is_leaf()) - { - if (!map_applied_conditions.empty()) + if (!is_replicated()) + { + // The common path + if (is_leaf()) { - applied_condition = Runtime::merge_events(map_applied_conditions); - map_applied_conditions.clear(); + if (!map_applied_conditions.empty()) + { + applied_condition = Runtime::merge_events(map_applied_conditions); + map_applied_conditions.clear(); + } + // If we mapped remotely we might have a deferred complete mapping + // that we can trigger now + if (deferred_complete_mapping.exists()) + { +#ifdef DEBUG_LEGION + assert(is_remote()); +#endif + Runtime::trigger_event(deferred_complete_mapping,applied_condition); + applied_condition = deferred_complete_mapping; + deferred_complete_mapping = RtUserEvent::NO_RT_USER_EVENT; + } } - // If we mapped remotely we might have a deferred complete mapping - // that we can trigger now - if (deferred_complete_mapping.exists()) + else if (!is_remote()) { + // We did this mapping on the owner #ifdef DEBUG_LEGION - assert(is_remote()); + assert(!deferred_complete_mapping.exists()); #endif - Runtime::trigger_event(deferred_complete_mapping, applied_condition); + deferred_complete_mapping = Runtime::create_rt_user_event(); applied_condition = deferred_complete_mapping; - deferred_complete_mapping = RtUserEvent::NO_RT_USER_EVENT; } - } - else if (!is_remote()) - { - // We did this mapping on the owner + else + { + // We did this mapping remotely so there better be an event #ifdef DEBUG_LEGION - assert(!deferred_complete_mapping.exists()); + assert(deferred_complete_mapping.exists()); #endif - deferred_complete_mapping = Runtime::create_rt_user_event(); - applied_condition = deferred_complete_mapping; + applied_condition = deferred_complete_mapping; + } } else { - // We did this mapping remotely so there better be an event + // Replciated case #ifdef DEBUG_LEGION - assert(deferred_complete_mapping.exists()); + assert(!deferred_complete_mapping.exists()); #endif + deferred_complete_mapping = Runtime::create_rt_user_event(); applied_condition = deferred_complete_mapping; +#ifdef LEGION_SPY + // Still need to do this for Legion Spy + LegionSpy::log_operation_events(unique_op_id, + ApEvent::NO_AP_EVENT, ApEvent::NO_AP_EVENT); +#endif } // Mark that we have completed mapping if (!acquired_instances.empty()) @@ -5418,32 +5795,10 @@ namespace Legion { //-------------------------------------------------------------------------- { return INDIVIDUAL_TASK_KIND; - } - - //-------------------------------------------------------------------------- - void IndividualTask::send_remote_context(AddressSpaceID remote_instance, - RemoteTask *remote_ctx) - //-------------------------------------------------------------------------- - { -#ifdef DEBUG_LEGION - assert(remote_instance != runtime->address_space); -#endif - Serializer rez; - { - RezCheck z(rez); - rez.serialize(remote_ctx); - execution_context->pack_remote_context(rez, remote_instance); - } - runtime->send_remote_context_response(remote_instance, rez); - AutoLock o_lock(op_lock); -#ifdef DEBUG_LEGION - assert(remote_instances.find(remote_instance) == remote_instances.end()); -#endif - remote_instances[remote_instance] = remote_ctx; - } + } //-------------------------------------------------------------------------- - void IndividualTask::trigger_task_complete(void) + void IndividualTask::trigger_task_complete(bool deferred /*=false*/) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, INDIVIDUAL_TRIGGER_COMPLETE_CALL); @@ -5547,7 +5902,8 @@ namespace Legion { } //-------------------------------------------------------------------------- - void IndividualTask::handle_post_mapped(RtEvent mapped_precondition) + void IndividualTask::handle_post_mapped(bool deferral, + RtEvent mapped_precondition) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, INDIVIDUAL_POST_MAPPED_CALL); @@ -5730,6 +6086,15 @@ namespace Legion { return true; } + //-------------------------------------------------------------------------- + void IndividualTask::pack_as_shard_task(Serializer &rez,AddressSpace target) + //-------------------------------------------------------------------------- + { + pack_single_task(rez, target); + // Finally pack our context information + rez.serialize(remote_owner_uid); + } + //-------------------------------------------------------------------------- void IndividualTask::perform_inlining(TaskContext *enclosing) //-------------------------------------------------------------------------- @@ -5801,14 +6166,21 @@ namespace Legion { { DETAILED_PROFILER(runtime, INDIVIDUAL_PACK_REMOTE_COMPLETE_CALL); AddressSpaceID target = runtime->find_address_space(orig_proc); - if (execution_context->has_created_requirements()) + if ((execution_context != NULL) && + execution_context->has_created_requirements()) execution_context->send_back_created_state(target); // Send back the pointer to the task instance, then serialize // everything else that needs to be sent back rez.serialize(orig_task); RezCheck z(rez); // Pack the privilege state - execution_context->pack_resources_return(rez, context_index); + if (execution_context != NULL) + { + rez.serialize(true); + execution_context->pack_resources_return(rez, context_index); + } + else + rez.serialize(false); } //-------------------------------------------------------------------------- @@ -5818,8 +6190,12 @@ namespace Legion { DETAILED_PROFILER(runtime, INDIVIDUAL_UNPACK_REMOTE_COMPLETE_CALL); DerezCheck z(derez); // First unpack the privilege state - const RtEvent resources_returned = - ResourceTracker::unpack_resources_return(derez, parent_ctx); + bool has_privilege_state; + derez.deserialize(has_privilege_state); + RtEvent resources_returned; + if (has_privilege_state) + resources_returned = + ResourceTracker::unpack_resources_return(derez, parent_ctx); // Mark that we have both finished executing and that our // children are complete complete_execution(resources_returned); @@ -6205,7 +6581,7 @@ namespace Legion { RtEvent applied_condition; ApEvent effects_condition; // If we succeeded in mapping and we're a leaf so we are done mapping - if (is_leaf()) + if (is_leaf() && !is_replicated()) { if (!map_applied_conditions.empty()) { @@ -6263,6 +6639,15 @@ namespace Legion { return RtEvent::NO_RT_EVENT; } + //-------------------------------------------------------------------------- + void PointTask::shard_off(RtEvent mapped_precondition) + //-------------------------------------------------------------------------- + { + slice_owner->record_child_mapped(mapped_precondition, + ApEvent::NO_AP_EVENT); + SingleTask::shard_off(mapped_precondition); + } + //-------------------------------------------------------------------------- bool PointTask::is_stealable(void) const //-------------------------------------------------------------------------- @@ -6333,29 +6718,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - void PointTask::send_remote_context(AddressSpaceID remote_instance, - RemoteTask *remote_ctx) - //-------------------------------------------------------------------------- - { -#ifdef DEBUG_LEGION - assert(remote_instance != runtime->address_space); -#endif - Serializer rez; - { - RezCheck z(rez); - rez.serialize(remote_ctx); - execution_context->pack_remote_context(rez, remote_instance); - } - runtime->send_remote_context_response(remote_instance, rez); - AutoLock o_lock(op_lock); -#ifdef DEBUG_LEGION - assert(remote_instances.find(remote_instance) == remote_instances.end()); -#endif - remote_instances[remote_instance] = remote_ctx; - } - - //-------------------------------------------------------------------------- - void PointTask::trigger_task_complete(void) + void PointTask::trigger_task_complete(bool deferred /*=false*/) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, POINT_TASK_COMPLETE_CALL); @@ -6364,43 +6727,33 @@ namespace Legion { if (execution_context != NULL) { slice_owner->return_privileges(execution_context, preconditions); - if (!preconditions.empty()) - slice_owner->record_child_complete( - Runtime::merge_events(preconditions)); - else - slice_owner->record_child_complete(RtEvent::NO_RT_EVENT); - // Since this point is now complete we know - // that we can trigger it. Note we don't need to do - // this if we're a leaf task because we would have - // performed the leaf task early complete chaining operation. - if (!is_leaf()) - Runtime::trigger_event(NULL, point_termination); - if (runtime->legion_spy_enabled) execution_context->log_created_requirements(); // Invalidate any context that we had so that the child // operations can begin committing - execution_context->invalidate_region_tree_contexts(); - // See if we need to trigger that our children are complete - const bool need_commit = execution_context->attempt_children_commit(); - // Mark that this operation is now complete - complete_operation(); - if (need_commit) - trigger_children_committed(); - } - else - { - if (!preconditions.empty()) - slice_owner->record_child_complete( - Runtime::merge_events(preconditions)); - else - slice_owner->record_child_complete(RtEvent::NO_RT_EVENT); - + execution_context->invalidate_region_tree_contexts(); + // Since this point is now complete we know + // that we can trigger it. Note we don't need to do + // this if we're a leaf task with no virtual mappings + // because we would have performed the leaf task + // early complete chaining operation. if (!is_leaf()) Runtime::trigger_event(NULL, point_termination); - - complete_operation(); } + else + Runtime::trigger_event(NULL, point_termination); + if (!preconditions.empty()) + slice_owner->record_child_complete( + Runtime::merge_events(preconditions)); + else + slice_owner->record_child_complete(RtEvent::NO_RT_EVENT); + // See if we need to trigger that our children are complete + const bool need_commit = (execution_context != NULL) ? + execution_context->attempt_children_commit() : false; + // Mark that this operation is now complete + complete_operation(); + if (need_commit) + trigger_children_committed(); } //-------------------------------------------------------------------------- @@ -6467,6 +6820,15 @@ namespace Legion { return false; } + //-------------------------------------------------------------------------- + void PointTask::pack_as_shard_task(Serializer &rez, AddressSpace target) + //-------------------------------------------------------------------------- + { + pack_single_task(rez, target); + // Finally pack our context information + rez.serialize(slice_owner->get_remote_owner_uid()); + } + //-------------------------------------------------------------------------- void PointTask::handle_future(const void *res, size_t res_size, bool owner) //-------------------------------------------------------------------------- @@ -6475,7 +6837,8 @@ namespace Legion { } //-------------------------------------------------------------------------- - void PointTask::handle_post_mapped(RtEvent mapped_precondition) + void PointTask::handle_post_mapped(bool deferral, + RtEvent mapped_precondition) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, POINT_TASK_POST_MAPPED_CALL); @@ -6590,7 +6953,7 @@ namespace Legion { // Get our argument if (point_arguments.impl != NULL) { - Future f = point_arguments.impl->get_future(point); + Future f = point_arguments.impl->get_future(point, true/*internal*/); if (f.impl != NULL) { ApEvent ready = f.impl->get_ready_event(); @@ -6605,169 +6968,780 @@ namespace Legion { } } } - if (!point_futures.empty()) - { - for (std::vector::const_iterator it = - point_futures.begin(); it != point_futures.end(); it++) - this->futures.push_back(it->impl->get_future(point)); - } - // Make a new termination event for this point - point_termination = Runtime::create_ap_user_event(NULL); + if (!point_futures.empty()) + { + for (std::vector::const_iterator it = + point_futures.begin(); it != point_futures.end(); it++) + this->futures.push_back(it->impl->get_future(point,true/*internal*/)); + } + // Really unusual case here, if we're going to be doing remote tracing + // then we need to get an event from the owner node because some kinds + // of tracing (e.g. those with control replication) don't work otherwise + if ((remote_trace_info != NULL) && (remote_trace_info->recording)) + remote_trace_info->request_term_event(point_termination); + else // Make a new termination event for this point + point_termination = Runtime::create_ap_user_event(NULL); + } + + //-------------------------------------------------------------------------- + void PointTask::send_back_created_state(AddressSpaceID target) + //-------------------------------------------------------------------------- + { + if (execution_context->has_created_requirements()) + execution_context->send_back_created_state(target); + } + + //-------------------------------------------------------------------------- + void PointTask::replay_analysis(void) + //-------------------------------------------------------------------------- + { +#ifdef LEGION_SPY + LegionSpy::log_replay_operation(unique_op_id); +#endif + tpl->register_operation(this); + complete_mapping(); + } + + //-------------------------------------------------------------------------- + void PointTask::complete_replay(ApEvent instance_ready_event) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(is_leaf()); + assert(is_origin_mapped()); + assert(!target_processors.empty()); +#endif + const AddressSpaceID target_space = + runtime->find_address_space(target_processors.front()); + if (target_space != runtime->address_space) + { +#ifdef DEBUG_LEGION + assert(!deferred_effects.exists()); +#endif + deferred_effects = Runtime::create_ap_user_event(NULL); + slice_owner->record_child_mapped(RtEvent::NO_RT_EVENT,deferred_effects); + // This is the remote case, pack it up and ship it over + // Update our target_proc so that the sending code is correct + Serializer rez; + { + RezCheck z(rez); + rez.serialize(instance_ready_event); + rez.serialize(target_processors.front()); + rez.serialize(SLICE_TASK_KIND); + slice_owner->pack_task(rez, target_space); + } + runtime->send_remote_task_replay(target_space, rez); + // Record this slice as an origin-mapped slice so that it + // will be deactivated accordingly + slice_owner->index_owner->record_origin_mapped_slice(slice_owner); + } + else + { + // This is the local case + // Check to see if we're replaying this locally or remotely + for (std::deque::iterator it = physical_instances.begin(); + it != physical_instances.end(); ++it) + for (unsigned idx = 0; idx < it->size(); ++idx) + (*it)[idx].set_ready_event(instance_ready_event); + update_no_access_regions(); + launch_task(); + ApEvent postcondition = ApEvent::NO_AP_EVENT; + if (effects_postconditions.size() > 0) + postcondition = Runtime::merge_events(NULL, effects_postconditions); + if (is_remote()) + Runtime::trigger_event(NULL, deferred_effects, postcondition); + else + slice_owner->record_child_mapped(RtEvent::NO_RT_EVENT, postcondition); + } + } + + //-------------------------------------------------------------------------- + TraceLocalID PointTask::get_trace_local_id(void) const + //-------------------------------------------------------------------------- + { + if (remote_trace_info != NULL) + { + TraceLocalID result = + slice_owner->remote_trace_info->memo->get_trace_local_id(); + result.second = get_domain_point(); + return result; + } + else + return TraceLocalID(trace_local_id, get_domain_point()); + } + + //-------------------------------------------------------------------------- + CollectiveManager* PointTask::find_or_create_collective_instance( + MappingCallKind mapper_call, unsigned index, + const LayoutConstraintSet &constraints, + const std::vector ®ions, + Memory::Kind kind, size_t *footprint, + LayoutConstraintKind *unsat_kind, + unsigned *unsat_index, + DomainPoint &collective_point) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(index_point.get_dim() > 0); +#endif + collective_point = index_point; + return slice_owner->find_or_create_collective_instance(mapper_call, index, + constraints, regions, kind, footprint, unsat_kind, unsat_index, + collective_point); + } + + //-------------------------------------------------------------------------- + bool PointTask::finalize_collective_instance(MappingCallKind call_kind, + unsigned index, bool success) + //-------------------------------------------------------------------------- + { + return slice_owner->finalize_collective_instance(call_kind,index,success); + } + + //-------------------------------------------------------------------------- + void PointTask::report_total_collective_instance_calls( + MappingCallKind mapper_call, unsigned total_calls) + //-------------------------------------------------------------------------- + { + slice_owner->report_total_collective_instance_calls(mapper_call, + total_calls); + } + + //-------------------------------------------------------------------------- + void PointTask::record_intra_space_dependences(unsigned index, + const std::vector &dependences) + //-------------------------------------------------------------------------- + { + // Scan through the list until we find ourself + for (unsigned idx = 0; idx < dependences.size(); idx++) + { + if (dependences[idx] == index_point) + { + // If we've got a prior dependence then record it + if (idx > 0) + { + const DomainPoint &prev = dependences[idx-1]; + const RtEvent pre = slice_owner->find_intra_space_dependence(prev); + intra_space_mapping_dependences.insert(pre); + if (runtime->legion_spy_enabled) + LegionSpy::log_intra_space_dependence(unique_op_id, prev); + } + // If we're not the last dependence, then send our mapping event + // so that others can record a dependence on us + if (idx < (dependences.size()-1)) + slice_owner->record_intra_space_dependence(index_point, + dependences[idx+1], + get_mapped_event()); + return; + } + } + // We should never get here + assert(false); + } + + ///////////////////////////////////////////////////////////// + // Shard Task + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardTask::ShardTask(Runtime *rt, ShardManager *manager, + ShardID id, Processor proc) + : SingleTask(rt), shard_id(id) + //-------------------------------------------------------------------------- + { + activate_single(); + target_proc = proc; + current_proc = proc; + shard_manager = manager; + if (manager->original_task != NULL) + remote_owner_uid = + manager->original_task->get_context()->get_unique_id(); + } + + //-------------------------------------------------------------------------- + ShardTask::ShardTask(const ShardTask &rhs) + : SingleTask(NULL), shard_id(0) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ShardTask::~ShardTask(void) + //-------------------------------------------------------------------------- + { + // Set our shard manager to NULL since we are not supposed to delete it + shard_manager = NULL; + // We clear out instance top view here since we know that all + // our sibling shards are done at this point too, this allows + // us to remove any references to the context and hopefully to + // delete it + if ((execution_context != NULL) && execution_context->is_inner_context()) + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); +#endif + repl_ctx->clear_instance_top_views(); + } + deactivate_single(); + } + + //-------------------------------------------------------------------------- + ShardTask& ShardTask::operator=(const ShardTask &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ShardTask::activate(void) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardTask::deactivate(void) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + bool ShardTask::is_top_level_task(void) const + //-------------------------------------------------------------------------- + { + return shard_manager->top_level_task; + } + + //-------------------------------------------------------------------------- + void ShardTask::replay_analysis(void) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardTask::trigger_dependence_analysis(void) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardTask::resolve_false(bool speculated, bool launched) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardTask::early_map_task(void) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + bool ShardTask::distribute_task(void) + //-------------------------------------------------------------------------- + { + assert(false); + return false; + } + + //-------------------------------------------------------------------------- + RtEvent ShardTask::perform_must_epoch_version_analysis(MustEpochOp *own) + //-------------------------------------------------------------------------- + { + assert(false); + return RtEvent::NO_RT_EVENT; + } + + //-------------------------------------------------------------------------- + RtEvent ShardTask::perform_mapping(MustEpochOp *owner, + const DeferMappingArgs *args) + //-------------------------------------------------------------------------- + { + assert(false); + return RtEvent::NO_RT_EVENT; + } + + //-------------------------------------------------------------------------- + bool ShardTask::is_stealable(void) const + //-------------------------------------------------------------------------- + { + return false; + } + + //-------------------------------------------------------------------------- + bool ShardTask::can_early_complete(ApUserEvent &chain_event) + //-------------------------------------------------------------------------- + { + // no point for early completion for shard tasks + return false; + } + + //-------------------------------------------------------------------------- + std::map* + ShardTask::get_acquired_instances_ref(void) + //-------------------------------------------------------------------------- + { + // We shouldn't actually have any references for this kind of task + return NULL; + } + + //-------------------------------------------------------------------------- + ApEvent ShardTask::get_task_completion(void) const + //-------------------------------------------------------------------------- + { + return get_completion_event(); + } + + //-------------------------------------------------------------------------- + TaskOp::TaskKind ShardTask::get_task_kind(void) const + //-------------------------------------------------------------------------- + { + return SHARD_TASK_KIND; + } + + //-------------------------------------------------------------------------- + void ShardTask::trigger_mapping(void) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardTask::trigger_task_complete(bool deferred /*=false*/) + //-------------------------------------------------------------------------- + { + // First do the normal clean-up operations + // Remove profiling our guard and trigger the profiling event if necessary + if ((__sync_add_and_fetch(&outstanding_profiling_requests, -1) == 0) && + profiling_reported.exists()) + Runtime::trigger_event(profiling_reported); + // Invalidate any context that we had so that the child + // operations can begin committing + execution_context->invalidate_region_tree_contexts(); + if (runtime->legion_spy_enabled) + execution_context->log_created_requirements(); + // Then invoke the method on the shard manager + shard_manager->trigger_task_complete(true/*local*/); + // See if we need to trigger that our children are complete + const bool need_commit = execution_context->attempt_children_commit(); + // Mark that this operation is complete + complete_operation(); + if (need_commit) + trigger_children_committed(); + } + + //-------------------------------------------------------------------------- + void ShardTask::trigger_task_commit(void) + //-------------------------------------------------------------------------- + { + // Commit this operation + // Dont' deactivate ourselves, the shard manager will do that for us + commit_operation(false/*deactivate*/, profiling_reported); + // If we still have to report profiling information then we must + // block here to avoid a race with the shard manager deactivating + // us before we are done with this object + if (profiling_reported.exists() && !profiling_reported.has_triggered()) + profiling_reported.wait(); + // Lastly invoke the method on the shard manager, this could + // delete us so it has to be last + shard_manager->trigger_task_commit(true/*local*/); + } + + //-------------------------------------------------------------------------- + VersionInfo& ShardTask::get_version_info(unsigned idx) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(idx < version_infos.size()); +#endif + return version_infos[idx]; + } + + //-------------------------------------------------------------------------- + void ShardTask::perform_physical_traversal(unsigned idx, + RegionTreeContext ctx, InstanceSet &valid) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + bool ShardTask::pack_task(Serializer &rez, AddressSpaceID target) + //-------------------------------------------------------------------------- + { + RezCheck z(rez); + pack_single_task(rez, target); + rez.serialize(remote_owner_uid); + return false; + } + + //-------------------------------------------------------------------------- + bool ShardTask::unpack_task(Deserializer &derez, Processor current, + std::set &ready_events) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + unpack_single_task(derez, ready_events); + derez.deserialize(remote_owner_uid); + // Figure out what our parent context is + RtEvent ctx_ready; + parent_ctx = runtime->find_context(remote_owner_uid, false, &ctx_ready); + if (ctx_ready.exists()) + ready_events.insert(ctx_ready); + // Set our parent task for the user + parent_task = parent_ctx->get_task(); + return false; + } + + //-------------------------------------------------------------------------- + void ShardTask::pack_as_shard_task(Serializer &rez, AddressSpace target) + //-------------------------------------------------------------------------- + { + pack_single_task(rez, target); + // Finally pack our context information + rez.serialize(remote_owner_uid); + } + + //-------------------------------------------------------------------------- + RtEvent ShardTask::unpack_shard_task(Deserializer &derez) + //-------------------------------------------------------------------------- + { + std::set ready_events; + unpack_single_task(derez, ready_events); + derez.deserialize(remote_owner_uid); + // Figure out our parent context + RtEvent ctx_ready; + parent_ctx = runtime->find_context(remote_owner_uid, false, &ctx_ready); + if (ctx_ready.exists()) + ready_events.insert(ctx_ready); + // Set our parent task + parent_task = parent_ctx->get_task(); + if (!ready_events.empty()) + return Runtime::merge_events(ready_events); + else + return RtEvent::NO_RT_EVENT; + } + + //-------------------------------------------------------------------------- + void ShardTask::perform_inlining(TaskContext *enclosing) + //-------------------------------------------------------------------------- + { + assert(false); + } + + //-------------------------------------------------------------------------- + void ShardTask::handle_future(const void *res, size_t res_size, bool owned) + //-------------------------------------------------------------------------- + { + shard_manager->handle_post_execution(res, res_size, owned, true/*local*/); + } + + //-------------------------------------------------------------------------- + void ShardTask::handle_post_mapped(bool deferral, + RtEvent mapped_precondition) + //-------------------------------------------------------------------------- + { + shard_manager->handle_post_mapped(true/*local*/, mapped_precondition); + } + + //-------------------------------------------------------------------------- + void ShardTask::handle_misspeculation(void) + //-------------------------------------------------------------------------- + { + // TODO: figure out how misspeculation works with control replication + assert(false); + } + + //-------------------------------------------------------------------------- + InnerContext* ShardTask::initialize_inner_execution_context(VariantImpl *v) + //-------------------------------------------------------------------------- + { + if (runtime->legion_spy_enabled) + LegionSpy::log_shard(shard_manager->repl_id, shard_id, get_unique_id()); + // Check to see if we are control replicated or not + if (shard_manager->control_replicated) + { + // If we have a control replication context then we do the special path + ReplicateContext *repl_ctx = new ReplicateContext(runtime, this, + get_depth(), v->is_inner(), regions, parent_req_indexes, + virtual_mapped, unique_op_id, execution_fence_event, shard_manager); + if (mapper == NULL) + mapper = runtime->find_mapper(current_proc, map_id); + repl_ctx->configure_context(mapper, task_priority); + // Save the execution context early since we'll need it + execution_context = repl_ctx; + // Wait until all the other shards are ready too + shard_manager->complete_startup_initialization(); + // Hold a reference during this to prevent collectives + // from deleting the context prematurely + repl_ctx->add_reference(); + // The replicate contexts all need to sync up to exchange resources + repl_ctx->exchange_common_resources(); + // Remove our reference, DO NOT CHECK FOR DELETION + repl_ctx->remove_reference(); + return repl_ctx; + } + else // No control replication so do the normal thing + return SingleTask::initialize_inner_execution_context(v); + } + + //-------------------------------------------------------------------------- + InnerContext* ShardTask::create_implicit_context(void) + //-------------------------------------------------------------------------- + { + ReplicateContext *repl_ctx = new ReplicateContext(runtime, this, + get_depth(), false/*is inner*/, regions, parent_req_indexes, + virtual_mapped, unique_op_id, execution_fence_event, shard_manager); + if (mapper == NULL) + mapper = runtime->find_mapper(current_proc, map_id); + repl_ctx->configure_context(mapper, task_priority); + // Save the execution context early since we'll need it + execution_context = repl_ctx; + // Wait until all the other shards are ready too + shard_manager->complete_startup_initialization(); + // Hold a reference during this to prevent collectives + // from deleting the context prematurely + repl_ctx->add_reference(); + // The replicate contexts all need to sync up to exchange resources + repl_ctx->exchange_common_resources(); + return repl_ctx; + } + + //-------------------------------------------------------------------------- + void ShardTask::launch_shard(void) + //-------------------------------------------------------------------------- + { + // If it is a leaf then we can mark it mapped right now, + // otherwise wait for the call back, note we already know + // that it has no virtual instances because it is a + // replicated task + if (is_leaf()) + shard_manager->handle_post_mapped(true/*local*/, RtEvent::NO_RT_EVENT); + // Speculation can always be resolved here + resolve_speculation(); + // Then launch the task for execution + launch_task(); + } + + //-------------------------------------------------------------------------- + void ShardTask::extract_event_preconditions( + const std::deque &all_instances) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(all_instances.size() == physical_instances.size()); +#endif + for (unsigned region_idx = 0; + region_idx < physical_instances.size(); region_idx++) + { + InstanceSet &local_instances = physical_instances[region_idx]; + const InstanceSet &instances = all_instances[region_idx]; + for (unsigned idx1 = 0; idx1 < local_instances.size(); idx1++) + { + InstanceRef &ref = local_instances[idx1]; +#ifdef DEBUG_LEGION + bool found = false; +#endif + for (unsigned idx2 = 0; idx2 < instances.size(); idx2++) + { + const InstanceRef &other_ref = instances[idx2]; + if (ref.get_manager() != other_ref.get_manager()) + continue; + ref.set_ready_event(other_ref.get_ready_event()); +#ifdef DEBUG_LEGION + found = true; +#endif + break; + } +#ifdef DEBUG_LEGION + assert(found); +#endif + } + } } //-------------------------------------------------------------------------- - void PointTask::send_back_created_state(AddressSpaceID target) + void ShardTask::return_resources(ResourceTracker *target, + std::set &preconditions) //-------------------------------------------------------------------------- { - if (execution_context->has_created_requirements()) - execution_context->send_back_created_state(target); - } +#ifdef DEBUG_LEGION + assert(execution_context != NULL); +#endif + execution_context->return_resources(target, context_index, preconditions); + } //-------------------------------------------------------------------------- - void PointTask::replay_analysis(void) + void ShardTask::report_leaks_and_duplicates( + std::set &preconditions) //-------------------------------------------------------------------------- { -#ifdef LEGION_SPY - LegionSpy::log_replay_operation(unique_op_id); +#ifdef DEBUG_LEGION + assert(execution_context != NULL); #endif - tpl->register_operation(this); - complete_mapping(); + execution_context->report_leaks_and_duplicates(preconditions); } //-------------------------------------------------------------------------- - void PointTask::complete_replay(ApEvent instance_ready_event) + void ShardTask::handle_collective_message(Deserializer &derez) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert(is_leaf()); - assert(is_origin_mapped()); - assert(!target_processors.empty()); + assert(execution_context != NULL); + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); #endif - const AddressSpaceID target_space = - runtime->find_address_space(target_processors.front()); - if (target_space != runtime->address_space) - { + repl_ctx->handle_collective_message(derez); + } + + //-------------------------------------------------------------------------- + void ShardTask::handle_future_map_request(Deserializer &derez) + //-------------------------------------------------------------------------- + { #ifdef DEBUG_LEGION - assert(!deferred_effects.exists()); + assert(execution_context != NULL); + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); #endif - deferred_effects = Runtime::create_ap_user_event(NULL); - slice_owner->record_child_mapped(RtEvent::NO_RT_EVENT,deferred_effects); - // This is the remote case, pack it up and ship it over - // Update our target_proc so that the sending code is correct - Serializer rez; - { - RezCheck z(rez); - rez.serialize(instance_ready_event); - rez.serialize(target_processors.front()); - rez.serialize(SLICE_TASK_KIND); - slice_owner->pack_task(rez, target_space); - } - runtime->send_remote_task_replay(target_space, rez); - // Record this slice as an origin-mapped slice so that it - // will be deactivated accordingly - slice_owner->index_owner->record_origin_mapped_slice(slice_owner); - } - else - { - // This is the local case - // Check to see if we're replaying this locally or remotely - for (std::deque::iterator it = physical_instances.begin(); - it != physical_instances.end(); ++it) - for (unsigned idx = 0; idx < it->size(); ++idx) - (*it)[idx].set_ready_event(instance_ready_event); - update_no_access_regions(); - launch_task(); - ApEvent postcondition = ApEvent::NO_AP_EVENT; - if (effects_postconditions.size() > 0) - postcondition = Runtime::merge_events(NULL, effects_postconditions); - if (is_remote()) - Runtime::trigger_event(NULL, deferred_effects, postcondition); - else - slice_owner->record_child_mapped(RtEvent::NO_RT_EVENT, postcondition); - } + repl_ctx->handle_future_map_request(derez); } //-------------------------------------------------------------------------- - TraceLocalID PointTask::get_trace_local_id(void) const + void ShardTask::handle_equivalence_set_request(Deserializer &derez) //-------------------------------------------------------------------------- { - if (remote_trace_info != NULL) - { - TraceLocalID result = - slice_owner->remote_trace_info->memo->get_trace_local_id(); - result.second = get_domain_point(); - return result; - } - else - return TraceLocalID(trace_local_id, get_domain_point()); +#ifdef DEBUG_LEGION + assert(execution_context != NULL); + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); +#endif + repl_ctx->handle_equivalence_set_request(derez); } //-------------------------------------------------------------------------- - CollectiveManager* PointTask::find_or_create_collective_instance( - MappingCallKind mapper_call, unsigned index, - const LayoutConstraintSet &constraints, - const std::vector ®ions, - Memory::Kind kind, size_t *footprint, - LayoutConstraintKind *unsat_kind, - unsigned *unsat_index, - DomainPoint &collective_point) + void ShardTask::handle_intra_space_dependence(Deserializer &derez) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert(index_point.get_dim() > 0); + assert(execution_context != NULL); + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); #endif - collective_point = index_point; - return slice_owner->find_or_create_collective_instance(mapper_call, index, - constraints, regions, kind, footprint, unsat_kind, unsat_index, - collective_point); + repl_ctx->handle_intra_space_dependence(derez); } //-------------------------------------------------------------------------- - bool PointTask::finalize_collective_instance(MappingCallKind call_kind, - unsigned index, bool success) + void ShardTask::handle_resource_update(Deserializer &derez, + std::set &applied) //-------------------------------------------------------------------------- { - return slice_owner->finalize_collective_instance(call_kind,index,success); +#ifdef DEBUG_LEGION + assert(execution_context != NULL); + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); +#endif + repl_ctx->handle_resource_update(derez, applied); } //-------------------------------------------------------------------------- - void PointTask::report_total_collective_instance_calls( - MappingCallKind mapper_call, unsigned total_calls) + void ShardTask::handle_trace_update(Deserializer &derez, + AddressSpaceID source) //-------------------------------------------------------------------------- { - slice_owner->report_total_collective_instance_calls(mapper_call, - total_calls); +#ifdef DEBUG_LEGION + assert(execution_context != NULL); + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); +#endif + repl_ctx->handle_trace_update(derez, source); } //-------------------------------------------------------------------------- - void PointTask::record_intra_space_dependences(unsigned index, - const std::vector &dependences) + ApBarrier ShardTask::handle_find_trace_shard_event(size_t template_index, + ApEvent event, ShardID remote_shard) //-------------------------------------------------------------------------- { - // Scan through the list until we find ourself - for (unsigned idx = 0; idx < dependences.size(); idx++) - { - if (dependences[idx] == index_point) - { - // If we've got a prior dependence then record it - if (idx > 0) - { - const DomainPoint &prev = dependences[idx-1]; - const RtEvent pre = slice_owner->find_intra_space_dependence(prev); - intra_space_mapping_dependences.insert(pre); - if (runtime->legion_spy_enabled) - LegionSpy::log_intra_space_dependence(unique_op_id, prev); - } - // If we're not the last dependence, then send our mapping event - // so that others can record a dependence on us - if (idx < (dependences.size()-1)) - slice_owner->record_intra_space_dependence(index_point, - get_mapped_event()); - return; - } - } - // We should never get here - assert(false); +#ifdef DEBUG_LEGION + assert(execution_context != NULL); + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); +#endif + return repl_ctx->handle_find_trace_shard_event(template_index, event, + remote_shard); + } + + //-------------------------------------------------------------------------- + InstanceView* ShardTask::create_instance_top_view(PhysicalManager *manager, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(execution_context != NULL); + ReplicateContext *repl_ctx = + dynamic_cast(execution_context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = + static_cast(execution_context); +#endif + return repl_ctx->create_replicate_instance_top_view(manager, source); + } + + //-------------------------------------------------------------------------- + void ShardTask::initialize_implicit_task(InnerContext *context, TaskID tid, + MapperID mid, Processor proxy) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(parent_ctx == NULL); +#endif + parent_ctx = context; + task_id = tid; + map_id = mid; + orig_proc = proxy; + current_proc = proxy; } ///////////////////////////////////////////////////////////// @@ -6810,6 +7784,13 @@ namespace Legion { //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, INDEX_ACTIVATE_CALL); + activate_index_task(); + } + + //-------------------------------------------------------------------------- + void IndexTask::activate_index_task(void) + //-------------------------------------------------------------------------- + { activate_multi(); serdez_redop_fns = NULL; total_points = 0; @@ -6828,6 +7809,14 @@ namespace Legion { //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, INDEX_DEACTIVATE_CALL); + deactivate_index_task(); + runtime->free_index_task(this); + } + + //-------------------------------------------------------------------------- + void IndexTask::deactivate_index_task(void) + //-------------------------------------------------------------------------- + { deactivate_multi(); privilege_paths.clear(); if (!origin_mapped_slices.empty()) @@ -6861,13 +7850,12 @@ namespace Legion { point_requirements.clear(); assert(pending_intra_space_dependences.empty()); #endif - runtime->free_index_task(this); } //-------------------------------------------------------------------------- FutureMap IndexTask::initialize_task(InnerContext *ctx, const IndexTaskLauncher &launcher, - IndexSpace launch_sp, + IndexSpace launch_sp, bool track /*= true*/) //-------------------------------------------------------------------------- { @@ -6930,15 +7918,10 @@ namespace Legion { if (launcher.predicate != Predicate::TRUE_PRED) initialize_predicate(launcher.predicate_false_future, launcher.predicate_false_result); - future_map_ready = Runtime::create_rt_user_event(); - future_map = FutureMap(new FutureMapImpl(ctx, this, future_map_ready, - runtime, runtime->get_available_distributed_id(), - runtime->address_space)); -#ifdef DEBUG_LEGION - future_map.impl->add_valid_domain(index_domain); -#endif + future_map = FutureMap( + create_future_map(ctx, launch_space->handle, launcher.sharding_space)); check_empty_field_requirements(); - + if (runtime->legion_spy_enabled) { LegionSpy::log_index_task(parent_ctx->get_unique_id(), @@ -7116,11 +8099,6 @@ namespace Legion { { set_must_epoch(epoch, index, do_registration); future_map = epoch->get_future_map(); -#ifdef DEBUG_LEGION - Domain launch_domain; - launch_space->get_launch_space_domain(launch_domain); - future_map.impl->add_valid_domain(launch_domain); -#endif } //-------------------------------------------------------------------------- @@ -7183,6 +8161,21 @@ namespace Legion { void IndexTask::trigger_dependence_analysis(void) //-------------------------------------------------------------------------- { + perform_base_dependence_analysis(); + for (unsigned idx = 0; idx < regions.size(); idx++) + { + ProjectionInfo projection_info(runtime, regions[idx], launch_space); + runtime->forest->perform_dependence_analysis(this, idx, regions[idx], + projection_info, + privilege_paths[idx], + map_applied_conditions); + } + } + + //-------------------------------------------------------------------------- + void IndexTask::perform_base_dependence_analysis(void) + //-------------------------------------------------------------------------- + { #ifdef DEBUG_LEGION assert(memo_state != MEMO_REQ); assert(privilege_paths.size() == regions.size()); @@ -7212,14 +8205,6 @@ namespace Legion { it->impl->register_dependence(this); // Also have to register any dependences on our predicate register_predicate_dependence(); - for (unsigned idx = 0; idx < regions.size(); idx++) - { - ProjectionInfo projection_info(runtime, regions[idx], launch_space); - runtime->forest->perform_dependence_analysis(this, idx, regions[idx], - projection_info, - privilege_paths[idx], - map_applied_conditions); - } } //-------------------------------------------------------------------------- @@ -7275,48 +8260,57 @@ namespace Legion { // Fill in the index task map with the default future value if (redop == 0) { - // Handling the future map case - if (predicate_false_future.impl != NULL) + // Only need to do this if the internal domain exists, it + // might not in a control replication context + if (internal_space.exists()) { - ApEvent wait_on = predicate_false_future.impl->get_ready_event(); - if (wait_on.has_triggered()) + // Get the domain that we will have to iterate over + Domain local_domain; + runtime->forest->find_launch_space_domain(internal_space, + local_domain); + // Handling the future map case + if (predicate_false_future.impl != NULL) { - const size_t result_size = - check_future_size(predicate_false_future.impl); - const void *result = - predicate_false_future.impl->get_untyped_result(true, NULL, true); - for (Domain::DomainPointIterator itr(index_domain); itr; itr++) + ApEvent wait_on = predicate_false_future.impl->get_ready_event(); + if (wait_on.has_triggered()) { - Future f = future_map.impl->get_future(itr.p); - if (result_size > 0) - f.impl->set_result(result, result_size, false/*own*/); - else - f.impl->set_result(NULL, 0, false/*own*/); + const size_t result_size = + check_future_size(predicate_false_future.impl); + const void *result = + predicate_false_future.impl->get_untyped_result(true,NULL,true); + for (Domain::DomainPointIterator itr(local_domain); itr; itr++) + { + Future f = future_map.impl->get_future(itr.p, true/*internal*/); + if (result_size > 0) + f.impl->set_result(result, result_size, false/*own*/); + else + f.impl->set_result(NULL, 0, false/*own*/); + } } - } - else - { - // Add references so things won't be prematurely collected - future_map.impl->add_base_resource_ref(DEFERRED_TASK_REF); - predicate_false_future.impl->add_base_gc_ref(DEFERRED_TASK_REF, - this); - DeferredFutureMapSetArgs args(future_map.impl, - predicate_false_future.impl, index_domain, this); - execution_condition = - runtime->issue_runtime_meta_task(args, LG_LATENCY_WORK_PRIORITY, + else + { + // Add references so things won't be prematurely collected + future_map.impl->add_base_resource_ref(DEFERRED_TASK_REF); + predicate_false_future.impl->add_base_gc_ref(DEFERRED_TASK_REF, + this); + DeferredFutureMapSetArgs args(future_map.impl, + predicate_false_future.impl, local_domain, this); + execution_condition = + runtime->issue_runtime_meta_task(args, LG_LATENCY_WORK_PRIORITY, Runtime::protect_event(wait_on)); + } } - } - else - { - for (Domain::DomainPointIterator itr(index_domain); itr; itr++) + else { - Future f = future_map.impl->get_future(itr.p); - if (predicate_false_size > 0) - f.impl->set_result(predicate_false_result, - predicate_false_size, false/*own*/); - else - f.impl->set_result(NULL, 0, false/*own*/); + for (Domain::DomainPointIterator itr(local_domain); itr; itr++) + { + Future f = future_map.impl->get_future(itr.p, true/*internal*/); + if (predicate_false_size > 0) + f.impl->set_result(predicate_false_result, + predicate_false_size, false/*own*/); + else + f.impl->set_result(NULL, 0, false/*own*/); + } } } } @@ -7662,8 +8656,7 @@ namespace Legion { (target_proc != current_proc)) { // Make a slice copy and send it away - SliceTask *clone = clone_as_slice_task(launch_space->handle, - target_proc, + SliceTask *clone = clone_as_slice_task(internal_space, target_proc, true/*needs slice*/, stealable); runtime->send_task(clone); @@ -7740,7 +8733,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - void IndexTask::trigger_task_complete(void) + void IndexTask::trigger_task_complete(bool deferred /*=false*/) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, INDEX_COMPLETE_CALL); @@ -7953,7 +8946,8 @@ namespace Legion { // Get our local args if (point_arguments.impl != NULL) { - Future local_arg = point_arguments.impl->get_future(index_point); + Future local_arg = + point_arguments.impl->get_future(index_point, false/*internal*/); if (local_arg.impl != NULL) { local_args = local_arg.impl->get_untyped_result(true, NULL, true); @@ -7998,7 +8992,7 @@ namespace Legion { { if (redop == 0) { - Future f = future_map.impl->get_future(index_point); + Future f = future_map.impl->get_future(index_point, true/*internal*/); f.impl->set_result(res, res_size, owned); } else @@ -8189,6 +9183,19 @@ namespace Legion { assert(false); } + //-------------------------------------------------------------------------- + FutureMapImpl* IndexTask::create_future_map(TaskContext *ctx, + IndexSpace launch_space, IndexSpace sharding_space) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!future_map_ready.exists() || future_map_ready.has_triggered()); +#endif + future_map_ready = Runtime::create_rt_user_event(); + return new FutureMapImpl(ctx, this, future_map_ready,index_domain,runtime, + runtime->get_available_distributed_id(), runtime->address_space); + } + //-------------------------------------------------------------------------- RtEvent IndexTask::find_intra_space_dependence(const DomainPoint &point) //-------------------------------------------------------------------------- @@ -8208,6 +9215,7 @@ namespace Legion { //-------------------------------------------------------------------------- void IndexTask::record_intra_space_dependence(const DomainPoint &point, + const DomainPoint &next, RtEvent point_mapped) //-------------------------------------------------------------------------- { @@ -8216,17 +9224,17 @@ namespace Legion { intra_space_dependences.find(point); if (finder != intra_space_dependences.end()) { + if (finder->second != point_mapped) + { + std::map::iterator pending_finder = + pending_intra_space_dependences.find(point); #ifdef DEBUG_LEGION - assert(finder->second != point_mapped); -#endif - std::map::iterator pending_finder = - pending_intra_space_dependences.find(point); -#ifdef DEBUG_LEGION - assert(pending_finder != pending_intra_space_dependences.end()); + assert(pending_finder != pending_intra_space_dependences.end()); #endif - Runtime::trigger_event(pending_finder->second, point_mapped); - pending_intra_space_dependences.erase(pending_finder); - finder->second = point_mapped; + Runtime::trigger_event(pending_finder->second, point_mapped); + pending_intra_space_dependences.erase(pending_finder); + finder->second = point_mapped; + } } else intra_space_dependences[point] = point_mapped; @@ -8492,15 +9500,14 @@ namespace Legion { TaskOp::log_requirement(unique_op_id, idx, regions[idx]); runtime->forest->log_launch_space(launch_space->handle, unique_op_id); } - // Count how many total points we need for this index space task - total_points = index_domain.get_volume(); // Mark that this is origin mapped effectively in case we // have any remote tasks, do this before we clone it map_origin = true; SliceTask *new_slice = this->clone_as_slice_task(internal_space, current_proc, false, false); - new_slice->enumerate_points(); + // Count how many total points we need for this index space task + total_points = new_slice->enumerate_points(); // We need to make one slice per point here in case we need to move // points to remote nodes. The way we do slicing right now prevents // us from knowing which point tasks are going remote until later in @@ -8562,11 +9569,12 @@ namespace Legion { { IndexTask *task; derez.deserialize(task); - DomainPoint point; + DomainPoint point, next; derez.deserialize(point); + derez.deserialize(next); RtEvent mapped_event; derez.deserialize(mapped_event); - task->record_intra_space_dependence(point, mapped_event); + task->record_intra_space_dependence(point, next, mapped_event); } #ifdef DEBUG_LEGION @@ -8919,6 +9927,7 @@ namespace Legion { created_field_spaces.clear(); created_index_spaces.clear(); created_index_partitions.clear(); + unique_intra_space_deps.clear(); runtime->free_slice_task(this); } @@ -9125,6 +10134,7 @@ namespace Legion { assert(future_map.impl != NULL); #endif rez.serialize(future_map.impl->did); + rez.serialize(future_map.impl->future_map_domain); rez.serialize(future_map.impl->get_ready_event()); } if (predicate_false_future.impl != NULL) @@ -9172,6 +10182,7 @@ namespace Legion { if (point_arguments.impl != NULL) { rez.serialize(point_arguments.impl->did); + rez.serialize(point_arguments.impl->future_map_domain); rez.serialize(point_arguments.impl->get_ready_event()); } else @@ -9181,6 +10192,7 @@ namespace Legion { { FutureMapImpl *impl = point_futures[idx].impl; rez.serialize(impl->did); + rez.serialize(impl->future_map_domain); rez.serialize(impl->get_ready_event()); } } @@ -9241,12 +10253,14 @@ namespace Legion { { DistributedID future_map_did; derez.deserialize(future_map_did); + Domain future_map_domain; + derez.deserialize(future_map_domain); RtEvent ready_event; derez.deserialize(ready_event); WrapperReferenceMutator mutator(ready_events); future_map = FutureMap( runtime->find_or_create_future_map(future_map_did, parent_ctx, - ready_event, &mutator)); + future_map_domain, ready_event, &mutator)); } // Unpack the predicate false infos DistributedID pred_false_did; @@ -9291,11 +10305,14 @@ namespace Legion { derez.deserialize(future_map_did); if (future_map_did > 0) { + Domain future_map_domain; + derez.deserialize(future_map_domain); RtEvent ready_event; derez.deserialize(ready_event); WrapperReferenceMutator mutator(ready_events); FutureMapImpl *impl = runtime->find_or_create_future_map( - future_map_did, parent_ctx, ready_event, &mutator); + future_map_did, parent_ctx, future_map_domain, + ready_event, &mutator); impl->add_base_gc_ref(FUTURE_HANDLE_REF, &mutator); point_arguments = FutureMap(impl, false/*need reference*/); } @@ -9304,14 +10321,17 @@ namespace Legion { if (num_point_futures > 0) { RtEvent ready_event; + Domain future_map_domain; point_futures.resize(num_point_futures); WrapperReferenceMutator mutator(ready_events); for (unsigned idx = 0; idx < num_point_futures; idx++) { derez.deserialize(future_map_did); + derez.deserialize(future_map_domain); derez.deserialize(ready_event); FutureMapImpl *impl = runtime->find_or_create_future_map( - future_map_did, parent_ctx, ready_event, &mutator); + future_map_did, parent_ctx, future_map_domain, + ready_event, &mutator); impl->add_base_gc_ref(FUTURE_HANDLE_REF, &mutator); point_futures[idx] = FutureMap(impl, false/*need reference*/); } @@ -9404,7 +10424,7 @@ namespace Legion { } else { - Future f = future_map.impl->get_future(point); + Future f = future_map.impl->get_future(point, true/*internal only*/); f.impl->set_result(result, result_size, owner); } } @@ -9441,8 +10461,6 @@ namespace Legion { result->index_domain = this->index_domain; result->tpl = tpl; result->memo_state = memo_state; - // Now figure out our local point information - result->initialize_point(this, point, point_arguments, point_futures); // Grab any remote trace info that we need from the slice if (remote_trace_info != NULL) { @@ -9451,6 +10469,8 @@ namespace Legion { #endif result->remote_trace_info = new TraceInfo(*remote_trace_info, result); } + // Now figure out our local point information + result->initialize_point(this, point, point_arguments, point_futures); if (runtime->legion_spy_enabled) LegionSpy::log_slice_point(get_unique_id(), result->get_unique_id(), @@ -9459,13 +10479,13 @@ namespace Legion { } //-------------------------------------------------------------------------- - void SliceTask::enumerate_points(void) + size_t SliceTask::enumerate_points(void) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, SLICE_ENUMERATE_POINTS_CALL); Domain internal_domain; runtime->forest->find_launch_space_domain(internal_space,internal_domain); - size_t num_points = internal_domain.get_volume(); + const size_t num_points = internal_domain.get_volume(); #ifdef DEBUG_LEGION assert(num_points > 0); #endif @@ -9485,16 +10505,17 @@ namespace Legion { ProjectionFunction *function = runtime->find_projection_function(regions[idx].projection); function->project_points(regions[idx], idx, runtime, - points, launch_space); + index_domain, points); } } // Update the no access regions - for (unsigned idx = 0; idx < points.size(); idx++) + for (unsigned idx = 0; idx < num_points; idx++) points[idx]->complete_point_projection(); // Mark how many points we have - num_unmapped_points = points.size(); - num_uncomplete_points = points.size(); - num_uncommitted_points = points.size(); + num_unmapped_points = num_points; + num_uncomplete_points = num_points; + num_uncommitted_points = num_points; + return num_points; } //-------------------------------------------------------------------------- @@ -9517,7 +10538,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - void SliceTask::trigger_task_complete(void) + void SliceTask::trigger_task_complete(bool deferred /*=false*/) //-------------------------------------------------------------------------- { trigger_slice_complete(); @@ -9528,7 +10549,7 @@ namespace Legion { //-------------------------------------------------------------------------- { trigger_slice_commit(); - } + } //-------------------------------------------------------------------------- void SliceTask::record_reference_mutation_effect(RtEvent event) @@ -10178,11 +11199,13 @@ namespace Legion { //-------------------------------------------------------------------------- void SliceTask::record_intra_space_dependence(const DomainPoint &point, + const DomainPoint &next, RtEvent point_mapped) //-------------------------------------------------------------------------- { // Check to see if we already sent it already { + const std::pair key(point, next); AutoLock o_lock(op_lock); std::map::const_iterator finder = intra_space_dependences.find(point); @@ -10191,10 +11214,20 @@ namespace Legion { #ifdef DEBUG_LEGION assert(finder->second == point_mapped); #endif - return; + // For control replication we need the index owner to see all + // the unique sets of dependences, see if we've seen this + // combination before, if not, allow it to be sent back + // to the index owner for it's own visibility + std::set >::const_iterator + key_finder = unique_intra_space_deps.find(key); + if (key_finder != unique_intra_space_deps.end()) + return; } - // Otherwise save it and then let it flow back to the index owner - intra_space_dependences[point] = point_mapped; + else + // Otherwise save it and then let it flow back to the index owner + intra_space_dependences[point] = point_mapped; + // Always save this if we make it here + unique_intra_space_deps.insert(key); } if (is_remote()) { @@ -10203,12 +11236,13 @@ namespace Legion { RezCheck z(rez); rez.serialize(index_owner); rez.serialize(point); + rez.serialize(next); rez.serialize(point_mapped); } runtime->send_slice_record_intra_space_dependence(orig_proc, rez); } else - index_owner->record_intra_space_dependence(point, point_mapped); + index_owner->record_intra_space_dependence(point, next, point_mapped); } //-------------------------------------------------------------------------- diff --git a/runtime/legion/legion_tasks.h b/runtime/legion/legion_tasks.h index 9a7501994c..78fe45af2f 100644 --- a/runtime/legion/legion_tasks.h +++ b/runtime/legion/legion_tasks.h @@ -110,6 +110,7 @@ namespace Legion { POINT_TASK_KIND, INDEX_TASK_KIND, SLICE_TASK_KIND, + SHARD_TASK_KIND, }; public: struct TriggerTaskArgs : public LgTaskArgs { @@ -199,6 +200,17 @@ namespace Legion { ProcessorManager *const manager; TaskOp *const task; }; + struct DeferredTaskCompleteArgs : + public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_DEFERRED_TASK_COMPLETE_TASK_ID; + public: + DeferredTaskCompleteArgs(TaskOp *t) + : LgTaskArgs(t->get_unique_op_id()), + task(t) { } + public: + TaskOp *const task; + }; public: TaskOp(Runtime *rt); virtual ~TaskOp(void); @@ -215,10 +227,12 @@ namespace Legion { bool is_remote(void) const; inline bool is_stolen(void) const { return (steal_count > 0); } inline bool is_origin_mapped(void) const { return map_origin; } + inline bool is_replicated(void) const { return replicate; } int get_depth(void) const; public: void set_current_proc(Processor current); inline void set_origin_mapped(bool origin) { map_origin = origin; } + inline void set_replicated(bool repl) { replicate = repl; } inline void set_target_proc(Processor next) { target_proc = next; } protected: void activate_task(void); @@ -332,11 +346,13 @@ namespace Legion { // Tasks have two requirements to complete: // - all speculation must be resolved // - all children must be complete - virtual void trigger_task_complete(void) = 0; + virtual void trigger_task_complete(bool deferred = false) = 0; // Tasks have two requirements to commit: // - all commit dependences must be satisfied (trigger_commit) // - all children must commit (children_committed) virtual void trigger_task_commit(void) = 0; + public: + static void handle_deferred_task_complete(const void *args); protected: // Early mapped regions std::map early_mapped_regions; @@ -354,6 +370,7 @@ namespace Legion { bool memoize_selected; bool map_origin; bool request_valid_instances; + bool replicate; protected: // For managing predication PredEvent true_guard; @@ -442,6 +459,7 @@ namespace Legion { bool is_inner(void) const; bool is_created_region(unsigned index) const; void update_no_access_regions(void); + void clone_single_from(SingleTask *task); public: inline void clone_virtual_mapped(std::vector &target) const { target = virtual_mapped; } @@ -453,6 +471,8 @@ namespace Legion { { return no_access_regions; } inline VariantID get_selected_variant(void) const { return selected_variant; } + inline const std::set& get_map_applied_conditions(void) const + { return map_applied_conditions; } inline RtEvent get_profiling_reported(void) const { return profiling_reported; } public: @@ -466,20 +486,24 @@ namespace Legion { MustEpochOp *must_epoch_owner, std::vector &valid_instances); void replay_map_task_output(void); - InnerContext* create_implicit_context(void); + virtual InnerContext* create_implicit_context(void); + void set_shard_manager(ShardManager *manager); protected: // mapper helper calls void validate_target_processors(const std::vector &prcs) const; void validate_variant_selection(MapperManager *local_mapper, VariantImpl *impl, Processor::Kind kind, const char *call_name) const; protected: void invoke_mapper(MustEpochOp *must_epoch_owner); + void invoke_mapper_replicated(MustEpochOp *must_epoch_owner); RtEvent map_all_regions(ApEvent user_event, MustEpochOp *must_epoch_owner, const DeferMappingArgs *defer_args); void perform_post_mapping(const TraceInfo &trace_info); + void replicate_task(void); protected: void pack_single_task(Serializer &rez, AddressSpaceID target); void unpack_single_task(Deserializer &derez, std::set &ready_events); + void send_remote_context(AddressSpaceID target, RemoteTask *dst); public: virtual void pack_profiling_requests(Serializer &rez, std::set &applied) const; @@ -494,6 +518,7 @@ namespace Legion { virtual void activate(void) = 0; virtual void deactivate(void) = 0; virtual bool is_top_level_task(void) const { return false; } + virtual bool is_shard_task(void) const { return false; } virtual SingleTask* get_origin_task(void) const = 0; public: virtual void resolve_false(bool speculated, bool launched) = 0; @@ -502,29 +527,32 @@ namespace Legion { virtual bool distribute_task(void) = 0; virtual RtEvent perform_mapping(MustEpochOp *owner = NULL, const DeferMappingArgs *args = NULL) = 0; + // For tasks that are sharded off by control replication + virtual void shard_off(RtEvent mapped_precondition); virtual bool is_stealable(void) const = 0; virtual bool can_early_complete(ApUserEvent &chain_event) = 0; public: virtual ApEvent get_task_completion(void) const = 0; virtual TaskKind get_task_kind(void) const = 0; - public: - virtual void send_remote_context(AddressSpaceID target, - RemoteTask *dst) = 0; public: // Override these methods from operation class virtual void trigger_mapping(void); protected: - virtual void trigger_task_complete(void) = 0; + friend class ShardManager; + virtual void trigger_task_complete(bool deferred = false) = 0; virtual void trigger_task_commit(void) = 0; public: virtual bool pack_task(Serializer &rez, AddressSpaceID target) = 0; virtual bool unpack_task(Deserializer &derez, Processor current, std::set &ready_events) = 0; + virtual void pack_as_shard_task(Serializer &rez, + AddressSpaceID target) = 0; virtual void perform_inlining(TaskContext *enclosing) = 0; public: virtual void handle_future(const void *res, size_t res_size, bool owned) = 0; - virtual void handle_post_mapped(RtEvent pre = RtEvent::NO_RT_EVENT) = 0; + virtual void handle_post_mapped(bool deferral, + RtEvent pre = RtEvent::NO_RT_EVENT) = 0; virtual void handle_misspeculation(void) = 0; public: // From Memoizable @@ -536,6 +564,8 @@ namespace Legion { public: void handle_remote_profiling_response(Deserializer &derez); static void process_remote_profiling_response(Deserializer &derez); + protected: + virtual InnerContext* initialize_inner_execution_context(VariantImpl *v); protected: // Boolean for each region saying if it is virtual mapped std::vector virtual_mapped; @@ -557,11 +587,15 @@ namespace Legion { bool first_mapping; std::set intra_space_mapping_dependences; // Events that must be triggered before we are done mapping - std::set map_applied_conditions; + std::set map_applied_conditions; RtUserEvent deferred_complete_mapping; protected: TaskContext* execution_context; TraceInfo* remote_trace_info; + // For replication of this task + ShardManager* shard_manager; + protected: + std::map remote_instances; protected: mutable bool leaf_cached, is_leaf_result; mutable bool inner_cached, is_inner_result; @@ -625,7 +659,7 @@ namespace Legion { public: virtual void trigger_mapping(void); protected: - virtual void trigger_task_complete(void) = 0; + virtual void trigger_task_complete(bool deferred = false) = 0; virtual void trigger_task_commit(void) = 0; public: virtual bool pack_task(Serializer &rez, AddressSpaceID target) = 0; @@ -642,6 +676,7 @@ namespace Legion { // Methods for supporting intra-index-space mapping dependences virtual RtEvent find_intra_space_dependence(const DomainPoint &point) = 0; virtual void record_intra_space_dependence(const DomainPoint &point, + const DomainPoint &next, RtEvent point_mapped) = 0; public: void pack_multi_task(Serializer &rez, AddressSpaceID target); @@ -701,6 +736,9 @@ namespace Legion { public: virtual void activate(void); virtual void deactivate(void); + protected: + void activate_individual_task(void); + void deactivate_individual_task(void); virtual SingleTask* get_origin_task(void) const { return orig_task; } public: Future initialize_task(InnerContext *ctx, @@ -709,6 +747,7 @@ namespace Legion { bool implicit_top_level = false); void initialize_must_epoch(MustEpochOp *epoch, unsigned index, bool do_registration); + void perform_base_dependence_analysis(void); public: virtual bool has_prepipeline_stage(void) const { return need_prepipeline_stage; } @@ -733,15 +772,13 @@ namespace Legion { virtual ApEvent get_task_completion(void) const; virtual TaskKind get_task_kind(void) const; public: - virtual void send_remote_context(AddressSpaceID target, - RemoteTask *dst); - public: - virtual void trigger_task_complete(void); + virtual void trigger_task_complete(bool deferred = false); virtual void trigger_task_commit(void); public: virtual void handle_future(const void *res, size_t res_size, bool owned); - virtual void handle_post_mapped(RtEvent pre = RtEvent::NO_RT_EVENT); + virtual void handle_post_mapped(bool deferral, + RtEvent pre = RtEvent::NO_RT_EVENT); virtual void handle_misspeculation(void); public: virtual void record_reference_mutation_effect(RtEvent event); @@ -749,11 +786,13 @@ namespace Legion { virtual bool pack_task(Serializer &rez, AddressSpaceID target); virtual bool unpack_task(Deserializer &derez, Processor current, std::set &ready_events); + virtual void pack_as_shard_task(Serializer &rez, AddressSpaceID target); virtual void perform_inlining(TaskContext *enclosing); virtual bool is_top_level_task(void) const { return top_level_task; } virtual void end_inline_task(const void *result, size_t result_size, bool owned); protected: + void pack_remote_versions(Serializer &rez); void pack_remote_complete(Serializer &rez); void pack_remote_commit(Serializer &rez); void unpack_remote_complete(Deserializer &derez); @@ -767,7 +806,6 @@ namespace Legion { static void process_unpack_remote_commit(Deserializer &derez); protected: Future result; - std::set child_operations; std::vector privilege_paths; protected: // Information for remotely executing task @@ -789,8 +827,6 @@ namespace Legion { bool local_function_task; // Whether we have to do intra-task alias analysis bool need_intra_task_alias_analysis; - protected: - std::map remote_instances; protected: std::map acquired_instances; }; @@ -824,6 +860,7 @@ namespace Legion { virtual bool distribute_task(void); virtual RtEvent perform_mapping(MustEpochOp *owner = NULL, const DeferMappingArgs *args = NULL); + virtual void shard_off(RtEvent mapped_precondition); virtual bool is_stealable(void) const; virtual bool can_early_complete(ApUserEvent &chain_event); virtual VersionInfo& get_version_info(unsigned idx); @@ -832,22 +869,21 @@ namespace Legion { virtual ApEvent get_task_completion(void) const; virtual TaskKind get_task_kind(void) const; public: - virtual void send_remote_context(AddressSpaceID target, - RemoteTask *dst); - public: - virtual void trigger_task_complete(void); + virtual void trigger_task_complete(bool deferred = false); virtual void trigger_task_commit(void); public: virtual bool pack_task(Serializer &rez, AddressSpaceID target); virtual bool unpack_task(Deserializer &derez, Processor current, std::set &ready_events); + virtual void pack_as_shard_task(Serializer &rez, AddressSpaceID target); virtual void perform_inlining(TaskContext *enclosing); virtual std::map* get_acquired_instances_ref(void); public: virtual void handle_future(const void *res, size_t res_size, bool owned); - virtual void handle_post_mapped(RtEvent pre = RtEvent::NO_RT_EVENT); + virtual void handle_post_mapped(bool deferral, + RtEvent pre = RtEvent::NO_RT_EVENT); virtual void handle_misspeculation(void); public: // ProjectionPoint methods @@ -894,6 +930,100 @@ namespace Legion { std::map remote_instances; }; + /** + * \class ShardTask + * A shard task is copy of a single task that is used for + * executing a single copy of a control replicated task. + * It implements the functionality of a single task so that + * we can use it mostly transparently for the execution of + * a single shard. + */ + class ShardTask : public SingleTask { + public: + ShardTask(Runtime *rt, ShardManager *manager, + ShardID shard_id, Processor target); + ShardTask(const ShardTask &rhs); + virtual ~ShardTask(void); + public: + ShardTask& operator=(const ShardTask &rhs); + public: + virtual void activate(void); + virtual void deactivate(void); + virtual SingleTask* get_origin_task(void) const + { assert(false); return NULL; } + virtual bool is_shard_task(void) const { return true; } + virtual bool is_top_level_task(void) const; + public: + // From MemoizableOp + virtual void replay_analysis(void); + public: + virtual void trigger_dependence_analysis(void); + virtual void resolve_false(bool speculated, bool launched); + virtual void early_map_task(void); + virtual bool distribute_task(void); + virtual RtEvent perform_must_epoch_version_analysis(MustEpochOp *own); + virtual RtEvent perform_mapping(MustEpochOp *owner = NULL, + const DeferMappingArgs *args = NULL); + virtual bool is_stealable(void) const; + virtual bool can_early_complete(ApUserEvent &chain_event); + virtual std::map* + get_acquired_instances_ref(void); + public: + virtual ApEvent get_task_completion(void) const; + virtual TaskKind get_task_kind(void) const; + public: + // Override these methods from operation class + virtual void trigger_mapping(void); + protected: + virtual void trigger_task_complete(bool deferred = false); + virtual void trigger_task_commit(void); + public: + virtual VersionInfo& get_version_info(unsigned idx); + public: + virtual void perform_physical_traversal(unsigned idx, + RegionTreeContext ctx, InstanceSet &valid); + virtual bool pack_task(Serializer &rez, AddressSpaceID target); + virtual bool unpack_task(Deserializer &derez, Processor current, + std::set &ready_events); + virtual void pack_as_shard_task(Serializer &rez, AddressSpaceID target); + RtEvent unpack_shard_task(Deserializer &derez); + virtual void perform_inlining(TaskContext *enclosing); + public: + virtual void handle_future(const void *res, + size_t res_size, bool owned); + virtual void handle_post_mapped(bool deferral, + RtEvent pre = RtEvent::NO_RT_EVENT); + virtual void handle_misspeculation(void); + protected: + virtual InnerContext* initialize_inner_execution_context(VariantImpl *v); + public: + virtual InnerContext* create_implicit_context(void); + public: + void launch_shard(void); + void extract_event_preconditions(const std::deque &insts); + void return_resources(ResourceTracker *target, + std::set &preconditions); + void report_leaks_and_duplicates(std::set &preconditions); + void handle_collective_message(Deserializer &derez); + void handle_future_map_request(Deserializer &derez); + void handle_equivalence_set_request(Deserializer &derez); + void handle_intra_space_dependence(Deserializer &derez); + void handle_resource_update(Deserializer &derez, + std::set &applied); + void handle_trace_update(Deserializer &derez, AddressSpaceID source); + ApBarrier handle_find_trace_shard_event(size_t temp_index, ApEvent event, + ShardID remote_shard); + public: + InstanceView* create_instance_top_view(PhysicalManager *manager, + AddressSpaceID source); + void initialize_implicit_task(InnerContext *context, TaskID tid, + MapperID mid, Processor proxy); + public: + const ShardID shard_id; + protected: + UniqueID remote_owner_uid; + }; + /** * \class IndexTask * An index task is used to represent an index space task @@ -928,9 +1058,13 @@ namespace Legion { const TaskArgument &pred_arg); void initialize_must_epoch(MustEpochOp *epoch, unsigned index, bool do_registration); + void perform_base_dependence_analysis(void); public: virtual void activate(void); virtual void deactivate(void); + protected: + void activate_index_task(void); + void deactivate_index_task(void); public: virtual bool has_prepipeline_stage(void) const { return need_prepipeline_stage; } @@ -951,7 +1085,7 @@ namespace Legion { virtual ApEvent get_task_completion(void) const; virtual TaskKind get_task_kind(void) const; protected: - virtual void trigger_task_complete(void); + virtual void trigger_task_complete(bool deferred = false); virtual void trigger_task_commit(void); public: virtual bool pack_task(Serializer &rez, AddressSpaceID target); @@ -980,10 +1114,16 @@ namespace Legion { virtual void handle_profiling_update(int count); public: virtual void register_must_epoch(void); + public: + // Make this a virtual method so for control replication we can + // create a different type of future map for the task + virtual FutureMapImpl* create_future_map(TaskContext *ctx, + IndexSpace launch_space, IndexSpace shard_space); public: // Methods for supporting intra-index-space mapping dependences virtual RtEvent find_intra_space_dependence(const DomainPoint &point); virtual void record_intra_space_dependence(const DomainPoint &point, + const DomainPoint &next, RtEvent point_mapped); public: virtual void record_reference_mutation_effect(RtEvent event); @@ -1085,6 +1225,9 @@ namespace Legion { virtual ~SliceTask(void); public: SliceTask& operator=(const SliceTask &rhs); + public: + inline UniqueID get_remote_owner_uid(void) const + { return remote_owner_uid; } public: virtual void activate(void); virtual void deactivate(void); @@ -1115,7 +1258,7 @@ namespace Legion { public: virtual void register_must_epoch(void); PointTask* clone_as_point_task(const DomainPoint &point); - void enumerate_points(void); + size_t enumerate_points(void); const void* get_predicate_false_result(size_t &result_size); public: virtual std::map* @@ -1125,7 +1268,7 @@ namespace Legion { void expand_replay_slices(std::list &slices); void find_profiling_reported(std::set &preconditions); protected: - virtual void trigger_task_complete(void); + virtual void trigger_task_complete(bool deferred = false); virtual void trigger_task_commit(void); public: virtual void record_reference_mutation_effect(RtEvent event); @@ -1169,6 +1312,7 @@ namespace Legion { // Methods for supporting intra-index-space mapping dependences virtual RtEvent find_intra_space_dependence(const DomainPoint &point); virtual void record_intra_space_dependence(const DomainPoint &point, + const DomainPoint &next, RtEvent point_mapped); public: // For collective instance creation @@ -1191,6 +1335,7 @@ namespace Legion { protected: friend class IndexTask; friend class PointTask; + friend class ReplMustEpochOp; std::vector points; protected: unsigned num_unmapped_points; @@ -1209,6 +1354,8 @@ namespace Legion { std::set map_applied_conditions; std::set complete_preconditions; std::set commit_preconditions; + protected: + std::set > unique_intra_space_deps; }; }; // namespace Internal diff --git a/runtime/legion/legion_trace.cc b/runtime/legion/legion_trace.cc index ec191026ad..1e307b2aae 100644 --- a/runtime/legion/legion_trace.cc +++ b/runtime/legion/legion_trace.cc @@ -22,6 +22,9 @@ #include "legion/legion_instances.h" #include "legion/legion_views.h" #include "legion/legion_context.h" +#include "legion/legion_replication.h" + +#include "realm/id.h" // TODO: remove this hackiness namespace Legion { namespace Internal { @@ -150,7 +153,7 @@ namespace Legion { // Register for this fence on every one of the operations in // the trace and then clear out the operations data structure for (std::set >::iterator it = - frontiers.begin(); it != frontiers.end(); ++it) + frontiers.begin(); it != frontiers.end(); ++it) { const std::pair &target = *it; #ifdef DEBUG_LEGION @@ -390,7 +393,7 @@ namespace Legion { // If this is the case we can do the normal registration if ((it->prev_idx == -1) || (it->next_idx == -1)) { - internal_op->register_dependence(target.first, target.second); + internal_op->register_dependence(target.first, target.second); #ifdef LEGION_SPY LegionSpy::log_mapping_dependence( op->get_context()->get_unique_id(), @@ -507,8 +510,8 @@ namespace Legion { const FieldMask dependence_mask = forest->get_node(field_space)->get_field_mask(it->dependent_fields); translation.push_back(DependenceRecord(index - it->previous_offset, - it->previous_req_index, it->current_req_index, - it->validates, it->dependence_type, dependence_mask)); + it->previous_req_index, it->current_req_index, it->validates, + it->dependence_type, dependence_mask)); } } return translated_deps[index]; @@ -714,7 +717,7 @@ namespace Legion { if ((it->prev_idx == -1) || (it->next_idx == -1)) { - op->register_dependence(target.first, target.second); + op->register_dependence(target.first, target.second); #ifdef LEGION_SPY LegionSpy::log_mapping_dependence( op->get_context()->get_unique_id(), @@ -776,7 +779,7 @@ namespace Legion { // If this is the case we can do the normal registration if ((it->prev_idx == -1) || (it->next_idx == -1)) { - internal_op->register_dependence(target.first, target.second); + internal_op->register_dependence(target.first, target.second); #ifdef LEGION_SPY LegionSpy::log_mapping_dependence( op->get_context()->get_unique_id(), @@ -899,8 +902,9 @@ namespace Legion { if (!source->is_internal_op()) { // Normal case - insert_dependence(DependenceRecord(finder->second, target_idx, - source_idx, validates, dtype, dep_mask)); + insert_dependence( + DependenceRecord(finder->second, target_idx, source_idx, + validates, dtype, dep_mask)); } else { @@ -916,7 +920,7 @@ namespace Legion { #endif insert_dependence(src_key, DependenceRecord(finder->second, target_idx, source_idx, - validates, dtype, dep_mask)); + validates, dtype, dep_mask)); } } } @@ -942,8 +946,9 @@ namespace Legion { FieldMask overlap = it->dependent_mask & dep_mask; if (!overlap) continue; - insert_dependence(DependenceRecord(it->operation_idx, - it->prev_idx, source_idx, it->validates, it->dtype, overlap)); + insert_dependence( + DependenceRecord(it->operation_idx, it->prev_idx, + source_idx, it->validates, it->dtype, overlap)); } } else @@ -1220,14 +1225,23 @@ namespace Legion { { PhysicalTrace *physical_trace = local_trace->get_physical_trace(); #ifdef DEBUG_LEGION - assert(current_template != NULL); assert(physical_trace != NULL); + assert(current_template != NULL); + assert(current_template->is_recording()); #endif - RtEvent pending_deletion = - physical_trace->fix_trace(current_template, this, has_blocking_call); - if (pending_deletion.exists()) - execution_precondition = Runtime::merge_events(NULL, - execution_precondition, ApEvent(pending_deletion)); + current_template->finalize(has_blocking_call); + if (!current_template->is_replayable()) + { + const RtEvent pending_deletion = + current_template->defer_template_deletion(); + if (pending_deletion.exists()) + execution_precondition = Runtime::merge_events(NULL, + execution_precondition, ApEvent(pending_deletion)); + physical_trace->record_failed_capture(current_template); + } + else + physical_trace->record_replayable_capture(current_template); + // Reset the local trace local_trace->initialize_tracing_state(); } if (remove_trace_reference && local_trace->remove_reference()) @@ -1361,7 +1375,6 @@ namespace Legion { get_completion_event()); current_template = physical_trace->get_current_template(); physical_trace->clear_cached_template(); - } FenceOp::trigger_dependence_analysis(); } @@ -1376,13 +1389,21 @@ namespace Legion { #ifdef DEBUG_LEGION assert(current_template != NULL); assert(local_trace->get_physical_trace() != NULL); + assert(current_template->is_recording()); #endif - RtEvent pending_deletion = - local_trace->get_physical_trace()->fix_trace(current_template, this, - has_blocking_call); - if (pending_deletion.exists()) - execution_precondition = Runtime::merge_events(NULL, - execution_precondition, ApEvent(pending_deletion)); + current_template->finalize(has_blocking_call); + PhysicalTrace *physical_trace = local_trace->get_physical_trace(); + if (!current_template->is_replayable()) + { + const RtEvent pending_deletion = + current_template->defer_template_deletion(); + if (pending_deletion.exists()) + execution_precondition = Runtime::merge_events(NULL, + execution_precondition, ApEvent(pending_deletion)); + physical_trace->record_failed_capture(current_template); + } + else + physical_trace->record_replayable_capture(current_template); local_trace->initialize_tracing_state(); } else if (replayed) @@ -1704,7 +1725,7 @@ namespace Legion { void TraceSummaryOp::deactivate(void) //-------------------------------------------------------------------------- { - deactivate_operation(); + deactivate_fence(); runtime->free_summary_op(this); } @@ -1761,8 +1782,9 @@ namespace Legion { //-------------------------------------------------------------------------- PhysicalTrace::PhysicalTrace(Runtime *rt, LegionTrace *lt) - : runtime(rt), logical_trace(lt), current_template(NULL), - nonreplayable_count(0), new_template_count(0), + : runtime(rt), logical_trace(lt), + repl_ctx(dynamic_cast(lt->ctx)), + current_template(NULL), nonreplayable_count(0), new_template_count(0), previous_template_completion(ApEvent::NO_AP_EVENT), execution_fence_event(ApEvent::NO_AP_EVENT) //-------------------------------------------------------------------------- @@ -1782,8 +1804,8 @@ namespace Legion { //-------------------------------------------------------------------------- PhysicalTrace::PhysicalTrace(const PhysicalTrace &rhs) - : runtime(NULL), logical_trace(NULL), current_template(NULL), - nonreplayable_count(0), new_template_count(0), + : runtime(NULL), logical_trace(NULL), repl_ctx(NULL), + current_template(NULL), nonreplayable_count(0), new_template_count(0), previous_template_completion(ApEvent::NO_AP_EVENT), execution_fence_event(ApEvent::NO_AP_EVENT) //-------------------------------------------------------------------------- @@ -1808,57 +1830,52 @@ namespace Legion { { // should never be called assert(false); + return *this; } //-------------------------------------------------------------------------- - RtEvent PhysicalTrace::fix_trace( - PhysicalTemplate *tpl, Operation *op, bool has_blocking_call) + void PhysicalTrace::record_replayable_capture(PhysicalTemplate *tpl) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(tpl->is_recording()); -#endif - tpl->finalize(op, has_blocking_call); - RtEvent pending_deletion = RtEvent::NO_RT_EVENT; - if (!tpl->is_replayable()) + templates.push_back(tpl); + if (++new_template_count > LEGION_NEW_TEMPLATE_WARNING_COUNT) { - pending_deletion = tpl->defer_template_deletion(); - if (++nonreplayable_count > LEGION_NON_REPLAYABLE_WARNING) - { - const std::string &message = tpl->get_replayable_message(); - const char *message_buffer = message.c_str(); - REPORT_LEGION_WARNING(LEGION_WARNING_NON_REPLAYABLE_COUNT_EXCEEDED, - "WARNING: The runtime has failed to memoize the trace more than " - "%u times, due to the absence of a replayable template. It is " - "highly likely that trace %u will not be memoized for the rest " - "of execution. The most recent template was not replayable " - "for the following reason: %s. Please change the mapper to stop " - "making memoization requests.", LEGION_NON_REPLAYABLE_WARNING, - logical_trace->get_trace_id(), message_buffer) - nonreplayable_count = 0; - } + REPORT_LEGION_WARNING(LEGION_WARNING_NEW_TEMPLATE_COUNT_EXCEEDED, + "WARNING: The runtime has created %d new replayable templates " + "for trace %u without replaying any existing templates. This " + "may mean that your mapper is not making mapper decisions " + "conducive to replaying templates. Please check that your " + "mapper is making decisions that align with prior templates. " + "If you believe that this number of templates is reasonable " + "please adjust the settings for LEGION_NEW_TEMPLATE_WARNING_COUNT " + "in legion_config.h.", LEGION_NEW_TEMPLATE_WARNING_COUNT, + logical_trace->get_trace_id()) + new_template_count = 0; } - else + // Reset the nonreplayable count when we find a replayable template + nonreplayable_count = 0; + current_template = NULL; + } + + //-------------------------------------------------------------------------- + void PhysicalTrace::record_failed_capture(PhysicalTemplate *tpl) + //-------------------------------------------------------------------------- + { + if (++nonreplayable_count > LEGION_NON_REPLAYABLE_WARNING) { - // Reset the nonreplayable count when we find a replayable template + const std::string &message = tpl->get_replayable_message(); + const char *message_buffer = message.c_str(); + REPORT_LEGION_WARNING(LEGION_WARNING_NON_REPLAYABLE_COUNT_EXCEEDED, + "WARNING: The runtime has failed to memoize the trace more than " + "%u times, due to the absence of a replayable template. It is " + "highly likely that trace %u will not be memoized for the rest " + "of execution. The most recent template was not replayable " + "for the following reason: %s. Please change the mapper to stop " + "making memoization requests.", LEGION_NON_REPLAYABLE_WARNING, + logical_trace->get_trace_id(), message_buffer) nonreplayable_count = 0; - templates.push_back(tpl); - if (++new_template_count > LEGION_NEW_TEMPLATE_WARNING_COUNT) - { - REPORT_LEGION_WARNING(LEGION_WARNING_NEW_TEMPLATE_COUNT_EXCEEDED, - "WARNING: The runtime has created %d new replayable templates " - "for trace %u without replaying any existing templates. This " - "may mean that your mapper is not making mapper decisions " - "conducive to replaying templates. Please check that your " - "mapper is making decisions that align with prior templates. " - "If you believe that this number of templates is reasonable " - "please adjust the settings for LEGION_NEW_TEMPLATE_WARNING_COUNT" - " in legion_config.h.", LEGION_NEW_TEMPLATE_WARNING_COUNT, - logical_trace->get_trace_id()) - new_template_count = 0; - } } - return pending_deletion; + current_template = NULL; } //-------------------------------------------------------------------------- @@ -1886,11 +1903,57 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + bool PhysicalTrace::find_viable_templates(ReplTraceReplayOp *op, + std::set &applied_events, + unsigned templates_to_find, + std::vector &viable_templates) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(templates_to_find > 0); +#endif + for (int index = viable_templates.empty() ? templates.size() - 1 : + viable_templates.back() - 1; index >= 0; index--) + { + PhysicalTemplate *tpl = templates[index]; + if (tpl->check_preconditions(op, applied_events)) + { + // A good tmplate so add it to the list + viable_templates.push_back(index); + // If we've found all our templates then we're done + if (--templates_to_find == 0) + return (index == 0); // whether we are done + } + } + return true; // Iterated over all the templates + } + + //-------------------------------------------------------------------------- + PhysicalTemplate* PhysicalTrace::select_template(unsigned index) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(index < templates.size()); + assert(templates[index]->is_replayable()); +#endif + // Reset the nonreplayable count when a replayable template satisfies + // the precondition + nonreplayable_count = 0; + current_template = templates[index]; + return current_template; + } + //-------------------------------------------------------------------------- PhysicalTemplate* PhysicalTrace::start_new_template(void) //-------------------------------------------------------------------------- { - current_template = new PhysicalTemplate(this, execution_fence_event); + // If we have a replicated context then we are making sharded templates + if (repl_ctx != NULL) + current_template = + new ShardedPhysicalTemplate(this, execution_fence_event, repl_ctx); + else + current_template = new PhysicalTemplate(this, execution_fence_event); return current_template; } @@ -2017,6 +2080,10 @@ namespace Legion { IndexSpaceExpression *expr1 = eq->set_expr; IndexSpaceExpression *expr2 = it->first->set_expr; + // THIS IS NOT A COMPLETE DOMINANCE TEST!!! + // We are assuming that equivalence sets are always uniquely + // represented at the leaves of the equivalence set tree and + // therefore we can test expressions more directly if (expr1 == expr2) { non_dominated -= overlap; @@ -2046,19 +2113,19 @@ namespace Legion { it != conditions.end(); ++it) for (FieldMaskSet::const_iterator eit = it->second.begin(); eit != it->second.end(); ++eit) - { - FieldMask mask = eit->second; - if (!set.dominates(it->first, eit->first, mask)) { - if (condition != NULL) + FieldMask mask = eit->second; + if (!set.dominates(it->first, eit->first, mask)) { - condition->view = it->first; - condition->eq = eit->first; - condition->mask = mask; + if (condition != NULL) + { + condition->view = it->first; + condition->eq = eit->first; + condition->mask = mask; + } + return false; } - return false; } - } return true; } @@ -2407,9 +2474,25 @@ namespace Legion { post.ensure(op, applied_events); } + //-------------------------------------------------------------------------- + bool PhysicalTemplate::check_preconditions(ReplTraceReplayOp *op, + std::set &applied_events) + //-------------------------------------------------------------------------- + { + return pre.require(op, applied_events); + } + + //-------------------------------------------------------------------------- + void PhysicalTemplate::apply_postcondition(ReplTraceSummaryOp *op, + std::set &applied_events) + //-------------------------------------------------------------------------- + { + post.ensure(op, applied_events); + } + //-------------------------------------------------------------------------- PhysicalTemplate::Replayable PhysicalTemplate::check_replayable( - bool has_blocking_call) const + ReplTraceOp *op, bool has_blocking_call) const //-------------------------------------------------------------------------- { if (has_blocking_call) @@ -2513,19 +2596,18 @@ namespace Legion { } //-------------------------------------------------------------------------- - void PhysicalTemplate::finalize(Operation *op, bool has_blocking_call) + void PhysicalTemplate::finalize(bool has_blocking_call, ReplTraceOp *op) //-------------------------------------------------------------------------- { - if (!recording_done.has_triggered()) - Runtime::trigger_event(recording_done); + trigger_recording_done(); recording = false; - replayable = check_replayable(has_blocking_call); + replayable = check_replayable(op, has_blocking_call); if (!replayable) { if (trace->runtime->dump_physical_traces) { - optimize(); + optimize(op); dump_template(); } if (!remote_memos.empty()) @@ -2533,7 +2615,7 @@ namespace Legion { return; } generate_conditions(); - optimize(); + optimize(op); if (trace->runtime->dump_physical_traces) dump_template(); size_t num_events = events.size(); events.clear(); @@ -2552,13 +2634,13 @@ namespace Legion { } //-------------------------------------------------------------------------- - void PhysicalTemplate::optimize(void) + void PhysicalTemplate::optimize(ReplTraceOp *op) //-------------------------------------------------------------------------- { std::vector gen; if (!(trace->runtime->no_trace_optimization || trace->runtime->no_fence_elision)) - elide_fences(gen); + elide_fences(gen, op); else { #ifdef DEBUG_LEGION @@ -2580,7 +2662,8 @@ namespace Legion { } //-------------------------------------------------------------------------- - void PhysicalTemplate::elide_fences(std::vector &gen) + void PhysicalTemplate::elide_fences(std::vector &gen, + ReplTraceOp *op) //-------------------------------------------------------------------------- { // Reserve some events for merges to be added during fence elision @@ -2636,6 +2719,8 @@ namespace Legion { unsigned merge_starts = events.size(); events.resize(events.size() + num_merges); + elide_fences_pre_sync(op); + // We are now going to break the invariant that // the generator of events[idx] is instructions[idx]. // After fence elision, the generator of events[idx] is @@ -2649,6 +2734,7 @@ namespace Legion { InstructionKind kind = inst->get_kind(); std::set users; unsigned *precondition_idx = NULL; + std::set ready_events; switch (kind) { case COMPLETE_REPLAY: @@ -2657,7 +2743,7 @@ namespace Legion { std::map::iterator finder = op_views.find(replay->owner); if (finder == op_views.end()) break; - find_all_last_users(finder->second, users); + find_all_last_users(finder->second, users, ready_events); precondition_idx = &replay->rhs; break; } @@ -2669,7 +2755,7 @@ namespace Legion { #ifdef DEBUG_LEGION assert(finder != copy_views.end()); #endif - find_all_last_users(finder->second, users); + find_all_last_users(finder->second, users, ready_events); precondition_idx = ©->precondition_idx; break; } @@ -2681,7 +2767,7 @@ namespace Legion { #ifdef DEBUG_LEGION assert(finder != copy_views.end()); #endif - find_all_last_users(finder->second, users); + find_all_last_users(finder->second, users, ready_events); precondition_idx = &fill->precondition_idx; break; } @@ -2694,7 +2780,7 @@ namespace Legion { #ifdef DEBUG_LEGION assert(finder != copy_views.end()); #endif - find_all_last_users(finder->second, users); + find_all_last_users(finder->second, users, ready_events); precondition_idx = &reduction->precondition_idx; break; } @@ -2704,7 +2790,14 @@ namespace Legion { break; } } - + // If we have any ready events then wait for them to be ready + if (!ready_events.empty()) + { + const RtEvent wait_on = Runtime::merge_events(ready_events); + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + } + // Now see if we have any users to update if (users.size() > 0) { Instruction *generator_inst = instructions[*precondition_idx]; @@ -2730,6 +2823,7 @@ namespace Legion { } instructions.swap(new_instructions); new_instructions.clear(); + elide_fences_post_sync(op); // If we added events for fence elision then resize events so that // all the new events from a previous trace are generated by the // fence instruction at the beginning of the template @@ -2779,6 +2873,12 @@ namespace Legion { used[gen[trigger->rhs]] = true; break; } + case BARRIER_ARRIVAL: + { + BarrierArrival *arrival = inst->as_barrier_arrival(); + used[gen[arrival->rhs]] = true; + break; + } case ISSUE_COPY: { IssueCopy *copy = inst->as_issue_copy(); @@ -2815,6 +2915,7 @@ namespace Legion { case CREATE_AP_USER_EVENT: case SET_OP_SYNC_EVENT: case ASSIGN_FENCE_COMPLETION: + case BARRIER_ADVANCE: { break; } @@ -2838,9 +2939,7 @@ namespace Legion { } std::vector to_delete; std::vector new_gen(gen.size(), -1U); - for (std::map::iterator it = frontiers.begin(); - it != frontiers.end(); ++it) - new_gen[it->second] = 0; + initialize_generators(new_gen); for (unsigned idx = 0; idx < instructions.size(); ++idx) if (used[idx]) { @@ -2870,6 +2969,29 @@ namespace Legion { delete to_delete[idx]; } + //-------------------------------------------------------------------------- + void PhysicalTemplate::initialize_generators(std::vector &new_gen) + //-------------------------------------------------------------------------- + { + for (std::map::iterator it = + frontiers.begin(); it != frontiers.end(); ++it) + new_gen[it->second] = 0; + } + + //-------------------------------------------------------------------------- + void PhysicalTemplate::initialize_eliminate_dead_code_frontiers( + const std::vector &gen, std::vector &used) + //-------------------------------------------------------------------------- + { + for (std::map::iterator it = frontiers.begin(); + it != frontiers.end(); ++it) + { + unsigned g = gen[it->first]; + if (g != -1U && g < instructions.size()) + used[g] = true; + } + } + //-------------------------------------------------------------------------- void PhysicalTemplate::prepare_parallel_replay( const std::vector &gen) @@ -2998,6 +3120,11 @@ namespace Legion { event_to_check = &inst->as_trigger_event()->rhs; break; } + case BARRIER_ARRIVAL: + { + event_to_check = &inst->as_barrier_arrival()->rhs; + break; + } case ISSUE_COPY : { event_to_check = &inst->as_issue_copy()->precondition_idx; @@ -3063,6 +3190,19 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + void PhysicalTemplate::initialize_transitive_reduction_frontiers( + std::vector &topo_order, std::vector &inv_topo_order) + //-------------------------------------------------------------------------- + { + for (std::map::iterator it = + frontiers.begin(); it != frontiers.end(); ++it) + { + inv_topo_order[it->second] = topo_order.size(); + topo_order.push_back(it->second); + } + } + //-------------------------------------------------------------------------- void PhysicalTemplate::transitive_reduction(void) //-------------------------------------------------------------------------- @@ -3077,12 +3217,7 @@ namespace Legion { std::vector > incoming(events.size()); std::vector > outgoing(events.size()); - for (std::map::iterator it = frontiers.begin(); - it != frontiers.end(); ++it) - { - inv_topo_order[it->second] = topo_order.size(); - topo_order.push_back(it->second); - } + initialize_transitive_reduction_frontiers(topo_order, inv_topo_order); std::map term_insts; for (unsigned idx = 0; idx < instructions.size(); ++idx) @@ -3110,6 +3245,13 @@ namespace Legion { outgoing[trigger->rhs].push_back(trigger->lhs); break; } + case BARRIER_ARRIVAL: + { + BarrierArrival *arrival = inst->as_barrier_arrival(); + incoming[arrival->lhs].push_back(arrival->rhs); + outgoing[arrival->rhs].push_back(arrival->lhs); + break; + } case MERGE_EVENT : { MergeEvent *merge = inst->as_merge_event(); @@ -3151,6 +3293,13 @@ namespace Legion { topo_order.push_back(sync->lhs); break; } + case BARRIER_ADVANCE: + { + BarrierAdvance *advance = inst->as_barrier_advance(); + inv_topo_order[advance->lhs] = topo_order.size(); + topo_order.push_back(advance->lhs); + break; + } case SET_EFFECTS : { break; @@ -3337,9 +3486,7 @@ namespace Legion { instructions.swap(new_instructions); std::vector new_gen(gen.size(), -1U); - for (std::map::iterator it = frontiers.begin(); - it != frontiers.end(); ++it) - new_gen[it->second] = 0; + initialize_generators(new_gen); for (unsigned idx = 0; idx < instructions.size(); ++idx) { @@ -3366,6 +3513,13 @@ namespace Legion { if (subst >= 0) trigger->rhs = (unsigned)subst; break; } + case BARRIER_ARRIVAL: + { + BarrierArrival *arrival = inst->as_barrier_arrival(); + int subst = substs[arrival->rhs]; + if (subst >= 0) arrival->rhs = (unsigned)subst; + break; + } case MERGE_EVENT : { MergeEvent *merge = inst->as_merge_event(); @@ -3420,6 +3574,12 @@ namespace Legion { lhs = sync->lhs; break; } + case BARRIER_ADVANCE: + { + BarrierAdvance *advance = inst->as_barrier_advance(); + lhs = advance->lhs; + break; + } case ASSIGN_FENCE_COMPLETION : { lhs = fence_completion_id; @@ -3526,10 +3686,20 @@ namespace Legion { used[gen[complete->rhs]] = true; break; } + case BARRIER_ARRIVAL: + { + BarrierArrival *arrival = inst->as_barrier_arrival(); +#ifdef DEBUG_LEGION + assert(gen[arrival->rhs] != -1U); +#endif + used[gen[arrival->rhs]] = true; + break; + } case GET_TERM_EVENT: case CREATE_AP_USER_EVENT: case SET_OP_SYNC_EVENT: case ASSIGN_FENCE_COMPLETION: + case BARRIER_ADVANCE: { break; } @@ -3540,13 +3710,7 @@ namespace Legion { } } } - for (std::map::iterator it = frontiers.begin(); - it != frontiers.end(); ++it) - { - unsigned g = gen[it->first]; - if (g != -1U && g < instructions.size()) - used[g] = true; - } + initialize_eliminate_dead_code_frontiers(gen, used); std::vector inv_gen(instructions.size(), -1U); for (unsigned idx = 0; idx < gen.size(); ++idx) @@ -3559,9 +3723,7 @@ namespace Legion { std::vector new_instructions; std::vector to_delete; std::vector new_gen(gen.size(), -1U); - for (std::map::iterator it = frontiers.begin(); - it != frontiers.end(); ++it) - new_gen[it->second] = 0; + initialize_generators(new_gen); for (unsigned idx = 0; idx < instructions.size(); ++idx) { if (used[idx]) @@ -3748,6 +3910,16 @@ namespace Legion { insert_instruction(new GetTermEvent(*this, lhs_, key)); } + //-------------------------------------------------------------------------- + void PhysicalTemplate::request_term_event(ApUserEvent &term_event) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!term_event.exists() || term_event.has_triggered()); +#endif + term_event = Runtime::create_ap_user_event(NULL); + } + //-------------------------------------------------------------------------- void PhysicalTemplate::record_create_ap_user_event( ApUserEvent lhs, Memoizable *memo) @@ -3784,10 +3956,11 @@ namespace Legion { #ifdef DEBUG_LEGION assert(is_recording()); #endif - + // Do this first in case it gets pre-empted + const unsigned rhs_ = find_event(rhs, tpl_lock); unsigned lhs_ = find_or_convert_event(lhs); events.push_back(ApEvent()); - insert_instruction(new TriggerEvent(*this, lhs_, find_event(rhs), + insert_instruction(new TriggerEvent(*this, lhs_, rhs_, find_trace_local_id(memo))); } @@ -3853,12 +4026,9 @@ namespace Legion { #ifndef LEGION_DISABLE_EVENT_PRUNING if (!lhs.exists() || (rhs.find(lhs) != rhs.end())) { - Realm::UserEvent rename(Realm::UserEvent::create_user_event()); - if (rhs.find(lhs) != rhs.end()) - rename.trigger(lhs); - else - rename.trigger(); - lhs = ApEvent(rename); + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename, lhs); + lhs = rename; } #endif @@ -3895,7 +4065,8 @@ namespace Legion { #ifdef DEBUG_LEGION assert(is_recording()); #endif - + // Do this first in case it gets preempted + const unsigned rhs_ = find_event(precondition, tpl_lock); unsigned lhs_ = convert_event(lhs); insert_instruction(new IssueCopy( *this, lhs_, expr, find_trace_local_id(memo), @@ -3903,7 +4074,7 @@ namespace Legion { #ifdef LEGION_SPY src_tree_id, dst_tree_id, #endif - find_event(precondition), redop, reduction_fold)); + rhs_, redop, reduction_fold)); } //-------------------------------------------------------------------------- @@ -3938,16 +4109,17 @@ namespace Legion { #endif if (!lhs.exists()) { - Realm::UserEvent rename(Realm::UserEvent::create_user_event()); - rename.trigger(); - lhs = ApEvent(rename); + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename); + lhs = rename; } AutoLock tpl_lock(template_lock); #ifdef DEBUG_LEGION assert(is_recording()); #endif - + // Do this first in case it gets preempted + const unsigned rhs_ = find_event(precondition, tpl_lock); unsigned lhs_ = convert_event(lhs); insert_instruction(new IssueFill(*this, lhs_, expr, find_trace_local_id(memo), @@ -3955,7 +4127,7 @@ namespace Legion { #ifdef LEGION_SPY handle, tree_id, #endif - find_event(precondition))); + rhs_)); } #ifdef LEGION_GPU_REDUCTIONS @@ -3984,12 +4156,12 @@ namespace Legion { #ifdef DEBUG_LEGION assert(is_recording()); #endif - + unsigned rhs_ = find_event(precondition, tpl_lock); unsigned lhs_ = convert_event(lhs); insert_instruction(new GPUReduction( *this, lhs_, expr, find_trace_local_id(memo), src_fields, dst_fields, gpu, gpu_task_id, src, dst, - find_event(precondition), redop, reduction_fold)); + rhs_, redop, reduction_fold)); } #endif @@ -4011,7 +4183,8 @@ namespace Legion { InstanceView *view, const RegionUsage &usage, const FieldMask &user_mask, - bool update_validity) + bool update_validity, + std::set &applied) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -4042,8 +4215,8 @@ namespace Legion { views.insert(expr, mask); if (update_validity) { - update_valid_views(view, *eit, usage, mask, true); - add_view_user(view, usage, entry, expr, mask); + update_valid_views(view, *eit, usage, mask, true, applied); + add_view_user(view, usage, entry, expr, mask, applied); } } } @@ -4055,6 +4228,8 @@ namespace Legion { //-------------------------------------------------------------------------- { AutoLock tpl_lock(template_lock); + // If you change this then also change + // ShardedPhysicalTemplate::handle_update_post_fill #ifdef DEBUG_LEGION assert(is_recording()); #endif @@ -4085,10 +4260,10 @@ namespace Legion { #ifdef DEBUG_LEGION assert(is_recording()); #endif - const unsigned lhs_ = find_event(lhs); - record_fill_views(tracing_srcs); + const unsigned lhs_ = find_event(lhs, tpl_lock); + record_fill_views(tracing_srcs, applied_events); record_views(lhs_, expr, RegionUsage(LEGION_WRITE_ONLY, - LEGION_EXCLUSIVE, 0), tracing_dsts, eqs); + LEGION_EXCLUSIVE, 0), tracing_dsts, eqs, applied_events); record_copy_views(lhs_, expr, tracing_dsts); } @@ -4123,12 +4298,12 @@ namespace Legion { #ifdef DEBUG_LEGION assert(is_recording()); #endif - const unsigned lhs_ = find_event(lhs); + const unsigned lhs_ = find_event(lhs, tpl_lock); record_views(lhs_, expr, RegionUsage(LEGION_READ_ONLY, - LEGION_EXCLUSIVE, 0), tracing_srcs, src_eqs); + LEGION_EXCLUSIVE, 0), tracing_srcs, src_eqs, applied_events); record_copy_views(lhs_, expr, tracing_srcs); record_views(lhs_, expr, RegionUsage(LEGION_WRITE_ONLY, - LEGION_EXCLUSIVE, 0), tracing_dsts, dst_eqs); + LEGION_EXCLUSIVE, 0), tracing_dsts, dst_eqs, applied_events); record_copy_views(lhs_, expr, tracing_dsts); } @@ -4137,7 +4312,8 @@ namespace Legion { IndexSpaceExpression *expr, const RegionUsage &usage, const FieldMaskSet &views, - const LegionList >::aligned &eqs) + const LegionList >::aligned &eqs, + std::set &applied) //-------------------------------------------------------------------------- { RegionTreeForest *forest = trace->runtime->forest; @@ -4158,8 +4334,8 @@ namespace Legion { forest->intersect_index_spaces((*eit)->set_expr, expr); if (intersect->is_empty()) continue; - update_valid_views(vit->first, *eit, usage, mask, false); - add_view_user(vit->first, usage, entry, intersect, mask); + update_valid_views(vit->first, *eit, usage, mask, false, applied); + add_view_user(vit->first, usage, entry, intersect, mask, applied); } } } @@ -4170,7 +4346,8 @@ namespace Legion { EquivalenceSet *eq, const RegionUsage &usage, const FieldMask &user_mask, - bool invalidates) + bool invalidates, + std::set &applied) //-------------------------------------------------------------------------- { std::set &views= view_groups[view->get_manager()->tree_id]; @@ -4225,10 +4402,12 @@ namespace Legion { const RegionUsage &usage, unsigned user_index, IndexSpaceExpression *user_expr, - const FieldMask &user_mask) + const FieldMask &user_mask, + std::set &applied, + int owner_shard) //-------------------------------------------------------------------------- { - ViewUser *user = new ViewUser(usage, user_index, user_expr); + ViewUser *user = new ViewUser(usage, user_index, user_expr, owner_shard); all_users.insert(user); RegionTreeForest *forest = trace->runtime->forest; FieldMaskSet &users = view_users[view]; @@ -4281,7 +4460,8 @@ namespace Legion { } //-------------------------------------------------------------------------- - void PhysicalTemplate::record_fill_views(const FieldMaskSet&views) + void PhysicalTemplate::record_fill_views(const FieldMaskSet&views, + std::set &applied_events) //-------------------------------------------------------------------------- { for (FieldMaskSet::const_iterator it = views.begin(); @@ -4311,9 +4491,9 @@ namespace Legion { #endif if (!lhs.exists()) { - Realm::UserEvent rename(Realm::UserEvent::create_user_event()); - rename.trigger(); - lhs = ApEvent(rename); + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename); + lhs = rename; } AutoLock tpl_lock(template_lock); #ifdef DEBUG_LEGION @@ -4336,10 +4516,9 @@ namespace Legion { #ifdef DEBUG_LEGION assert(is_recording()); #endif - + const unsigned rhs_ = find_event(rhs, tpl_lock); events.push_back(ApEvent()); - insert_instruction(new SetEffects(*this, find_trace_local_id(memo), - find_event(rhs))); + insert_instruction(new SetEffects(*this, find_trace_local_id(memo),rhs_)); } //-------------------------------------------------------------------------- @@ -4351,9 +4530,62 @@ namespace Legion { #ifdef DEBUG_LEGION assert(is_recording()); #endif - + // Do this first in case it gets preempted + const unsigned rhs_ = find_event(rhs, tpl_lock); events.push_back(ApEvent()); - insert_instruction(new CompleteReplay(*this, lhs, find_event(rhs))); + insert_instruction(new CompleteReplay(*this, lhs, rhs_)); + } + + //-------------------------------------------------------------------------- + void PhysicalTemplate::record_owner_shard(unsigned tid, ShardID owner) + //-------------------------------------------------------------------------- + { + // Only called on sharded physical template + assert(false); + } + + //-------------------------------------------------------------------------- + void PhysicalTemplate::record_local_space(unsigned tid, IndexSpace sp) + //-------------------------------------------------------------------------- + { + // Only called on sharded physical template + assert(false); + } + + //-------------------------------------------------------------------------- + void PhysicalTemplate::record_sharding_function(unsigned tid, + ShardingFunction *function) + //-------------------------------------------------------------------------- + { + // Only called on sharded physical template + assert(false); + } + + //-------------------------------------------------------------------------- + ShardID PhysicalTemplate::find_owner_shard(unsigned tid) + //-------------------------------------------------------------------------- + { + // Only called on sharded physical template + assert(false); + return 0; + } + + //-------------------------------------------------------------------------- + IndexSpace PhysicalTemplate::find_local_space(unsigned tid) + //-------------------------------------------------------------------------- + { + // Only called on sharded physical template + assert(false); + return IndexSpace::NO_SPACE; + } + + //-------------------------------------------------------------------------- + ShardingFunction* PhysicalTemplate::find_sharding_function(unsigned tid) + //-------------------------------------------------------------------------- + { + // Only called on sharded physical template + assert(false); + return NULL; } //-------------------------------------------------------------------------- @@ -4433,7 +4665,11 @@ namespace Legion { } //-------------------------------------------------------------------------- +#ifdef DEBUG_LEGION + unsigned PhysicalTemplate::convert_event(const ApEvent &event, bool check) +#else inline unsigned PhysicalTemplate::convert_event(const ApEvent &event) +#endif //-------------------------------------------------------------------------- { unsigned event_ = events.size(); @@ -4446,10 +4682,11 @@ namespace Legion { } //-------------------------------------------------------------------------- - inline unsigned PhysicalTemplate::find_event(const ApEvent &event) const + inline unsigned PhysicalTemplate::find_event(const ApEvent &event, + AutoLock &tpl_lock) //-------------------------------------------------------------------------- { - std::map::const_iterator finder= event_map.find(event); + std::map::const_iterator finder = event_map.find(event); #ifdef DEBUG_LEGION assert(finder != event_map.end()); #endif @@ -4489,31 +4726,33 @@ namespace Legion { //-------------------------------------------------------------------------- void PhysicalTemplate::find_all_last_users(ViewExprs &view_exprs, - std::set &users) + std::set &users, + std::set &ready_events) //-------------------------------------------------------------------------- { for (ViewExprs::iterator it = view_exprs.begin(); it != view_exprs.end(); ++it) for (FieldMaskSet::iterator eit = it->second.begin(); eit != it->second.end(); ++eit) - find_last_users(it->first, eit->first, eit->second, users); + find_last_users(it->first,eit->first,eit->second,users,ready_events); } //-------------------------------------------------------------------------- void PhysicalTemplate::find_last_users(InstanceView *view, IndexSpaceExpression *expr, const FieldMask &mask, - std::set &users) + std::set &users, + std::set &ready_events) //-------------------------------------------------------------------------- { if (expr->is_empty()) return; - ViewUsers::iterator finder = view_users.find(view); + ViewUsers::const_iterator finder = view_users.find(view); if (finder == view_users.end()) return; RegionTreeForest *forest = trace->runtime->forest; - for (FieldMaskSet::iterator uit = finder->second.begin(); uit != - finder->second.end(); ++uit) + for (FieldMaskSet::const_iterator uit = + finder->second.begin(); uit != finder->second.end(); ++uit) if (!!(uit->second & mask)) { ViewUser *user = uit->first; @@ -4559,75 +4798,2138 @@ namespace Legion { } ///////////////////////////////////////////////////////////// - // Instruction - ///////////////////////////////////////////////////////////// - - //-------------------------------------------------------------------------- - Instruction::Instruction(PhysicalTemplate& tpl, const TraceLocalID &o) - : operations(tpl.operations), events(tpl.events), - user_events(tpl.user_events), owner(o) - //-------------------------------------------------------------------------- - { - } - - ///////////////////////////////////////////////////////////// - // GetTermEvent + // ShardedPhysicalTemplate ///////////////////////////////////////////////////////////// //-------------------------------------------------------------------------- - GetTermEvent::GetTermEvent(PhysicalTemplate& tpl, unsigned l, - const TraceLocalID& r) - : Instruction(tpl, r), lhs(l) + ShardedPhysicalTemplate::ShardedPhysicalTemplate(PhysicalTrace *trace, + ApEvent fence_event, ReplicateContext *ctx) + : PhysicalTemplate(trace, fence_event), repl_ctx(ctx), + local_shard(repl_ctx->owner_shard->shard_id), + total_shards(repl_ctx->shard_manager->total_shards), + template_index(repl_ctx->register_trace_template(this)), + total_replays(0), updated_advances(0), + recording_barrier(repl_ctx->get_next_trace_recording_barrier()), + recurrent_replays(0), updated_frontiers(0) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(lhs < events.size()); - assert(operations.find(owner) != operations.end()); -#endif + repl_ctx->add_reference(); } //-------------------------------------------------------------------------- - void GetTermEvent::execute(void) + ShardedPhysicalTemplate::ShardedPhysicalTemplate( + const ShardedPhysicalTemplate &rhs) + : PhysicalTemplate(rhs), repl_ctx(rhs.repl_ctx), + local_shard(rhs.local_shard), total_shards(rhs.total_shards), + template_index(rhs.template_index) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(operations.find(owner) != operations.end()); - assert(operations.find(owner)->second != NULL); -#endif - operations[owner]->replay_mapping_output(); - events[lhs] = operations[owner]->get_memo_completion(); + // should never be called + assert(false); } //-------------------------------------------------------------------------- - std::string GetTermEvent::to_string(void) + ShardedPhysicalTemplate::~ShardedPhysicalTemplate(void) //-------------------------------------------------------------------------- { - std::stringstream ss; - ss << "events[" << lhs << "] = operations[" << owner - << "].get_completion_event() (op kind: " - << Operation::op_names[operations[owner]->get_memoizable_kind()] - << ")"; - return ss.str(); + for (std::map::iterator it = + local_frontiers.begin(); it != local_frontiers.end(); it++) + it->second.destroy_barrier(); + // Unregister ourselves from the context and then remove our reference + repl_ctx->unregister_trace_template(template_index); + if (repl_ctx->remove_reference()) + delete repl_ctx; } - ///////////////////////////////////////////////////////////// - // CreateApUserEvent - ///////////////////////////////////////////////////////////// - //-------------------------------------------------------------------------- - CreateApUserEvent::CreateApUserEvent(PhysicalTemplate& tpl, unsigned l, - const TraceLocalID &o) - : Instruction(tpl, o), lhs(l) + void ShardedPhysicalTemplate::initialize(Runtime *runtime, + ApEvent completion, bool recurrent) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(lhs < events.size()); - assert(user_events.find(lhs) != user_events.end()); -#endif - } - - //-------------------------------------------------------------------------- - void CreateApUserEvent::execute(void) + // We have to make sure that the previous trace replay is done before + // we start changing these data structures for the next replay + if (replay_done.exists() && !replay_done.has_triggered()) + replay_done.wait(); + // Now update all of our barrier information + if (recurrent) + { + // If we've run out of generations update the local barriers and + // send out the updates to everyone + if (recurrent_replays++ == Realm::Barrier::MAX_PHASES) + { + std::map > + notifications; + // Update our barriers and record which updates to send out + for (std::map::iterator it = + local_frontiers.begin(); it != local_frontiers.end(); it++) + { + const ApBarrier new_barrier( + Realm::Barrier::create_barrier(1/*arrival count*/)); +#ifdef DEBUG_LEGION + assert(local_subscriptions.find(it->first) != + local_subscriptions.end()); +#endif + const std::set &shards = local_subscriptions[it->first]; + for (std::set::const_iterator sit = + shards.begin(); sit != shards.end(); sit++) + notifications[*sit][it->second] = new_barrier; + // destroy the old barrier and replace it with the new one + it->second.destroy_barrier(); + it->second = new_barrier; + } + // Send out the notifications to all the remote shards + ShardManager *manager = repl_ctx->shard_manager; + for (std::map >::const_iterator + nit = notifications.begin(); nit != notifications.end(); nit++) + { + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(nit->first); + rez.serialize(template_index); + rez.serialize(FRONTIER_BARRIER_REFRESH); + rez.serialize(nit->second.size()); + for (std::map::const_iterator it = + nit->second.begin(); it != nit->second.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + manager->send_trace_update(nit->first, rez); + } + // Now we wait to see that we get all of our remote barriers updated + RtEvent wait_on; + { + AutoLock tpl_lock(template_lock); + if (updated_frontiers < remote_frontiers.size()) + { + update_frontiers_ready = Runtime::create_rt_user_event(); + wait_on = update_frontiers_ready; + } + else // Reset this back to zero for the next round + updated_frontiers = 0; + } + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + // Reset this back to zero after barrier updates + recurrent_replays = 0; + } + // Now we can do the normal update of events based on our barriers + for (std::map::iterator it = + local_frontiers.begin(); it != local_frontiers.end(); it++) + { + Runtime::phase_barrier_arrive(it->second, 1/*count*/, + events[it->first]); + Runtime::advance_barrier(it->second); + } + PhysicalTemplate::initialize(runtime, completion, recurrent); + for (std::vector >::iterator it = + remote_frontiers.begin(); it != remote_frontiers.end(); it++) + { + events[it->second] = it->first; + Runtime::advance_barrier(it->first); + } + } + else + { + PhysicalTemplate::initialize(runtime, completion, recurrent); + for (std::vector >::const_iterator it = + remote_frontiers.begin(); it != remote_frontiers.end(); it++) + events[it->second] = completion; + } + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_merge_events(ApEvent &lhs, + const std::set &rhs, Memoizable *memo) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(memo != NULL); +#endif + AutoLock tpl_lock(template_lock); +#ifdef DEBUG_LEGION + assert(is_recording()); +#endif + std::set rhs_; + std::set wait_for; + std::vector pending_events; + std::map request_events; + for (std::set::const_iterator it = + rhs.begin(); it != rhs.end(); it++) + { + if (!it->exists()) + continue; + std::map::iterator finder = event_map.find(*it); + if (finder == event_map.end()) + { + // We're going to need to check this event later + pending_events.push_back(*it); + // See if anyone else has requested this event yet + std::map::const_iterator request_finder = + pending_event_requests.find(*it); + if (request_finder == pending_event_requests.end()) + { + const RtUserEvent request_event = Runtime::create_rt_user_event(); + pending_event_requests[*it] = request_event; + wait_for.insert(request_event); + request_events[*it] = request_event; + } + else + wait_for.insert(request_finder->second); + } + else if (finder->second != NO_INDEX) + rhs_.insert(finder->second); + } + // If we have anything to wait for we need to do that + if (!wait_for.empty()) + { + tpl_lock.release(); + // Send any request messages first + if (!request_events.empty()) + { + for (std::map::const_iterator it = + request_events.begin(); it != request_events.end(); it++) + request_remote_shard_event(it->first, it->second); + } + // Do the wait + const RtEvent wait_on = Runtime::merge_events(wait_for); + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + tpl_lock.reacquire(); + // All our pending events should be here now + for (std::vector::const_iterator it = + pending_events.begin(); it != pending_events.end(); it++) + { + std::map::iterator finder = event_map.find(*it); +#ifdef DEBUG_LEGION + assert(finder != event_map.end()); +#endif + if (finder->second != NO_INDEX) + rhs_.insert(finder->second); + } + } + if (rhs_.size() == 0) + rhs_.insert(fence_completion_id); + + // If the lhs event wasn't made on this node then we need to rename it + // because we need all events to go back to a node where we know that + // we have a shard that can answer queries about it + const AddressSpaceID event_space = find_event_space(lhs); + if (event_space != repl_ctx->runtime->address_space) + { + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename, lhs); + lhs = rename; + } +#ifndef LEGION_DISABLE_EVENT_PRUNING + else if (!lhs.exists() || (rhs.find(lhs) != rhs.end())) + { + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename, lhs); + lhs = rename; + } +#endif + insert_instruction(new MergeEvent(*this, convert_event(lhs), rhs_, + memo->get_trace_local_id())); + } + +#ifdef DEBUG_LEGION + //-------------------------------------------------------------------------- + unsigned ShardedPhysicalTemplate::convert_event(const ApEvent &event, + bool check) + //-------------------------------------------------------------------------- + { + // We should only be recording events made on our node + assert(!check || + (find_event_space(event) == repl_ctx->runtime->address_space)); + return PhysicalTemplate::convert_event(event, check); + } +#endif + + //-------------------------------------------------------------------------- + unsigned ShardedPhysicalTemplate::find_event(const ApEvent &event, + AutoLock &tpl_lock) + //-------------------------------------------------------------------------- + { + std::map::const_iterator finder = + event_map.find(event); + // If we've already got it then we're done + if (finder != event_map.end()) + { +#ifdef DEBUG_LEGION + assert(finder->second != NO_INDEX); +#endif + return finder->second; + } + // If we don't have it then we need to request it + // See if someone else already sent the request + RtEvent wait_for; + RtUserEvent request_event; + std::map::const_iterator request_finder = + pending_event_requests.find(event); + if (request_finder == pending_event_requests.end()) + { + // We're the first ones so send the request + request_event = Runtime::create_rt_user_event(); + wait_for = request_event; + pending_event_requests[event] = wait_for; + } + else + wait_for = request_finder->second; + // Can't be holding the lock while we wait + tpl_lock.release(); + // Send the request if necessary + if (request_event.exists()) + request_remote_shard_event(event, request_event); + if (wait_for.exists()) + wait_for.wait(); + tpl_lock.reacquire(); + // Once we get here then there better be an answer + finder = event_map.find(event); +#ifdef DEBUG_LEGION + assert(finder != event_map.end()); + assert(finder->second != NO_INDEX); +#endif + return finder->second; + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_issue_copy(Memoizable *memo, + ApEvent &lhs, IndexSpaceExpression *expr, + const std::vector& src_fields, + const std::vector& dst_fields, +#ifdef LEGION_SPY + RegionTreeID src_tree_id, + RegionTreeID dst_tree_id, +#endif + ApEvent precondition, PredEvent pred_guard, + ReductionOpID redop, bool reduction_fold) + //-------------------------------------------------------------------------- + { + // Make sure the lhs event is local to our shard + if (lhs.exists()) + { + const AddressSpaceID event_space = find_event_space(lhs); + if (event_space != repl_ctx->runtime->address_space) + { + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename, lhs); + lhs = rename; + } + } + // Then do the base call + PhysicalTemplate::record_issue_copy(memo, lhs, expr,src_fields,dst_fields, +#ifdef LEGION_SPY + src_tree_id, dst_tree_id, +#endif + precondition, pred_guard, + redop, reduction_fold); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_issue_indirect(Memoizable *memo, + ApEvent &lhs, IndexSpaceExpression *expr, + const std::vector& src_fields, + const std::vector& dst_fields, + const std::vector &indirections, + ApEvent precondition, PredEvent pred_guard) + //-------------------------------------------------------------------------- + { + // Make sure the lhs event is local to our shard + if (lhs.exists()) + { + const AddressSpaceID event_space = find_event_space(lhs); + if (event_space != repl_ctx->runtime->address_space) + { + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename, lhs); + lhs = rename; + } + } + // Then do the base call + PhysicalTemplate::record_issue_indirect(memo, lhs, expr, src_fields, + dst_fields, indirections, precondition, pred_guard); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_issue_fill(Memoizable *memo, + ApEvent &lhs, IndexSpaceExpression *expr, + const std::vector &fields, + const void *fill_value, size_t fill_size, +#ifdef LEGION_SPY + FieldSpace handle, RegionTreeID tree_id, +#endif + ApEvent precondition, PredEvent pred_guard) + //-------------------------------------------------------------------------- + { + // Make sure the lhs event is local to our shard + if (lhs.exists()) + { + const AddressSpaceID event_space = find_event_space(lhs); + if (event_space != repl_ctx->runtime->address_space) + { + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename, lhs); + lhs = rename; + } + } + // Then do the base call + PhysicalTemplate::record_issue_fill(memo, lhs, expr, fields, + fill_value, fill_size, +#ifdef LEGION_SPY + handle, tree_id, +#endif + precondition, pred_guard); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_set_op_sync_event(ApEvent &lhs, + Memoizable *memo) + //-------------------------------------------------------------------------- + { + // Make sure the lhs event is local to our shard + if (lhs.exists()) + { + const AddressSpaceID event_space = find_event_space(lhs); + if (event_space != repl_ctx->runtime->address_space) + { + ApUserEvent rename = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, rename, lhs); + lhs = rename; + } + } + // Then do the base call + PhysicalTemplate::record_set_op_sync_event(lhs, memo); + } + + //-------------------------------------------------------------------------- + ApBarrier ShardedPhysicalTemplate::find_trace_shard_event(ApEvent event, + ShardID remote_shard) + //-------------------------------------------------------------------------- + { + AutoLock tpl_lock(template_lock); + // Check to see if we made this event + std::map::const_iterator finder = + event_map.find(event); + // If we didn't make this event then we don't do anything + if (finder == event_map.end() || (finder->second == NO_INDEX)) + return ApBarrier::NO_AP_BARRIER; + // If we did make it then see if we have a remote barrier for it yet + std::map::const_iterator barrier_finder = + remote_arrivals.find(event); + if (barrier_finder == remote_arrivals.end()) + { + // Make a new barrier and record it in the events + ApBarrier barrier(Realm::Barrier::create_barrier(1/*arrival count*/)); + // Record this in the instruction stream +#ifdef DEBUG_LEGION + const unsigned index = convert_event(barrier, false/*check*/); +#else + const unsigned index = convert_event(barrier); +#endif + // Then add a new instruction to arrive on the barrier with the + // event as a precondition + BarrierArrival *arrival_instruction = + new BarrierArrival(*this, barrier, index, finder->second); + insert_instruction(arrival_instruction); + // Save this in the remote barriers + remote_arrivals[event] = arrival_instruction; + return arrival_instruction->record_subscribed_shard(remote_shard); + } + else + return barrier_finder->second->record_subscribed_shard(remote_shard); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_trace_shard_event( + ApEvent event, ApBarrier barrier) + //-------------------------------------------------------------------------- + { + AutoLock tpl_lock(template_lock); +#ifdef DEBUG_LEGION + assert(event.exists()); + assert(event_map.find(event) == event_map.end()); +#endif + if (barrier.exists()) + { +#ifdef DEBUG_LEGION + assert(local_advances.find(event) == local_advances.end()); + const unsigned index = convert_event(event, false/*check*/); +#else + const unsigned index = convert_event(event); +#endif + BarrierAdvance *advance = new BarrierAdvance(*this, barrier, index); + insert_instruction(advance); + local_advances[event] = advance; + // Don't remove it, just set it to NO_EVENT so we can tell the names + // of the remote events that we got from other shards + // See get_completion_for_deletion for where we use this + std::map::iterator finder = + pending_event_requests.find(event); +#ifdef DEBUG_LEGION + assert(finder != pending_event_requests.end()); +#endif + finder->second = RtEvent::NO_RT_EVENT; + } + else // no barrier means it's not part of the trace + { + event_map[event] = NO_INDEX; + // In this case we can remove it since we're not tracing it +#ifdef DEBUG_LEGION + std::map::iterator finder = + pending_event_requests.find(event); + assert(finder != pending_event_requests.end()); + pending_event_requests.erase(finder); +#else + pending_event_requests.erase(event); +#endif + } + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::handle_trace_update(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + Runtime *runtime = repl_ctx->runtime; + UpdateKind kind; + derez.deserialize(kind); + RtUserEvent done; + std::set applied; + switch (kind) + { + case UPDATE_VALID_VIEWS: + { + derez.deserialize(done); + DistributedID view_did, eq_did; + derez.deserialize(view_did); + RtEvent view_ready; + InstanceView *view = static_cast( + runtime->find_or_request_logical_view(view_did, view_ready)); + derez.deserialize(eq_did); + RtEvent eq_ready; + EquivalenceSet *eq = + runtime->find_or_request_equivalence_set(eq_did, eq_ready); + if ((view_ready.exists() && !view_ready.has_triggered()) || + (eq_ready.exists() && !eq_ready.has_triggered())) + { + const RtEvent pre = Runtime::merge_events(view_ready, eq_ready); + DeferTraceUpdateArgs args(this, kind, done, derez, view, eq); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + return; + } + else if (handle_update_valid_views(view, eq, derez, applied, done)) + return; + break; + } + case UPDATE_PRE_FILL: + { + derez.deserialize(done); + DistributedID view_did; + derez.deserialize(view_did); + RtEvent view_ready; + FillView *view = static_cast( + runtime->find_or_request_logical_view(view_did, view_ready)); + if (view_ready.exists() && !view_ready.has_triggered()) + { + DeferTraceUpdateArgs args(this, kind, done, derez, view); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, view_ready); + return; + } + else if (handle_update_pre_fill(view, derez, applied, done)) + return; + break; + } + case UPDATE_POST_FILL: + { + derez.deserialize(done); + DistributedID view_did; + derez.deserialize(view_did); + RtEvent view_ready; + FillView *view = static_cast( + runtime->find_or_request_logical_view(view_did, view_ready)); + if (view_ready.exists() && !view_ready.has_triggered()) + { + DeferTraceUpdateArgs args(this, kind, done, derez, view); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, view_ready); + return; + } + else if (handle_update_post_fill(view, derez, applied, done)) + return; + break; + } + case UPDATE_VIEW_USER: + { + derez.deserialize(done); + DistributedID view_did; + derez.deserialize(view_did); + RtEvent view_ready; + InstanceView *view = static_cast( + runtime->find_or_request_logical_view(view_did, view_ready)); + bool is_local, is_index_space; + IndexSpace handle; + IndexSpaceExprID remote_expr_id; + RtEvent expr_ready; + IndexSpaceExpression *user_expr = + IndexSpaceExpression::unpack_expression(derez, runtime->forest, + source, is_local, is_index_space, handle, + remote_expr_id, expr_ready); + if ((view_ready.exists() && !view_ready.has_triggered()) || + (expr_ready.exists() && !expr_ready.has_triggered())) + { + if (user_expr != NULL) + { +#ifdef DEBUG_LEGION + assert(!expr_ready.exists() || expr_ready.has_triggered()); +#endif + DeferTraceUpdateArgs args(this, kind,done,view,derez,user_expr); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, view_ready); + } + else if (is_index_space) + { + DeferTraceUpdateArgs args(this, kind, done, view, derez,handle); + const RtEvent pre = !view_ready.exists() ? expr_ready : + Runtime::merge_events(view_ready, expr_ready); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + } + else + { + DeferTraceUpdateArgs args(this, kind, done, view, + derez, remote_expr_id); + const RtEvent pre = !view_ready.exists() ? expr_ready : + Runtime::merge_events(view_ready, expr_ready); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + } + return; + } + else if (handle_update_view_user(view, user_expr, + derez, applied, done)) + return; + break; + } + case UPDATE_LAST_USER: + { + size_t num_users; + derez.deserialize(num_users); + { + AutoLock tpl_lock(template_lock); + for (unsigned idx = 0; idx < num_users; idx++) + { + unsigned user; + derez.deserialize(user); + local_last_users.insert(user); + } + } + derez.deserialize(done); + break; + } + case FIND_LAST_USERS_REQUEST: + { + derez.deserialize(done); + DistributedID view_did; + derez.deserialize(view_did); + RtEvent view_ready; + InstanceView *view = static_cast( + runtime->find_or_request_logical_view(view_did, view_ready)); + bool is_local, is_index_space; + IndexSpace handle; + IndexSpaceExprID remote_expr_id; + RtEvent expr_ready; + IndexSpaceExpression *user_expr = + IndexSpaceExpression::unpack_expression(derez, runtime->forest, + source, is_local, is_index_space, handle, + remote_expr_id, expr_ready); + if ((view_ready.exists() && !view_ready.has_triggered()) || + (expr_ready.exists() && !expr_ready.has_triggered())) + { + if (user_expr != NULL) + { +#ifdef DEBUG_LEGION + assert(!expr_ready.exists() || expr_ready.has_triggered()); +#endif + DeferTraceUpdateArgs args(this, kind,done,view,derez,user_expr); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, view_ready); + } + else if (is_index_space) + { + DeferTraceUpdateArgs args(this, kind, done, view, derez, handle); + const RtEvent pre = !view_ready.exists() ? expr_ready : + Runtime::merge_events(view_ready, expr_ready); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + } + else + { + DeferTraceUpdateArgs args(this, kind, done, view, + derez, remote_expr_id); + const RtEvent pre = !view_ready.exists() ? expr_ready : + Runtime::merge_events(view_ready, expr_ready); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + } + return; + } + else + handle_find_last_users(view, user_expr, derez, applied); + break; + } + case FIND_LAST_USERS_RESPONSE: + { + std::set *users; + derez.deserialize(users); + derez.deserialize(done); + size_t num_barriers; + derez.deserialize(num_barriers); + { + AutoLock tpl_lock(template_lock); + for (unsigned idx = 0; idx < num_barriers; idx++) + { + unsigned event_index; + derez.deserialize(event_index); + // Check to see if we already made a frontier for this + std::map::const_iterator finder = + frontiers.find(event_index); + // See if we have recorded this frontier yet or not + if (finder == frontiers.end()) + { + const unsigned next_event_id = events.size(); + frontiers[event_index] = next_event_id; + events.resize(next_event_id + 1); + users->insert(next_event_id); + } + else + users->insert(finder->second); + } + } + break; + } + case FIND_FRONTIER_REQUEST: + { + ShardID source_shard; + derez.deserialize(source_shard); +#ifdef DEBUG_LEGION + assert(source_shard != repl_ctx->owner_shard->shard_id); +#endif + std::set *target; + derez.deserialize(target); + size_t num_events; + derez.deserialize(num_events); + std::vector result_frontiers; + { + AutoLock tpl_lock(template_lock); + for (unsigned idx = 0; idx < num_events; idx++) + { + unsigned event_index; + derez.deserialize(event_index); + // Translate this to a local frontier first + std::map::const_iterator finder = + frontiers.find(event_index); + // See if we have recorded this frontier yet or not + if (finder == frontiers.end()) + { + const unsigned next_event_id = events.size(); + frontiers[event_index] = next_event_id; + events.resize(next_event_id + 1); + finder = frontiers.find(event_index); + } + // Check to see if we have a barrier for this event yet + std::map::const_iterator barrier_finder = + local_frontiers.find(finder->second); + if (barrier_finder == local_frontiers.end()) + { + // Make a barrier and record it + const ApBarrier result( + Realm::Barrier::create_barrier(1/*arrival count*/)); + local_frontiers[finder->second] = result; + result_frontiers.push_back(result); + } + else + result_frontiers.push_back(barrier_finder->second); + // Record that this shard depends on this event + local_subscriptions[finder->second].insert(source_shard); + } + } + RtUserEvent remote_done; + derez.deserialize(remote_done); + // Send the respose back to the source shard + ShardManager *manager = repl_ctx->shard_manager; + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(source_shard); + rez.serialize(template_index); + rez.serialize(FIND_FRONTIER_RESPONSE); + rez.serialize(target); + rez.serialize(result_frontiers.size()); + for (std::vector::const_iterator it = + result_frontiers.begin(); it != result_frontiers.end(); it++) + rez.serialize(*it); + rez.serialize(remote_done); + manager->send_trace_update(source_shard, rez); + break; + } + case FIND_FRONTIER_RESPONSE: + { + std::set *users; + derez.deserialize(users); + size_t num_barriers; + derez.deserialize(num_barriers); + { + AutoLock tpl_lock(template_lock); + for (unsigned idx = 0; idx < num_barriers; idx++) + { + ApBarrier barrier; + derez.deserialize(barrier); + // Scan through and see if we already have it + bool found = false; + for (std::vector >::const_iterator + it = remote_frontiers.begin(); + it != remote_frontiers.end(); it++) + { + if (it->first != barrier) + continue; + users->insert(it->second); + found = true; + break; + } + if (!found) + { + const unsigned next_event_id = events.size(); + remote_frontiers.push_back( + std::pair(barrier, next_event_id)); + events.resize(next_event_id + 1); + users->insert(next_event_id); + } + } + } + derez.deserialize(done); + break; + } + case TEMPLATE_BARRIER_REFRESH: + { + size_t num_barriers; + derez.deserialize(num_barriers); + for (unsigned idx = 0; idx < num_barriers; idx++) + { + ApEvent key; + derez.deserialize(key); + ApBarrier bar; + derez.deserialize(bar); + std::map::const_iterator finder = + local_advances.find(key); +#ifdef DEBUG_LEGION + assert(finder != local_advances.end()); +#endif + finder->second->refresh_barrier(bar); + } + { + AutoLock tpl_lock(template_lock); + updated_advances += num_barriers; +#ifdef DEBUG_LEGION + assert(updated_advances <= local_advances.size()); +#endif + // See if the wait has already been done by the local shard + // If so, trigger it, otherwise do nothing so it can come + // along and see that everything is done + if ((updated_advances == local_advances.size()) && + update_advances_ready.exists()) + { + done = update_advances_ready; + // We're done so reset everything for the next refresh + update_advances_ready = RtUserEvent::NO_RT_USER_EVENT; + updated_advances = 0; + } + } + break; + } + case FRONTIER_BARRIER_REFRESH: + { + size_t num_barriers; + derez.deserialize(num_barriers); + { + AutoLock tpl_lock(template_lock); + for (unsigned idx = 0; idx < num_barriers; idx++) + { + ApBarrier oldbar, newbar; + derez.deserialize(oldbar); + derez.deserialize(newbar); +#ifdef DEBUG_LEGION + bool found = false; +#endif + for (std::vector >::iterator it = + remote_frontiers.begin(); it != + remote_frontiers.end(); it++) + { + if (it->first != oldbar) + continue; + it->first = newbar; +#ifdef DEBUG_LEGION + found = true; +#endif + break; + } +#ifdef DEBUG_LEGION + assert(found); +#endif + } + updated_frontiers += num_barriers; +#ifdef DEBUG_LEGION + assert(updated_frontiers <= remote_frontiers.size()); +#endif + if ((updated_frontiers == remote_frontiers.size()) && + update_frontiers_ready.exists()) + { + done = update_frontiers_ready; + // We're done so reset everything for the next stage + update_frontiers_ready = RtUserEvent::NO_RT_USER_EVENT; + updated_frontiers = 0; + } + } + break; + } + default: + assert(false); + } + if (done.exists()) + { + if (!applied.empty()) + Runtime::trigger_event(done, Runtime::merge_events(applied)); + else + Runtime::trigger_event(done); + } + } + + //-------------------------------------------------------------------------- + ShardedPhysicalTemplate::DeferTraceUpdateArgs::DeferTraceUpdateArgs( + ShardedPhysicalTemplate *t, UpdateKind k, RtUserEvent d, + Deserializer &derez, LogicalView *v, EquivalenceSet *q, RtUserEvent u) + : LgTaskArgs(implicit_provenance), target(t), + kind(k), done(d), view(v), eq(q), expr(NULL), remote_expr_id(0), + buffer_size(derez.get_remaining_bytes()), buffer(malloc(buffer_size)), + deferral_event(u) + //-------------------------------------------------------------------------- + { + memcpy(buffer, derez.get_current_pointer(), buffer_size); + derez.advance_pointer(buffer_size); + } + + //-------------------------------------------------------------------------- + ShardedPhysicalTemplate::DeferTraceUpdateArgs::DeferTraceUpdateArgs( + ShardedPhysicalTemplate *t, UpdateKind k, RtUserEvent d, LogicalView *v, + Deserializer &derez, IndexSpaceExpression *x, RtUserEvent u) + : LgTaskArgs(implicit_provenance), target(t), + kind(k), done(d), view(v), eq(NULL), expr(x), remote_expr_id(0), + buffer_size(derez.get_remaining_bytes()), buffer(malloc(buffer_size)), + deferral_event(u) + //-------------------------------------------------------------------------- + { + memcpy(buffer, derez.get_current_pointer(), buffer_size); + derez.advance_pointer(buffer_size); + } + + //-------------------------------------------------------------------------- + ShardedPhysicalTemplate::DeferTraceUpdateArgs::DeferTraceUpdateArgs( + ShardedPhysicalTemplate *t, UpdateKind k, RtUserEvent d, + LogicalView *v, Deserializer &derez, IndexSpace h) + : LgTaskArgs(implicit_provenance), target(t), + kind(k), done(d), view(v), eq(NULL), expr(NULL), remote_expr_id(0), + handle(h), buffer_size(derez.get_remaining_bytes()), + buffer(malloc(buffer_size)) + //-------------------------------------------------------------------------- + { + memcpy(buffer, derez.get_current_pointer(), buffer_size); + derez.advance_pointer(buffer_size); + } + + //-------------------------------------------------------------------------- + ShardedPhysicalTemplate::DeferTraceUpdateArgs::DeferTraceUpdateArgs( + ShardedPhysicalTemplate *t, UpdateKind k, RtUserEvent d, + LogicalView *v, Deserializer &derez, IndexSpaceExprID x) + : LgTaskArgs(implicit_provenance), target(t), + kind(k), done(d), view(v), eq(NULL), expr(NULL), remote_expr_id(x), + buffer_size(derez.get_remaining_bytes()), buffer(malloc(buffer_size)) + //-------------------------------------------------------------------------- + { + memcpy(buffer, derez.get_current_pointer(), buffer_size); + derez.advance_pointer(buffer_size); + } + + //-------------------------------------------------------------------------- + ShardedPhysicalTemplate::DeferTraceUpdateArgs::DeferTraceUpdateArgs( + const DeferTraceUpdateArgs &rhs, RtUserEvent d) + : LgTaskArgs(rhs.provenance), target(rhs.target), + kind(rhs.kind), done(rhs.done), view(rhs.view), eq(rhs.eq), + expr(rhs.expr), remote_expr_id(rhs.remote_expr_id), handle(rhs.handle), + buffer_size(rhs.buffer_size), buffer(rhs.buffer), deferral_event(d) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardedPhysicalTemplate::handle_deferred_trace_update( + const void *args, Runtime *runtime) + //-------------------------------------------------------------------------- + { + const DeferTraceUpdateArgs *dargs = (const DeferTraceUpdateArgs*)args; + std::set applied; + Deserializer derez(dargs->buffer, dargs->buffer_size); + switch (dargs->kind) + { + case UPDATE_VALID_VIEWS: + { + if (dargs->target->handle_update_valid_views( + static_cast(dargs->view), + dargs->eq, derez, applied, dargs->done, dargs)) + return; + break; + } + case UPDATE_PRE_FILL: + { + if (dargs->target->handle_update_pre_fill( + static_cast(dargs->view), derez, applied, + dargs->done, dargs)) + return; + break; + } + case UPDATE_POST_FILL: + { + if (dargs->target->handle_update_post_fill( + static_cast(dargs->view), derez, applied, + dargs->done, dargs)) + return; + break; + } + case UPDATE_VIEW_USER: + { + if (dargs->expr != NULL) + { + if (dargs->target->handle_update_view_user( + static_cast(dargs->view), + dargs->expr, derez, applied, dargs->done, dargs)) + return; + } + else if (dargs->handle.exists()) + { + IndexSpaceNode *node = runtime->forest->get_node(dargs->handle); + if (dargs->target->handle_update_view_user( + static_cast(dargs->view), node, derez, + applied, dargs->done, dargs)) + return; + } + else + { + IndexSpaceExpression *expr = + runtime->forest->find_remote_expression(dargs->remote_expr_id); + if (dargs->target->handle_update_view_user( + static_cast(dargs->view), expr, derez, + applied, dargs->done, dargs)) + return; + } + break; + } + case FIND_LAST_USERS_REQUEST: + { + if (dargs->expr != NULL) + { + dargs->target->handle_find_last_users( + static_cast(dargs->view), + dargs->expr, derez, applied); + } + else if (dargs->handle.exists()) + { + IndexSpaceNode *node = runtime->forest->get_node(dargs->handle); + dargs->target->handle_find_last_users( + static_cast(dargs->view), node, derez,applied); + } + else + { + IndexSpaceExpression *expr = + runtime->forest->find_remote_expression(dargs->remote_expr_id); + dargs->target->handle_find_last_users( + static_cast(dargs->view), expr, derez,applied); + } + break; + } + default: + assert(false); // should never get here + } +#ifdef DEBUG_LEGION + assert(dargs->done.exists()); +#endif + if (!applied.empty()) + Runtime::trigger_event(dargs->done, Runtime::merge_events(applied)); + else + Runtime::trigger_event(dargs->done); + if (dargs->deferral_event.exists()) + Runtime::trigger_event(dargs->deferral_event); + free(dargs->buffer); + } + + //-------------------------------------------------------------------------- + bool ShardedPhysicalTemplate::handle_update_valid_views(InstanceView *view, + EquivalenceSet *eq, Deserializer &derez, std::set &applied, + RtUserEvent done, const DeferTraceUpdateArgs *dargs /*=NULL*/) + //-------------------------------------------------------------------------- + { + AutoTryLock tpl_lock(template_lock); + if (!tpl_lock.has_lock()) + { + RtUserEvent deferral; + if (dargs != NULL) + deferral = dargs->deferral_event; + RtEvent pre; + if (!deferral.exists()) + { + deferral = Runtime::create_rt_user_event(); + pre = chain_deferral_events(deferral); + } + else + pre = tpl_lock.try_next(); + if (dargs == NULL) + { + DeferTraceUpdateArgs args(this, UPDATE_VALID_VIEWS, done, + derez, view, eq, deferral); + repl_ctx->runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + } + else + { + DeferTraceUpdateArgs args(*dargs, deferral); + repl_ctx->runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); +#ifdef DEBUG_LEGION + // Keep the deserializer happy since we didn't use it + derez.advance_pointer(derez.get_remaining_bytes()); +#endif + } + return true; + } + RegionUsage usage; + derez.deserialize(usage); + FieldMask user_mask; + derez.deserialize(user_mask); + bool invalidates; + derez.deserialize(invalidates); + PhysicalTemplate::update_valid_views(view, eq, usage, user_mask, + invalidates, applied); + return false; + } + + //-------------------------------------------------------------------------- + bool ShardedPhysicalTemplate::handle_update_pre_fill(FillView *view, + Deserializer &derez, std::set &applied, + RtUserEvent done, const DeferTraceUpdateArgs *dargs) + //-------------------------------------------------------------------------- + { + AutoTryLock tpl_lock(template_lock); + if (!tpl_lock.has_lock()) + { + RtUserEvent deferral; + if (dargs != NULL) + deferral = dargs->deferral_event; + RtEvent pre; + if (!deferral.exists()) + { + deferral = Runtime::create_rt_user_event(); + pre = chain_deferral_events(deferral); + } + else + pre = tpl_lock.try_next(); + if (dargs == NULL) + { + DeferTraceUpdateArgs args(this, UPDATE_PRE_FILL, done, + derez, view, NULL, deferral); + repl_ctx->runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + } + else + { + DeferTraceUpdateArgs args(*dargs, deferral); + repl_ctx->runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); +#ifdef DEBUG_LEGION + // Keep the deserializer happy since we didn't use it + derez.advance_pointer(derez.get_remaining_bytes()); +#endif + } + return true; + } + FieldMask view_mask; + derez.deserialize(view_mask); + FieldMaskSet views; + views.insert(view, view_mask); + PhysicalTemplate::record_fill_views(views, applied); + return false; + } + + //-------------------------------------------------------------------------- + bool ShardedPhysicalTemplate::handle_update_post_fill(FillView *view, + Deserializer &derez, std::set &applied, + RtUserEvent done, const DeferTraceUpdateArgs *dargs) + //-------------------------------------------------------------------------- + { + AutoTryLock tpl_lock(template_lock); + if (!tpl_lock.has_lock()) + { + RtUserEvent deferral; + if (dargs != NULL) + deferral = dargs->deferral_event; + RtEvent pre; + if (!deferral.exists()) + { + deferral = Runtime::create_rt_user_event(); + pre = chain_deferral_events(deferral); + } + else + pre = tpl_lock.try_next(); + if (dargs == NULL) + { + DeferTraceUpdateArgs args(this, UPDATE_POST_FILL, done, + derez, view, NULL, deferral); + repl_ctx->runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + } + else + { + DeferTraceUpdateArgs args(*dargs, deferral); + repl_ctx->runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); +#ifdef DEBUG_LEGION + // Keep the deserializer happy since we didn't use it + derez.advance_pointer(derez.get_remaining_bytes()); +#endif + } + return true; + } + FieldMask view_mask; + derez.deserialize(view_mask); +#ifdef DEBUG_LEGION + assert(is_recording()); +#endif + post_fill_views.insert(view, view_mask); + return false; + } + + //-------------------------------------------------------------------------- + bool ShardedPhysicalTemplate::handle_update_view_user(InstanceView *view, + IndexSpaceExpression *user_expr, + Deserializer &derez, + std::set &applied, + RtUserEvent done, + const DeferTraceUpdateArgs *dargs) + //-------------------------------------------------------------------------- + { + AutoTryLock tpl_lock(template_lock); + if (!tpl_lock.has_lock()) + { + RtUserEvent deferral; + if (dargs != NULL) + deferral = dargs->deferral_event; + RtEvent pre; + if (!deferral.exists()) + { + deferral = Runtime::create_rt_user_event(); + pre = chain_deferral_events(deferral); + } + else + pre = tpl_lock.try_next(); + if (dargs == NULL) + { + DeferTraceUpdateArgs args(this, UPDATE_VIEW_USER, done, view, + derez, user_expr, deferral); + repl_ctx->runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); + } + else + { + DeferTraceUpdateArgs args(*dargs, deferral); + repl_ctx->runtime->issue_runtime_meta_task(args, + LG_LATENCY_MESSAGE_PRIORITY, pre); +#ifdef DEBUG_LEGION + // Keep the deserializer happy since we didn't use it + derez.advance_pointer(derez.get_remaining_bytes()); +#endif + } + return true; + } + RegionUsage usage; + derez.deserialize(usage); + unsigned user_index; + derez.deserialize(user_index); + FieldMask user_mask; + derez.deserialize(user_mask); + int owner_shard; + derez.deserialize(owner_shard); + PhysicalTemplate::add_view_user(view, usage, user_index, user_expr, + user_mask, applied, owner_shard); + return false; + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::handle_find_last_users(InstanceView *view, + IndexSpaceExpression *user_expr, + Deserializer &derez, + std::set &applied) + //-------------------------------------------------------------------------- + { + FieldMask user_mask; + derez.deserialize(user_mask); + ShardID source_shard; + derez.deserialize(source_shard); + std::set *target; + derez.deserialize(target); + // This is a local operation and all the data structures are + // read-only for this part so there is no need for the lock yet + std::set > sharded_users; + find_last_users_sharded(view, user_expr, user_mask, sharded_users); + // Sort these into where they should go + std::map > requests; + for (std::set >::const_iterator it = + sharded_users.begin(); it != sharded_users.end(); it++) + requests[it->second].push_back(it->first); + // Send out the requests/responses + ShardManager *manager = repl_ctx->shard_manager; + const ShardID local_shard = repl_ctx->owner_shard->shard_id; + for (std::map >::const_iterator rit = + requests.begin(); rit != requests.end(); rit++) + { + RtUserEvent remote_done = Runtime::create_rt_user_event(); + if (rit->first == source_shard) + { + // Special case for sending values directly back to the user + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(source_shard); + rez.serialize(template_index); + rez.serialize(FIND_LAST_USERS_RESPONSE); + rez.serialize(target); + rez.serialize(remote_done); + rez.serialize(rit->second.size()); + for (std::vector::const_iterator it = + rit->second.begin(); it != rit->second.end(); it++) + rez.serialize(*it); + manager->send_trace_update(source_shard, rez); + } + else if (rit->first == local_shard) + { + // Special case for ourselves so we can return the result as + // though we handled the remote frontier request + std::vector result_frontiers; + { + AutoLock tpl_lock(template_lock); + for (std::vector::const_iterator it = + rit->second.begin(); it != rit->second.end(); it++) + { + // These events have already been translated to frontiers + // so we just need to look up the local frontiers + // Check to see if we have a barrier for this event yet + std::map::const_iterator finder = + local_frontiers.find(*it); + if (finder == local_frontiers.end()) + { + // Make a barrier and record it + const ApBarrier result( + Realm::Barrier::create_barrier(1/*arrival count*/)); + local_frontiers[*it] = result; + result_frontiers.push_back(result); + } + else + result_frontiers.push_back(finder->second); + // Record that this shard depends on this event + local_subscriptions[*it].insert(source_shard); + } + } + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(source_shard); + rez.serialize(template_index); + rez.serialize(FIND_FRONTIER_RESPONSE); + rez.serialize(target); + rez.serialize(result_frontiers.size()); + for (std::vector::const_iterator it = + result_frontiers.begin(); it != + result_frontiers.end(); it++) + rez.serialize(*it); + rez.serialize(remote_done); + manager->send_trace_update(source_shard, rez); + } + else + { + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(rit->first); + rez.serialize(template_index); + rez.serialize(FIND_FRONTIER_REQUEST); + rez.serialize(source_shard); + rez.serialize(target); + rez.serialize(rit->second.size()); + for (std::vector::const_iterator it = + rit->second.begin(); it != rit->second.end(); it++) + rez.serialize(*it); + rez.serialize(remote_done); + manager->send_trace_update(rit->first, rez); + } + applied.insert(remote_done); + } + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::request_remote_shard_event(ApEvent event, + RtUserEvent done_event) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(event.exists()); +#endif + const AddressSpaceID event_space = find_event_space(event); + repl_ctx->shard_manager->send_trace_event_request(this, + repl_ctx->owner_shard->shard_id, repl_ctx->runtime->address_space, + template_index, event, event_space, done_event); + } + + //-------------------------------------------------------------------------- + /*static*/ AddressSpaceID ShardedPhysicalTemplate::find_event_space( + ApEvent event) + //-------------------------------------------------------------------------- + { + // TODO: Remove hack include at top of file when we fix this + return Realm::ID(event.id).event_creator_node(); + } + + //-------------------------------------------------------------------------- + PhysicalTemplate::Replayable ShardedPhysicalTemplate::check_replayable( + ReplTraceOp *op, bool has_blocking_call) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(op != NULL); +#endif + // We need everyone else to be done capturing their traces + // before we can do our own replayable check + op->sync_for_replayable_check(); + // Do the base call first to determine if our local shard is replayable + const Replayable result = + PhysicalTemplate::check_replayable(op, has_blocking_call); + if (result) + { + // One extra step to do here, since we sharded the view_users we + // need to send them back to the owner shards so that we can do + // the right thing for any calls to the get_completion + // Note we do this before the exchange so that we can use the + // exchange as a barrier for everyone being done with the exchange + // In some cases we might do some unnecessary extra work, but its + // only for non-replayable traces so it should be minimal + std::map > remote_last_users; + const ShardID local_shard = repl_ctx->owner_shard->shard_id; + for (ViewUsers::const_iterator vit = view_users.begin(); + vit != view_users.end(); vit++) + { + for (FieldMaskSet::const_iterator it = + vit->second.begin(); it != vit->second.end(); it++) + if (it->first->shard != local_shard) + remote_last_users[it->first->shard].insert(it->first->user); + } + if (!remote_last_users.empty()) + { + std::set done_events; + ShardManager *manager = repl_ctx->shard_manager; + for (std::map >::const_iterator sit = + remote_last_users.begin(); sit != + remote_last_users.end(); sit++) + { + RtUserEvent done = Runtime::create_rt_user_event(); + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(sit->first); + rez.serialize(template_index); + rez.serialize(UPDATE_LAST_USER); + rez.serialize(sit->second.size()); + for (std::set::const_iterator it = + sit->second.begin(); it != sit->second.end(); it++) + rez.serialize(*it); + rez.serialize(done); + manager->send_trace_update(sit->first, rez); + } + const RtEvent wait_on = Runtime::merge_events(done_events); + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + } + // Now we can do the exchange + if (op->exchange_replayable(repl_ctx, true/*replayable*/)) + return result; + else + return Replayable(false, "Remote shard not replyable"); + } + else + { + // Still need to do the exchange + op->exchange_replayable(repl_ctx, false/*replayable*/); + return result; + } + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_replayed(void) + //-------------------------------------------------------------------------- + { + if (total_replays++ == Realm::Barrier::MAX_PHASES) + { + std::map > notifications; + // Need to update all our barriers since we're out of generations + for (std::map::const_iterator it = + remote_arrivals.begin(); it != remote_arrivals.end(); it++) + it->second->refresh_barrier(it->first, notifications); + // Send out the notifications to all the shards + ShardManager *manager = repl_ctx->shard_manager; + for (std::map >::const_iterator + nit = notifications.begin(); nit != notifications.end(); nit++) + { +#ifdef DEBUG_LEGION + assert(nit->first != repl_ctx->owner_shard->shard_id); +#endif + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(nit->first); + rez.serialize(template_index); + rez.serialize(TEMPLATE_BARRIER_REFRESH); + rez.serialize(nit->second.size()); + for (std::map::const_iterator it = + nit->second.begin(); it != nit->second.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + manager->send_trace_update(nit->first, rez); + } + // Then wait for all our advances to be updated from other shards + RtEvent wait_on; + { + AutoLock tpl_lock(template_lock); + if (updated_advances < local_advances.size()) + { + update_advances_ready = Runtime::create_rt_user_event(); + wait_on = update_advances_ready; + } + else // Reset this back to zero for the next round + updated_advances = 0; + } + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + // Reset it back to zero after updating our barriers + total_replays = 0; + } + } + + //-------------------------------------------------------------------------- + ApEvent ShardedPhysicalTemplate::get_completion(void) const + //-------------------------------------------------------------------------- + { + std::set to_merge; + const ShardID local_shard = repl_ctx->owner_shard->shard_id; + for (ViewUsers::const_iterator it = view_users.begin(); + it != view_users.end(); ++it) + for (FieldMaskSet::const_iterator uit = it->second.begin(); + uit != it->second.end(); ++uit) + // Check to see if this is a user from our shard + if (uit->first->shard == local_shard) + to_merge.insert(events[uit->first->user]); + // Also get any events for users that are sharded to remote shards + // but which originated on this node + for (std::set::const_iterator it = + local_last_users.begin(); it != local_last_users.end(); it++) + to_merge.insert(events[*it]); + return Runtime::merge_events(NULL, to_merge); + } + + //-------------------------------------------------------------------------- + ApEvent ShardedPhysicalTemplate::get_completion_for_deletion(void) const + //-------------------------------------------------------------------------- + { + // Skip the any events that are from remote shards since we + std::set all_events; + std::set local_barriers; + for (std::map::const_iterator it = + remote_arrivals.begin(); it != remote_arrivals.end(); it++) + local_barriers.insert(it->second->get_current_barrier()); + for (std::map::const_iterator it = event_map.begin(); + it != event_map.end(); ++it) + { + // If this is a remote event or one of our barriers then don't use it + if ((local_barriers.find(it->first) == local_barriers.end()) && + (pending_event_requests.find(it->first) == + pending_event_requests.end())) + all_events.insert(it->first); + } + return Runtime::merge_events(NULL, all_events); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::update_valid_views(InstanceView *view, + EquivalenceSet *eq, + const RegionUsage &usage, + const FieldMask &user_mask, + bool invalidates, + std::set &applied) + //-------------------------------------------------------------------------- + { + const ShardID target_shard = find_equivalence_owner(eq); + // Check to see if we're on the right shard, if not send the message + if (target_shard != repl_ctx->owner_shard->shard_id) + { + RtUserEvent done = Runtime::create_rt_user_event(); + Serializer rez; + rez.serialize(repl_ctx->shard_manager->repl_id); + rez.serialize(target_shard); + rez.serialize(template_index); + rez.serialize(UPDATE_VALID_VIEWS); + rez.serialize(done); + rez.serialize(view->did); + rez.serialize(eq->did); + rez.serialize(usage); + rez.serialize(user_mask); + rez.serialize(invalidates); + repl_ctx->shard_manager->send_trace_update(target_shard, rez); + applied.insert(done); + } + else // Now that we are on the right shard we can do the update call + PhysicalTemplate::update_valid_views(view, eq, usage, user_mask, + invalidates, applied); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::add_view_user(InstanceView *view, + const RegionUsage &usage, + unsigned user_index, + IndexSpaceExpression *user_expr, + const FieldMask &user_mask, + std::set &applied, + int owner_shard) + //-------------------------------------------------------------------------- + { + const ShardID target_shard = find_view_owner(view); + // Check to see if we're on the right shard, if not send the message + if (target_shard != repl_ctx->owner_shard->shard_id) + { + RtUserEvent done = Runtime::create_rt_user_event(); + ShardManager *manager = repl_ctx->shard_manager; + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(target_shard); + rez.serialize(template_index); + rez.serialize(UPDATE_VIEW_USER); + rez.serialize(done); + rez.serialize(view->did); + user_expr->pack_expression(rez, manager->get_shard_space(target_shard)); + rez.serialize(usage); + rez.serialize(user_index); + rez.serialize(user_mask); +#ifdef DEBUG_LEGION + assert(owner_shard < 0); // shouldn't have set this yet +#endif + rez.serialize(repl_ctx->owner_shard->shard_id); + manager->send_trace_update(target_shard, rez); + applied.insert(done); + } + else if (owner_shard < 0) + PhysicalTemplate::add_view_user(view, usage, user_index, user_expr, + user_mask, applied, repl_ctx->owner_shard->shard_id); + else + PhysicalTemplate::add_view_user(view, usage, user_index, user_expr, + user_mask, applied, owner_shard); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_fill_views( + const FieldMaskSet &views, std::set &applied_events) + //-------------------------------------------------------------------------- + { + FieldMaskSet local_set; + for (FieldMaskSet::const_iterator it = + views.begin(); it != views.end(); it++) + { + // Figure out which shard these fill views should be stored on + // using the same algorithm that we use for other views above + const AddressSpaceID view_owner = it->first->owner_space; + std::vector owner_shards; + find_owner_shards(view_owner, owner_shards); +#ifdef DEBUG_LEGION + assert(!owner_shards.empty()); +#endif + // For now just send all views to the first shard on each node + const ShardID target_shard = owner_shards.front(); + if (target_shard != repl_ctx->owner_shard->shard_id) + { + RtUserEvent applied = Runtime::create_rt_user_event(); + Serializer rez; + rez.serialize(repl_ctx->shard_manager->repl_id); + rez.serialize(target_shard); + rez.serialize(template_index); + rez.serialize(UPDATE_PRE_FILL); + rez.serialize(applied); + rez.serialize(it->first->did); + rez.serialize(it->second); + repl_ctx->shard_manager->send_trace_update(target_shard,rez); + applied_events.insert(applied); + } + else + local_set.insert(it->first, it->second); + } + if (!local_set.empty()) + PhysicalTemplate::record_fill_views(local_set, applied_events); + } + + //-------------------------------------------------------------------------- + ShardID ShardedPhysicalTemplate::find_view_owner(InstanceView *view) + //-------------------------------------------------------------------------- + { + // Figure out where the owner for this view is and then send it to + // the appropriate shard trace. The algorithm we use for determining + // the right shard trace is to send a view to a shard trace on the node + // that owns the instance. If there is no shard on that node we + // round-robin views based on their owner node mod the number of nodes + // where there are shards. Once on the correct node, then we pick the + // shard corresponding to their tree_id mod the number of shards on + // that node. This algorithm guarantees that all the related instances + // end up on the same shard for analysis to determine if the trace is + // replayable or not. + PhysicalManager *manager = view->get_manager(); + const AddressSpaceID inst_owner = manager->owner_space; + std::vector owner_shards; + find_owner_shards(inst_owner, owner_shards); +#ifdef DEBUG_LEGION + assert(!owner_shards.empty()); +#endif + // Figure out which shard we should be sending this view to based on + // its tree ID + if (owner_shards.size() > 1) + { + const RegionTreeID tid = manager->tree_id; + return owner_shards[tid % owner_shards.size()]; + } + else // If there's only one shard then there is only one choice + return owner_shards.front(); + } + + //-------------------------------------------------------------------------- + ShardID ShardedPhysicalTemplate::find_equivalence_owner(EquivalenceSet *eq) + //-------------------------------------------------------------------------- + { + // This algorithm is the same as for views, except we do it based + // on the equivalence set owner + const AddressSpaceID eq_owner = eq->owner_space; + std::vector owner_shards; + find_owner_shards(eq_owner, owner_shards); +#ifdef DEBUG_LEGION + assert(!owner_shards.empty()); +#endif + // Figure out which shard we should be sending this view to based on + // its set expression + if (owner_shards.size() > 1) + { + const IndexSpaceExprID eid = eq->set_expr->expr_id; + return owner_shards[eid % owner_shards.size()]; + } + else // If there's only one shard then there is only one choice + return owner_shards.front(); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::find_owner_shards(AddressSpaceID owner, + std::vector &shards) + //-------------------------------------------------------------------------- + { + // See if we already computed it or not + std::map >::const_iterator finder = + did_shard_owners.find(owner); + if (finder != did_shard_owners.end()) + { + shards = finder->second; + return; + } + // If we haven't computed it yet, then we need to do that now + const ShardMapping &shard_spaces = repl_ctx->shard_manager->get_mapping(); + for (unsigned idx = 0; idx < shard_spaces.size(); idx++) + if (shard_spaces[idx] == owner) + shards.push_back(idx); + // If we didn't find any then take the owner mod the number of total + // spaces and then send it to the shards on that space + if (shards.empty()) + { + std::set unique_spaces; + for (unsigned idx = 0; idx < shard_spaces.size(); idx++) + unique_spaces.insert(shard_spaces[idx]); + const unsigned count = owner % unique_spaces.size(); + std::set::const_iterator target_space = + unique_spaces.begin(); + for (unsigned idx = 0; idx < count; idx++) + target_space++; + for (unsigned idx = 0; idx < shard_spaces.size(); idx++) + if (shard_spaces[idx] == *target_space) + shards.push_back(idx); + } +#ifdef DEBUG_LEGION + assert(!shards.empty()); +#endif + // Save the result so we don't have to do this again for this space + did_shard_owners[owner] = shards; + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_owner_shard(unsigned tid,ShardID owner) + //-------------------------------------------------------------------------- + { + AutoLock tpl_lock(template_lock); +#ifdef DEBUG_LEGION + assert(owner_shards.find(tid) == owner_shards.end()); +#endif + owner_shards[tid] = owner; + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_local_space(unsigned tid,IndexSpace sp) + //-------------------------------------------------------------------------- + { + AutoLock tpl_lock(template_lock); +#ifdef DEBUG_LEGION + assert(local_spaces.find(tid) == local_spaces.end()); +#endif + local_spaces[tid] = sp; + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::record_sharding_function(unsigned tid, + ShardingFunction *function) + //-------------------------------------------------------------------------- + { + AutoLock tpl_lock(template_lock); +#ifdef DEBUG_LEGION + assert(sharding_functions.find(tid) == sharding_functions.end()); +#endif + sharding_functions[tid] = function; + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::issue_summary_operations( + InnerContext *context, Operation *invalidator) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + ReplicateContext *repl_ctx = dynamic_cast(context); + assert(repl_ctx != NULL); +#else + ReplicateContext *repl_ctx = static_cast(context); +#endif + ReplTraceSummaryOp *op = trace->runtime->get_available_repl_summary_op(); + op->initialize_summary(repl_ctx, this, invalidator); +#ifdef LEGION_SPY + LegionSpy::log_summary_op_creator(op->get_unique_op_id(), + invalidator->get_unique_op_id()); +#endif + op->execute_dependence_analysis(); + } + + //-------------------------------------------------------------------------- + ShardID ShardedPhysicalTemplate::find_owner_shard(unsigned tid) + //-------------------------------------------------------------------------- + { + AutoLock tpl_lock(template_lock); +#ifdef DEBUG_LEGION + std::map::const_iterator finder = + owner_shards.find(tid); + assert(finder != owner_shards.end()); + return finder->second; +#else + return owner_shards[tid]; +#endif + } + + //-------------------------------------------------------------------------- + IndexSpace ShardedPhysicalTemplate::find_local_space(unsigned tid) + //-------------------------------------------------------------------------- + { + AutoLock tpl_lock(template_lock); +#ifdef DEBUG_LEGION + std::map::const_iterator finder = + local_spaces.find(tid); + assert(finder != local_spaces.end()); + return finder->second; +#else + return local_spaces[tid]; +#endif + } + + //-------------------------------------------------------------------------- + ShardingFunction* ShardedPhysicalTemplate::find_sharding_function( + unsigned tid) + //-------------------------------------------------------------------------- + { + AutoLock tpl_lock(template_lock); +#ifdef DEBUG_LEGION + std::map::const_iterator finder = + sharding_functions.find(tid); + assert(finder != sharding_functions.end()); + return finder->second; +#else + return sharding_functions[tid]; +#endif + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::trigger_recording_done(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!recording_barrier.has_triggered()); +#endif + Runtime::phase_barrier_arrive(recording_barrier, 1/*count*/); + Runtime::trigger_event(recording_done, recording_barrier); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::elide_fences_pre_sync(ReplTraceOp *op) + //-------------------------------------------------------------------------- + { + op->elide_fences_pre_sync(); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::elide_fences_post_sync(ReplTraceOp *op) + //-------------------------------------------------------------------------- + { + op->elide_fences_post_sync(); + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::find_last_users(InstanceView *view, + IndexSpaceExpression *expr, + const FieldMask &mask, + std::set &users, + std::set &ready_events) + //-------------------------------------------------------------------------- + { + if (expr->is_empty()) return; + + // Check to see if we own this view, if we do then we can handle this + // analysis locally, otherwise we'll need to message the owner + const ShardID owner_shard = find_view_owner(view); + const ShardID local_shard = repl_ctx->owner_shard->shard_id; + if (owner_shard != local_shard) + { + RtUserEvent done = Runtime::create_rt_user_event(); + ShardManager *manager = repl_ctx->shard_manager; + // This is the remote case, send a message to find the remote users + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(owner_shard); + rez.serialize(template_index); + rez.serialize(FIND_LAST_USERS_REQUEST); + rez.serialize(done); + rez.serialize(view->did); + expr->pack_expression(rez, manager->get_shard_space(owner_shard)); + rez.serialize(mask); + rez.serialize(repl_ctx->owner_shard->shard_id); + rez.serialize(&users); + manager->send_trace_update(owner_shard, rez); + ready_events.insert(done); + } + else + { + std::set > sharded_users; + find_last_users_sharded(view, expr, mask, sharded_users); + std::map > remote_requests; + for (std::set >::const_iterator it = + sharded_users.begin(); it != sharded_users.end(); it++) + { + if (it->second == local_shard) + { + // Need the lock to prevent races on return values + AutoLock tpl_lock(template_lock); + users.insert(it->first); + } + else + remote_requests[it->second].push_back(it->first); + } + // If we have any remote requests then send them now + if (!remote_requests.empty()) + { + ShardManager *manager = repl_ctx->shard_manager; + for (std::map >::const_iterator rit = + remote_requests.begin(); rit != remote_requests.end(); rit++) + { + RtUserEvent done = Runtime::create_rt_user_event(); + Serializer rez; + rez.serialize(manager->repl_id); + rez.serialize(rit->first); + rez.serialize(template_index); + rez.serialize(FIND_FRONTIER_REQUEST); + rez.serialize(local_shard); + rez.serialize(&users); + rez.serialize(rit->second.size()); + for (std::vector::const_iterator it = + rit->second.begin(); it != rit->second.end(); it++) + rez.serialize(*it); + rez.serialize(done); + manager->send_trace_update(rit->first, rez); + ready_events.insert(done); + } + } + } + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::find_last_users_sharded(InstanceView *view, + IndexSpaceExpression *expr, + const FieldMask &mask, + std::set > &sharded_users) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + // We should own this view if we are here + assert(find_view_owner(view) == repl_ctx->owner_shard->shard_id); +#endif + ViewUsers::const_iterator finder = view_users.find(view); + if (finder == view_users.end()) return; + + RegionTreeForest *forest = trace->runtime->forest; + const ShardID local_shard = repl_ctx->owner_shard->shard_id; + for (FieldMaskSet::const_iterator uit = + finder->second.begin(); uit != finder->second.end(); ++uit) + if (!!(uit->second & mask)) + { + ViewUser *user = uit->first; + IndexSpaceExpression *intersect = + forest->intersect_index_spaces(expr, user->expr); + if (!intersect->is_empty()) + { + // See if it is local or not + if (user->shard == local_shard) + { + // This is a local user so we can do the translation now + AutoLock tpl_lock(template_lock); + std::map::const_iterator finder = + frontiers.find(user->user); + // See if we have recorded this frontier yet or not + if (finder == frontiers.end()) + { + const unsigned next_event_id = events.size(); + frontiers[user->user] = next_event_id; + events.resize(next_event_id + 1); + sharded_users.insert( + std::pair(next_event_id, local_shard)); + } + else + sharded_users.insert( + std::pair(finder->second, local_shard)); + } + else // Not local so just record it + sharded_users.insert( + std::pair(user->user, user->shard)); + } + } + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::initialize_generators( + std::vector &new_gen) + //-------------------------------------------------------------------------- + { + PhysicalTemplate::initialize_generators(new_gen); + for (std::vector >::const_iterator it = + remote_frontiers.begin(); it != remote_frontiers.end(); it++) + new_gen[it->second] = 0; + } + + //-------------------------------------------------------------------------- + void ShardedPhysicalTemplate::initialize_transitive_reduction_frontiers( + std::vector &topo_order, std::vector &inv_topo_order) + //-------------------------------------------------------------------------- + { + PhysicalTemplate::initialize_transitive_reduction_frontiers(topo_order, + inv_topo_order); + for (std::vector >::const_iterator it = + remote_frontiers.begin(); it != remote_frontiers.end(); it++) + { + inv_topo_order[it->second] = topo_order.size(); + topo_order.push_back(it->second); + } + } + + ///////////////////////////////////////////////////////////// + // Instruction + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + Instruction::Instruction(PhysicalTemplate& tpl, const TraceLocalID &o) + : operations(tpl.operations), events(tpl.events), + user_events(tpl.user_events), owner(o) + //-------------------------------------------------------------------------- + { + } + + ///////////////////////////////////////////////////////////// + // GetTermEvent + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + GetTermEvent::GetTermEvent(PhysicalTemplate& tpl, unsigned l, + const TraceLocalID& r) + : Instruction(tpl, r), lhs(l) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(lhs < events.size()); + assert(operations.find(owner) != operations.end()); +#endif + } + + //-------------------------------------------------------------------------- + void GetTermEvent::execute(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(operations.find(owner) != operations.end()); + assert(operations.find(owner)->second != NULL); +#endif + operations[owner]->replay_mapping_output(); + events[lhs] = operations[owner]->get_memo_completion(); + } + + //-------------------------------------------------------------------------- + std::string GetTermEvent::to_string(void) + //-------------------------------------------------------------------------- + { + std::stringstream ss; + ss << "events[" << lhs << "] = operations[" << owner + << "].get_completion_event() (op kind: " + << Operation::op_names[operations[owner]->get_memoizable_kind()] + << ")"; + return ss.str(); + } + + ///////////////////////////////////////////////////////////// + // CreateApUserEvent + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + CreateApUserEvent::CreateApUserEvent(PhysicalTemplate& tpl, unsigned l, + const TraceLocalID &o) + : Instruction(tpl, o), lhs(l) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(lhs < events.size()); + assert(user_events.find(lhs) != user_events.end()); +#endif + } + + //-------------------------------------------------------------------------- + void CreateApUserEvent::execute(void) //-------------------------------------------------------------------------- { ApUserEvent ev = Runtime::create_ap_user_event(NULL); @@ -5167,6 +7469,112 @@ namespace Legion { return ss.str(); } + ///////////////////////////////////////////////////////////// + // BarrierArrival + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + BarrierArrival::BarrierArrival(PhysicalTemplate &tpl, + ApBarrier bar, unsigned _lhs, unsigned _rhs) + : Instruction(tpl, TraceLocalID(0,DomainPoint())), barrier(bar), + lhs(_lhs), rhs(_rhs) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(lhs < events.size()); + assert(rhs < events.size()); +#endif + } + + //-------------------------------------------------------------------------- + BarrierArrival::~BarrierArrival(void) + //-------------------------------------------------------------------------- + { + // Destroy our barrier + barrier.destroy_barrier(); + } + + //-------------------------------------------------------------------------- + void BarrierArrival::execute(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(rhs < events.size()); + assert(lhs < events.size()); +#endif + Runtime::phase_barrier_arrive(barrier, 1/*count*/, events[rhs]); + events[lhs] = barrier; + Runtime::advance_barrier(barrier); + } + + //-------------------------------------------------------------------------- + std::string BarrierArrival::to_string(void) + //-------------------------------------------------------------------------- + { + std::stringstream ss; + ss << "events[" << lhs << "] = Runtime::phase_barrier_arrive(" + << barrier.id << ", events[" << rhs << "])"; + return ss.str(); + } + + //-------------------------------------------------------------------------- + ApBarrier BarrierArrival::record_subscribed_shard(ShardID remote_shard) + //-------------------------------------------------------------------------- + { + subscribed_shards.push_back(remote_shard); + return barrier; + } + + //-------------------------------------------------------------------------- + void BarrierArrival::refresh_barrier(ApEvent key, + std::map > ¬ifications) + //-------------------------------------------------------------------------- + { + // Destroy the old barrier + barrier.destroy_barrier(); + // Make the new barrier + barrier = ApBarrier(Realm::Barrier::create_barrier(1/*arrival count*/)); + for (std::vector::const_iterator it = + subscribed_shards.begin(); it != subscribed_shards.end(); it++) + notifications[*it][key] = barrier; + } + + ///////////////////////////////////////////////////////////// + // BarrierAdvance + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + BarrierAdvance::BarrierAdvance(PhysicalTemplate &tpl, + ApBarrier bar, unsigned _lhs) + : Instruction(tpl, TraceLocalID(0,DomainPoint())), barrier(bar), lhs(_lhs) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(lhs < events.size()); +#endif + } + + //-------------------------------------------------------------------------- + void BarrierAdvance::execute(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(lhs < events.size()); +#endif + events[lhs] = barrier; + Runtime::advance_barrier(barrier); + } + + //-------------------------------------------------------------------------- + std::string BarrierAdvance::to_string(void) + //-------------------------------------------------------------------------- + { + std::stringstream ss; + ss << "events[" << lhs << "] = Runtime::barrier_advance(" + << barrier.id << ")"; + return ss.str(); + } + }; // namespace Internal }; // namespace Legion diff --git a/runtime/legion/legion_trace.h b/runtime/legion/legion_trace.h index 74c149ca2b..e8ab1ee70d 100644 --- a/runtime/legion/legion_trace.h +++ b/runtime/legion/legion_trace.h @@ -20,6 +20,7 @@ #include "legion.h" #include "legion/legion_ops.h" #include "legion/legion_analysis.h" +#include "legion/legion_allocation.h" namespace Legion { namespace Internal { @@ -40,8 +41,8 @@ namespace Legion { bool val, DependenceType d, const FieldMask &m) : operation_idx(op_idx), prev_idx(pidx), - next_idx(nidx), validates(val), dtype(d), - dependent_mask(m) { } + next_idx(nidx), validates(val), + dtype(d), dependent_mask(m) { } public: inline bool merge(const DependenceRecord &record) { @@ -456,6 +457,12 @@ namespace Legion { void clear_cached_template(void) { current_template = NULL; } void check_template_preconditions(TraceReplayOp *op, std::set &applied_events); + // Return true if we evaluated all the templates + bool find_viable_templates(ReplTraceReplayOp *op, + std::set &applied_events, + unsigned templates_to_find, + std::vector &viable_templates); + PhysicalTemplate* select_template(unsigned template_index); public: PhysicalTemplate* get_current_template(void) { return current_template; } bool has_any_templates(void) const { return templates.size() > 0; } @@ -471,9 +478,8 @@ namespace Legion { { return execution_fence_event; } public: PhysicalTemplate* start_new_template(void); - RtEvent fix_trace(PhysicalTemplate *tpl, - Operation *op, - bool has_blocking_call); + void record_replayable_capture(PhysicalTemplate *tpl); + void record_failed_capture(PhysicalTemplate *tpl); public: const std::vector &get_replay_targets(void) { return replay_targets; } @@ -482,6 +488,7 @@ namespace Legion { public: Runtime * const runtime; const LegionTrace *logical_trace; + ReplicateContext *const repl_ctx; private: mutable LocalLock trace_lock; PhysicalTemplate* current_template; @@ -530,7 +537,7 @@ namespace Legion { void dump(void) const; protected: typedef LegionMap >::aligned ViewSet; + FieldMaskSet >::aligned ViewSet; protected: RegionTreeForest * const forest; protected: @@ -589,14 +596,16 @@ namespace Legion { public: PhysicalTemplate *tpl; }; - private: + protected: struct ViewUser { - ViewUser(const RegionUsage &r, unsigned u, IndexSpaceExpression *e) - : usage(r), user(u), expr(e) + ViewUser(const RegionUsage &r, unsigned u, + IndexSpaceExpression *e, int s) + : usage(r), user(u), expr(e), shard(s) {} const RegionUsage usage; const unsigned user; IndexSpaceExpression *const expr; + const ShardID shard; }; private: struct CachedMapping @@ -608,7 +617,7 @@ namespace Legion { std::deque physical_instances; }; typedef LegionMap::aligned CachedMappings; - private: + protected: typedef LegionMap >::aligned ViewExprs; typedef LegionMap &gen); + void elide_fences(std::vector &gen, ReplTraceOp *op); void propagate_merges(std::vector &gen); void transitive_reduction(void); void propagate_copies(std::vector &gen); void eliminate_dead_code(std::vector &gen); void prepare_parallel_replay(const std::vector &gen); void push_complete_replays(void); - public: + protected: + virtual void initialize_generators(std::vector &new_gen); + virtual void initialize_eliminate_dead_code_frontiers( + const std::vector &gen, + std::vector &used); + virtual void initialize_transitive_reduction_frontiers( + std::vector &topo_order, + std::vector &inv_topo_order); + public: + // Variants for normal traces bool check_preconditions(TraceReplayOp *op, std::set &applied_events); void apply_postcondition(TraceSummaryOp *op, std::set &applied_events); + // Variants for control replication traces + bool check_preconditions(ReplTraceReplayOp *op, + std::set &applied_events); + void apply_postcondition(ReplTraceSummaryOp *op, + std::set &applied_events); public: void register_operation(Operation *op); void execute_all(void); void execute_slice(unsigned slice_idx); public: - void issue_summary_operations(InnerContext* context, - Operation *invalidator); + virtual void issue_summary_operations(InnerContext* context, + Operation *invalidator); public: void dump_template(void); private: @@ -703,6 +727,7 @@ namespace Legion { std::deque &physical_instances) const; public: virtual void record_get_term_event(Memoizable *memo); + virtual void request_term_event(ApUserEvent &term_event); virtual void record_create_ap_user_event(ApUserEvent lhs, Memoizable *memo); virtual void record_trigger_event(ApUserEvent lhs, ApEvent rhs, @@ -766,36 +791,51 @@ namespace Legion { InstanceView *view, const RegionUsage &usage, const FieldMask &user_mask, - bool update_validity); + bool update_validity, + std::set &applied); virtual void record_post_fill_view(FillView *view, const FieldMask &mask); virtual void record_fill_views(ApEvent lhs, Memoizable *memo, unsigned idx, IndexSpaceExpression *expr, const FieldMaskSet &tracing_srcs, const FieldMaskSet &tracing_dsts, std::set &applied_events); - private: + protected: void record_views(unsigned entry, IndexSpaceExpression *expr, const RegionUsage &usage, const FieldMaskSet &views, - const LegionList >::aligned &eqs); - void update_valid_views(InstanceView *view, - EquivalenceSet *eq, - const RegionUsage &usage, - const FieldMask &user_mask, - bool invalidates); - void add_view_user(InstanceView *view, + const LegionList >::aligned &eqs, + std::set &applied); + virtual void update_valid_views(InstanceView *view, + EquivalenceSet *eq, + const RegionUsage &usage, + const FieldMask &user_mask, + bool invalidates, + std::set &applied_events); + virtual void add_view_user(InstanceView *view, const RegionUsage &usage, unsigned user, IndexSpaceExpression *user_expr, - const FieldMask &user_mask); + const FieldMask &user_mask, + std::set &applied, + int owner_shard = -1); void record_copy_views(unsigned copy_id, IndexSpaceExpression *expr, const FieldMaskSet &views); - void record_fill_views(const FieldMaskSet &views); + virtual void record_fill_views(const FieldMaskSet &views, + std::set &applied_events); public: virtual void record_set_op_sync_event(ApEvent &lhs, Memoizable *memo); virtual void record_set_effects(Memoizable *memo, ApEvent &rhs); virtual void record_complete_replay(Memoizable *memo, ApEvent rhs); + public: + virtual void record_owner_shard(unsigned trace_local_id, ShardID owner); + virtual void record_local_space(unsigned trace_local_id, IndexSpace sp); + virtual void record_sharding_function(unsigned trace_local_id, + ShardingFunction *function); + public: + virtual ShardID find_owner_shard(unsigned trace_local_id); + virtual IndexSpace find_local_space(unsigned trace_local_id); + virtual ShardingFunction* find_sharding_function(unsigned trace_local_id); public: RtEvent defer_template_deletion(void); public: @@ -804,38 +844,52 @@ namespace Legion { public: RtEvent get_recording_done(void) const { return recording_done; } - void trigger_recording_done(void); + virtual void trigger_recording_done(void); virtual RtEvent get_collect_event(void) const { return recording_done; } private: TraceLocalID find_trace_local_id(Memoizable *memo); unsigned find_memo_entry(Memoizable *memo); TraceLocalID record_memo_entry(Memoizable *memo, unsigned entry); - private: + protected: +#ifdef DEBUG_LEGION + // This is a virtual method in debug mode only since we have an + // assertion that we want to check in the ShardedPhysicalTemplate + virtual unsigned convert_event(const ApEvent &event, bool check = true); +#else unsigned convert_event(const ApEvent &event); - unsigned find_event(const ApEvent &event) const; +#endif + virtual unsigned find_event(const ApEvent &event, AutoLock &tpl_lock); unsigned find_or_convert_event(const ApEvent &event); void insert_instruction(Instruction *inst); - private: + protected: // Returns the set of last users for all - // tuples in the view_exprs + // tuples in the view_exprs, not that this is the void find_all_last_users(ViewExprs &view_exprs, - std::set &users); + std::set &users, + std::set &ready_events); + // Synchronization methods for elide fences that do nothing in + // the base case but can synchronize for multiple shards + virtual void elide_fences_pre_sync(ReplTraceOp *op) { } + virtual void elide_fences_post_sync(ReplTraceOp *op) { } // Returns the set of last users for a given - // tuple - void find_last_users(InstanceView *view, - IndexSpaceExpression *expr, - const FieldMask &mask, - std::set &users); + // tuple, this is virtual so it can be overridden in the sharded case + virtual void find_last_users(InstanceView *view, + IndexSpaceExpression *expr, + const FieldMask &mask, + std::set &users, + std::set &ready_events); public: inline ApEvent get_fence_completion(void) { return fence_completion; } void record_remote_memoizable(Memoizable *memo); void release_remote_memos(void); - private: + protected: PhysicalTrace * const trace; volatile bool recording; Replayable replayable; + protected: mutable LocalLock template_lock; const unsigned fence_completion_id; + private: const unsigned replay_parallelism; private: std::map operations; @@ -845,16 +899,17 @@ namespace Legion { private: CachedMappings cached_mappings; bool has_virtual_mapping; - private: + protected: ApEvent fence_completion; std::vector events; std::map user_events; - std::map event_map; + protected: + std::map event_map; private: std::vector instructions; std::vector > slices; std::vector > slice_tasks; - private: + protected: std::map crossing_events; // Frontiers of a template are a set of users whose events must // be carried over to the next replay for eliding the fence at the @@ -866,9 +921,9 @@ namespace Legion { // - after each replay, we do assignment // events[frontiers[idx]] = events[idx] std::map frontiers; - private: + protected: RtUserEvent recording_done; - private: + protected: RtUserEvent replay_ready; RtEvent replay_done; #ifdef LEGION_SPY @@ -877,15 +932,17 @@ namespace Legion { private: std::map op_views; std::map copy_views; - private: + protected: + // THESE ARE SHARDED FOR CONTROL REPLICATION!!! TraceConditionSet pre, post; + // THIS IS SHARDED FOR CONTROL REPLICATION!!! ViewGroups view_groups; // This data structure holds a set of last users for each view. // Each user (which is an index in the event table) is associated with // a field mask, an index expression representing the working set within // the view, and privilege. For any given pair of view and index // expression, there can be either multiple readers/reducers or a single - // writer. + // writer. THIS IS SHARDED FOR CONTROL REPLICATION!!! ViewUsers view_users; std::set all_users; private: @@ -894,7 +951,7 @@ namespace Legion { TraceViewSet consumed_reductions; private: std::map > reduction_ready_events; - private: + protected: FieldMaskSet pre_fill_views; FieldMaskSet post_fill_views; private: @@ -902,6 +959,260 @@ namespace Legion { friend class Instruction; }; + /** + * \class ShardedPhysicalTemplate + * This is an extension of the PhysicalTemplate class for handling + * templates for control replicated contexts. It mostly behaves the + * same as a normal PhysicalTemplate but has some additional + * extensions for handling the effects of control replication. + */ + class ShardedPhysicalTemplate : public PhysicalTemplate { + public: + enum UpdateKind { + UPDATE_VALID_VIEWS, + UPDATE_PRE_FILL, + UPDATE_POST_FILL, + UPDATE_VIEW_USER, + UPDATE_LAST_USER, + FIND_LAST_USERS_REQUEST, + FIND_LAST_USERS_RESPONSE, + FIND_FRONTIER_REQUEST, + FIND_FRONTIER_RESPONSE, + TEMPLATE_BARRIER_REFRESH, + FRONTIER_BARRIER_REFRESH, + }; + public: + struct DeferTraceUpdateArgs : public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_DEFER_TRACE_UPDATE_TASK_ID; + public: + DeferTraceUpdateArgs(ShardedPhysicalTemplate *target, + UpdateKind kind, RtUserEvent done, + Deserializer &derez, LogicalView *view, + EquivalenceSet *set = NULL, + RtUserEvent deferral = + RtUserEvent::NO_RT_USER_EVENT); + DeferTraceUpdateArgs(ShardedPhysicalTemplate *target, + UpdateKind kind, RtUserEvent done, + LogicalView *view, Deserializer &derez, + IndexSpaceExpression *expr, + RtUserEvent deferral = + RtUserEvent::NO_RT_USER_EVENT); + DeferTraceUpdateArgs(ShardedPhysicalTemplate *target, + UpdateKind kind, RtUserEvent done, + LogicalView *view, Deserializer &derez, + IndexSpace handle); + DeferTraceUpdateArgs(ShardedPhysicalTemplate *target, + UpdateKind kind, RtUserEvent done, + LogicalView *view, Deserializer &derez, + IndexSpaceExprID expr_id); + DeferTraceUpdateArgs(const DeferTraceUpdateArgs &args, + RtUserEvent deferral); + public: + ShardedPhysicalTemplate *const target; + const UpdateKind kind; + const RtUserEvent done; + LogicalView *const view; + EquivalenceSet *const eq; + IndexSpaceExpression *const expr; + const IndexSpaceExprID remote_expr_id; + const IndexSpace handle; + const size_t buffer_size; + void *const buffer; + const RtUserEvent deferral_event; + }; + public: + ShardedPhysicalTemplate(PhysicalTrace *trace, ApEvent fence_event, + ReplicateContext *repl_ctx); + ShardedPhysicalTemplate(const ShardedPhysicalTemplate &rhs); + protected: + virtual ~ShardedPhysicalTemplate(void); + public: + // Have to provide explicit overrides of operator new and + // delete here to make sure we get the right ones. C++ does + // not let us have these in a sub-class or it doesn't know + // which ones to pick from. + static inline void* operator new(size_t count) + { return legion_alloc_aligned(count); } + static inline void operator delete(void *ptr) + { free(ptr); } + inline RtEvent chain_deferral_events(RtUserEvent deferral_event) + { + volatile Realm::Event::id_t *ptr = &next_deferral_precondition.id; + RtEvent continuation_pre; + do { + continuation_pre.id = *ptr; + } while (!__sync_bool_compare_and_swap(ptr, + continuation_pre.id, deferral_event.id)); + return continuation_pre; + } + public: + virtual void initialize(Runtime *runtime, ApEvent fence_completion, + bool recurrent); + virtual ApEvent get_completion(void) const; + virtual ApEvent get_completion_for_deletion(void) const; + virtual void record_merge_events(ApEvent &lhs, + const std::set& rhs, Memoizable *memo); + virtual void record_issue_copy(Memoizable *memo, ApEvent &lhs, + IndexSpaceExpression *expr, + const std::vector& src_fields, + const std::vector& dst_fields, +#ifdef LEGION_SPY + RegionTreeID src_tree_id, RegionTreeID dst_tree_id, +#endif + ApEvent precondition, PredEvent guard_event, + ReductionOpID redop, bool reduction_fold); + virtual void record_issue_indirect(Memoizable *memo, ApEvent &lhs, + IndexSpaceExpression *expr, + const std::vector& src_fields, + const std::vector& dst_fields, + const std::vector &indirections, + ApEvent precondition, PredEvent pred_guard); + virtual void record_issue_fill(Memoizable *memo, ApEvent &lhs, + IndexSpaceExpression *expr, + const std::vector &fields, + const void *fill_value, size_t fill_size, +#ifdef LEGION_SPY + FieldSpace handle, RegionTreeID tree_id, +#endif + ApEvent precondition, PredEvent guard_event); + virtual void record_set_op_sync_event(ApEvent &lhs, Memoizable *memo); + public: + virtual void record_owner_shard(unsigned trace_local_id, ShardID owner); + virtual void record_local_space(unsigned trace_local_id, IndexSpace sp); + virtual void record_sharding_function(unsigned trace_local_id, + ShardingFunction *function); + virtual void issue_summary_operations(InnerContext *context, + Operation *invalidator); + public: + virtual ShardID find_owner_shard(unsigned trace_local_id); + virtual IndexSpace find_local_space(unsigned trace_local_id); + virtual ShardingFunction* find_sharding_function(unsigned trace_local_id); + public: + virtual void trigger_recording_done(void); + public: + ApBarrier find_trace_shard_event(ApEvent event, ShardID remote_shard); + void record_trace_shard_event(ApEvent event, ApBarrier result); + void handle_trace_update(Deserializer &derez, AddressSpaceID source); + static void handle_deferred_trace_update(const void *args, Runtime *rt); + protected: + bool handle_update_valid_views(InstanceView *view, EquivalenceSet *eq, + Deserializer &derez, std::set &applied, + RtUserEvent done, + const DeferTraceUpdateArgs *dargs = NULL); + bool handle_update_pre_fill(FillView *view, Deserializer &derez, + std::set &applied, RtUserEvent done, + const DeferTraceUpdateArgs *dargs = NULL); + bool handle_update_post_fill(FillView *view, Deserializer &derez, + std::set &applied, RtUserEvent done, + const DeferTraceUpdateArgs *dargs = NULL); + bool handle_update_view_user(InstanceView *view, IndexSpaceExpression *ex, + Deserializer &derez, std::set &applied, + RtUserEvent done, + const DeferTraceUpdateArgs *dargs = NULL); + void handle_find_last_users(InstanceView *view, IndexSpaceExpression *ex, + Deserializer &derez, std::set &applied); + protected: +#ifdef DEBUG_LEGION + virtual unsigned convert_event(const ApEvent &event, bool check = true); +#endif + virtual unsigned find_event(const ApEvent &event, AutoLock &tpl_lock); + void request_remote_shard_event(ApEvent event, RtUserEvent done_event); + static AddressSpaceID find_event_space(ApEvent event); + virtual Replayable check_replayable(ReplTraceOp *op, + bool has_blocking_call) const; + virtual void update_valid_views(InstanceView *view, + EquivalenceSet *eq, + const RegionUsage &usage, + const FieldMask &user_mask, + bool invalidates, + std::set &applied); + virtual void add_view_user(InstanceView *view, + const RegionUsage &usage, + unsigned user, IndexSpaceExpression *user_expr, + const FieldMask &user_mask, + std::set &applied, + int owner_shard = -1); + virtual void record_fill_views(const FieldMaskSet &views, + std::set &applied_events); + public: + void record_replayed(void); + protected: + ShardID find_view_owner(InstanceView *view); + ShardID find_equivalence_owner(EquivalenceSet *set); + void find_owner_shards(AddressSpace owner, std::vector &shards); + void find_last_users_sharded(InstanceView *view, + IndexSpaceExpression *expr, + const FieldMask &mask, + std::set > &sharded_users); + protected: + virtual void elide_fences_pre_sync(ReplTraceOp *op); + virtual void elide_fences_post_sync(ReplTraceOp *op); + virtual void find_last_users(InstanceView *view, + IndexSpaceExpression *expr, + const FieldMask &mask, + std::set &users, + std::set &ready_events); + virtual void initialize_generators(std::vector &new_gen); + virtual void initialize_transitive_reduction_frontiers( + std::vector &topo_order, + std::vector &inv_topo_order); + public: + ReplicateContext *const repl_ctx; + const ShardID local_shard; + const size_t total_shards; + // Make this last since it registers the template with the + // context which can trigger calls into the template so + // everything must valid at this point + const size_t template_index; + private: + static const unsigned NO_INDEX = UINT_MAX; + protected: + std::map pending_event_requests; + std::map remote_arrivals; + std::map local_advances; + std::map > did_shard_owners; + std::map owner_shards; + std::map local_spaces; + std::map sharding_functions; + protected: + // Count how many times we've been replayed so we know when we're going + // to run out of phase barrier generations + size_t total_replays; + // Count how many advance instructions we've seen updated for when + // we need to reset the phase barriers for a new round of generations + size_t updated_advances; + // An event to signal when our advances are ready + RtUserEvent update_advances_ready; + // An event for chainging deferrals of update tasks + RtEvent next_deferral_precondition; + // Barrier for signaliing when we are done recording our template + RtBarrier recording_barrier; + protected: + // Count how many times we've done recurrent replay so we know when we're + // going to run out of phase barrier generations + size_t recurrent_replays; + // Count how many frontiers ahave been updated so that we know when + // they are done being updated + size_t updated_frontiers; + // An event to signal when our frontiers are ready + RtUserEvent update_frontiers_ready; + protected: + // This is a data structure that tracks last users whose events we + // own eventhough their instance is on a remote node + std::set local_last_users; + protected: + // Data structures for fence elision + // Local frontiers records barriers that should be arrived on + // based on events that we have here locally + std::map local_frontiers; + // Remote shards that are subscribed to our local frontiers + std::map > local_subscriptions; + // Remote frontiers records barriers that we should fill in as + // events from remote shards + std::vector > remote_frontiers; + }; + enum InstructionKind { GET_TERM_EVENT = 0, @@ -914,6 +1225,8 @@ namespace Legion { SET_EFFECTS, ASSIGN_FENCE_COMPLETION, COMPLETE_REPLAY, + BARRIER_ARRIVAL, + BARRIER_ADVANCE, #ifdef LEGION_GPU_REDUCTIONS GPU_REDUCTION, #endif @@ -942,6 +1255,8 @@ namespace Legion { virtual SetOpSyncEvent* as_set_op_sync_event(void) { return NULL; } virtual SetEffects* as_set_effects(void) { return NULL; } virtual CompleteReplay* as_complete_replay(void) { return NULL; } + virtual BarrierArrival* as_barrier_arrival(void) { return NULL; } + virtual BarrierAdvance* as_barrier_advance(void) { return NULL; } #ifdef LEGION_GPU_REDUCTIONS virtual GPUReduction* as_gpu_reduction(void) { return NULL; } #endif @@ -1249,6 +1564,57 @@ namespace Legion { unsigned rhs; }; + /** + * \class BarrierArrival + * This instruction has the following semantics: + * events[lhs] = barrier.arrive(events[rhs]) + */ + class BarrierArrival : public Instruction { + public: + BarrierArrival(PhysicalTemplate &tpl, + ApBarrier bar, unsigned lhs, unsigned rhs); + virtual ~BarrierArrival(void); + virtual void execute(void); + virtual std::string to_string(void); + + virtual InstructionKind get_kind(void) + { return BARRIER_ARRIVAL; } + virtual BarrierArrival* as_barrier_arrival(void) + { return this; } + ApBarrier record_subscribed_shard(ShardID remote_shard); + inline ApEvent get_current_barrier(void) const { return barrier; } + void refresh_barrier(ApEvent key, + std::map > ¬ifications); + private: + friend class PhysicalTemplate; + ApBarrier barrier; + unsigned lhs, rhs; + std::vector subscribed_shards; + }; + + /** + * \class BarrierAdvance + * This instruction has the following semantics + * events[lhs] = barrier + * barrier.advance(); + */ + class BarrierAdvance : public Instruction { + public: + BarrierAdvance(PhysicalTemplate &tpl, ApBarrier bar, unsigned lhs); + virtual void execute(void); + virtual std::string to_string(void); + + virtual InstructionKind get_kind(void) + { return BARRIER_ADVANCE; } + virtual BarrierAdvance* as_barrier_advance(void) + { return this; } + inline void refresh_barrier(ApBarrier next) { barrier = next; } + private: + friend class PhysicalTemplate; + ApBarrier barrier; + unsigned lhs; + }; + }; // namespace Internal }; // namespace Legion diff --git a/runtime/legion/legion_types.h b/runtime/legion/legion_types.h index 00f74f1532..d20aaa1e84 100644 --- a/runtime/legion/legion_types.h +++ b/runtime/legion/legion_types.h @@ -159,6 +159,7 @@ namespace Legion { template struct ColoredPoints; struct InputArgs; class ProjectionFunctor; + class ShardingFunctor; class Task; class Copy; class InlineMapping; @@ -258,12 +259,12 @@ namespace Legion { // Only projection states below here OPEN_READ_ONLY_PROJ = 5, // read-only projection OPEN_READ_WRITE_PROJ = 6, // read-write projection - OPEN_READ_WRITE_PROJ_DISJOINT_SHALLOW = 7, // depth=0, children disjoint - OPEN_REDUCE_PROJ = 8, // reduction-only projection - OPEN_REDUCE_PROJ_DIRTY = 9, // same as above but already open dirty + OPEN_REDUCE_PROJ = 7, // reduction-only projection + OPEN_REDUCE_PROJ_DIRTY = 8, // same as above but already open dirty }; - // redop IDs - none used in HLR right now, but 0 isn't allowed + // Internal reduction operators + // Currently we don't use any, but 0 is reserved enum { REDOP_ID_AVAILABLE = 1, }; @@ -322,7 +323,6 @@ namespace Legion { LG_DEFER_PHYSICAL_REGISTRATION_TASK_ID, LG_PART_INDEPENDENCE_TASK_ID, LG_SPACE_INDEPENDENCE_TASK_ID, - LG_PENDING_CHILD_TASK_ID, LG_POST_DECREMENT_TASK_ID, LG_ISSUE_FRAME_TASK_ID, LG_MAPPER_CONTINUATION_TASK_ID, @@ -339,6 +339,7 @@ namespace Legion { LG_SELECT_TUNABLE_TASK_ID, LG_DEFERRED_ENQUEUE_OP_ID, LG_DEFERRED_ENQUEUE_TASK_ID, + LG_DEFERRED_TASK_COMPLETE_TASK_ID, LG_DEFER_MAPPER_MESSAGE_TASK_ID, LG_REMOTE_VIEW_CREATION_TASK_ID, LG_DEFER_DISTRIBUTE_TASK_ID, @@ -350,6 +351,10 @@ namespace Legion { LG_DEFER_REDUCTION_VIEW_TASK_ID, LG_DEFER_PHI_VIEW_REF_TASK_ID, LG_DEFER_PHI_VIEW_REGISTRATION_TASK_ID, + LG_CONTROL_REP_LAUNCH_TASK_ID, + LG_CONTROL_REP_DELETE_TASK_ID, + LG_RECLAIM_FUTURE_MAP_TASK_ID, + LG_DEFER_COMPOSITE_COPY_TASK_ID, LG_TIGHTEN_INDEX_SPACE_TASK_ID, LG_REMOTE_PHYSICAL_REQUEST_TASK_ID, LG_REMOTE_PHYSICAL_RESPONSE_TASK_ID, @@ -388,7 +393,9 @@ namespace Legion { LG_DEFER_RELEASE_ACQUIRED_TASK_ID, LG_MALLOC_INSTANCE_TASK_ID, LG_FREE_INSTANCE_TASK_ID, + LG_DEFER_CONSENSUS_MATCH_TASK_ID, LG_YIELD_TASK_ID, + LG_DEFER_TRACE_UPDATE_TASK_ID, // this marks the beginning of task IDs tracked by the shutdown algorithm LG_BEGIN_SHUTDOWN_TASK_IDS, LG_RETRY_SHUTDOWN_TASK_ID = LG_BEGIN_SHUTDOWN_TASK_IDS, @@ -452,7 +459,6 @@ namespace Legion { "Defer Physical Registration", \ "Partition Independence Test", \ "Index Space Independence Test", \ - "Remove Pending Child", \ "Post Decrement Task", \ "Issue Frame", \ "Mapper Continuation", \ @@ -469,6 +475,7 @@ namespace Legion { "Select Tunable", \ "Deferred Enqueue Op", \ "Deferred Enqueue Task", \ + "Deferred Task Complete", \ "Deferred Mapper Message", \ "Remote View Creation", \ "Defer Task Distribution", \ @@ -480,6 +487,10 @@ namespace Legion { "Defer Reduction View Registration", \ "Defer Phi View Reference", \ "Defer Phi View Registration", \ + "Control Replication Launch", \ + "Control Replciation Delete", \ + "Reclaim Future Map", \ + "Defer Composite Copy", \ "Tighten Index Space", \ "Remote Physical Context Request", \ "Remote Physical Context Response", \ @@ -518,7 +529,9 @@ namespace Legion { "Defer Release Acquired Instances", \ "Malloc Instance", \ "Free Instance", \ + "Defer Consensus Match", \ "Yield", \ + "Defer Trace Update", \ "Retry Shutdown", \ "Remote Message", \ }; @@ -530,11 +543,13 @@ namespace Legion { PREMAP_TASK_CALL, SLICE_TASK_CALL, MAP_TASK_CALL, + MAP_REPLICATE_TASK_CALL, SELECT_VARIANT_CALL, POSTMAP_TASK_CALL, TASK_SELECT_SOURCES_CALL, TASK_SPECULATE_CALL, TASK_REPORT_PROFILING_CALL, + TASK_SELECT_SHARDING_FUNCTOR_CALL, MAP_INLINE_CALL, INLINE_SELECT_SOURCES_CALL, INLINE_REPORT_PROFILING_CALL, @@ -542,21 +557,28 @@ namespace Legion { COPY_SELECT_SOURCES_CALL, COPY_SPECULATE_CALL, COPY_REPORT_PROFILING_CALL, + COPY_SELECT_SHARDING_FUNCTOR_CALL, CLOSE_SELECT_SOURCES_CALL, CLOSE_REPORT_PROFILING_CALL, + CLOSE_SELECT_SHARDING_FUNCTOR_CALL, MAP_ACQUIRE_CALL, ACQUIRE_SPECULATE_CALL, ACQUIRE_REPORT_PROFILING_CALL, + ACQUIRE_SELECT_SHARDING_FUNCTOR_CALL, MAP_RELEASE_CALL, RELEASE_SELECT_SOURCES_CALL, RELEASE_SPECULATE_CALL, RELEASE_REPORT_PROFILING_CALL, + RELEASE_SELECT_SHARDING_FUNCTOR_CALL, SELECT_PARTITION_PROJECTION_CALL, MAP_PARTITION_CALL, PARTITION_SELECT_SOURCES_CALL, PARTITION_REPORT_PROFILING_CALL, + PARTITION_SELECT_SHARDING_FUNCTOR_CALL, + FILL_SELECT_SHARDING_FUNCTOR_CALL, CONFIGURE_CONTEXT_CALL, SELECT_TUNABLE_VALUE_CALL, + MUST_EPOCH_SELECT_SHARDING_FUNCTOR_CALL, MAP_MUST_EPOCH_CALL, MAP_DATAFLOW_GRAPH_CALL, MEMOIZE_OPERATION_CALL, @@ -576,11 +598,13 @@ namespace Legion { "premap_task", \ "slice_task", \ "map_task", \ + "map_replicate_task", \ "select_task_variant", \ "postmap_task", \ "select_task_sources", \ "speculate (for task)", \ "report profiling (for task)", \ + "select sharding functor (for task)", \ "map_inline", \ "select_inline_sources", \ "report profiling (for inline)", \ @@ -588,21 +612,28 @@ namespace Legion { "select_copy_sources", \ "speculate (for copy)", \ "report_profiling (for copy)", \ + "select sharding functor (for copy)", \ "select_close_sources", \ "report_profiling (for close)", \ + "select sharding functor (for close)", \ "map_acquire", \ "speculate (for acquire)", \ "report_profiling (for acquire)", \ + "select sharding functor (for acquire)", \ "map_release", \ "select_release_sources", \ "speculate (for release)", \ "report_profiling (for release)", \ + "select sharding functor (for release)", \ "select partition projection", \ "map_partition", \ "select_partition_sources", \ "report_profiling (for partition)", \ + "select sharding functor (for partition)", \ + "select sharding functor (for fill)", \ "configure_context", \ "select_tunable_value", \ + "select sharding functor (for must epoch)", \ "map_must_epoch", \ "map_dataflow_graph", \ "memoize_operation", \ @@ -750,6 +781,7 @@ namespace Legion { SEND_MATERIALIZED_VIEW, SEND_FILL_VIEW, SEND_PHI_VIEW, + SEND_SHARDED_VIEW, SEND_REDUCTION_VIEW, SEND_INSTANCE_MANAGER, SEND_COLLECTIVE_MANAGER, @@ -773,6 +805,19 @@ namespace Legion { SEND_FUTURE_BROADCAST, SEND_FUTURE_MAP_REQUEST, SEND_FUTURE_MAP_RESPONSE, + SEND_REPL_FUTURE_MAP_REQUEST, + SEND_REPL_FUTURE_MAP_RESPONSE, + SEND_REPL_TOP_VIEW_REQUEST, + SEND_REPL_TOP_VIEW_RESPONSE, + SEND_REPL_EQ_REQUEST, + SEND_REPL_EQ_RESPONSE, + SEND_REPL_INTRA_SPACE_DEP, + SEND_REPL_RESOURCE_UPDATE, + SEND_REPL_TRACE_EVENT_REQUEST, + SEND_REPL_TRACE_EVENT_RESPONSE, + SEND_REPL_TRACE_UPDATE, + SEND_REPL_IMPLICIT_REQUEST, + SEND_REPL_IMPLICIT_RESPONSE, SEND_MAPPER_MESSAGE, SEND_MAPPER_BROADCAST, SEND_TASK_IMPL_SEMANTIC_REQ, @@ -833,12 +878,21 @@ namespace Legion { SEND_TOP_LEVEL_TASK_REQUEST, SEND_TOP_LEVEL_TASK_COMPLETE, SEND_MPI_RANK_EXCHANGE, + SEND_REPLICATE_LAUNCH, + SEND_REPLICATE_DELETE, + SEND_REPLICATE_POST_MAPPED, + SEND_REPLICATE_POST_EXECUTION, + SEND_REPLICATE_TRIGGER_COMPLETE, + SEND_REPLICATE_TRIGGER_COMMIT, + SEND_CONTROL_REPLICATE_COLLECTIVE_MESSAGE, SEND_LIBRARY_MAPPER_REQUEST, SEND_LIBRARY_MAPPER_RESPONSE, SEND_LIBRARY_TRACE_REQUEST, SEND_LIBRARY_TRACE_RESPONSE, SEND_LIBRARY_PROJECTION_REQUEST, SEND_LIBRARY_PROJECTION_RESPONSE, + SEND_LIBRARY_SHARDING_REQUEST, + SEND_LIBRARY_SHARDING_RESPONSE, SEND_LIBRARY_TASK_REQUEST, SEND_LIBRARY_TASK_RESPONSE, SEND_LIBRARY_REDOP_REQUEST, @@ -932,6 +986,7 @@ namespace Legion { "Send Materialized View", \ "Send Fill View", \ "Send Phi View", \ + "Send Sharded View", \ "Send Reduction View", \ "Send Instance Manager", \ "Send Collective Instance Manager", \ @@ -955,6 +1010,19 @@ namespace Legion { "Send Future Broadcast", \ "Send Future Map Future Request", \ "Send Future Map Future Response", \ + "Send Replicate Future Map Request", \ + "Send Replicate Future Map Response", \ + "Send Replicate Top View Request", \ + "Send Replicate Top View Response", \ + "Send Replicate Equivalence Set Request", \ + "Send Replicate Equivalence Set Response", \ + "Send Replicate Intra Space Dependence", \ + "Send Replicate Resource Update", \ + "Send Replicate Trace Event Request", \ + "Send Replicate Trace Event Response", \ + "Send Replicate Trace Update", \ + "Send Replicate Implicit Request", \ + "Send Replicate Implicit Response", \ "Send Mapper Message", \ "Send Mapper Broadcast", \ "Send Task Impl Semantic Req", \ @@ -1015,12 +1083,21 @@ namespace Legion { "Top Level Task Request", \ "Top Level Task Complete", \ "Send MPI Rank Exchange", \ + "Send Replication Launch", \ + "Send Replication Delete", \ + "Send Replication Post Mapped", \ + "Send Replication Post Execution", \ + "Send Replication Trigger Complete", \ + "Send Replication Trigger Commit", \ + "Send Control Replication Collective Message", \ "Send Library Mapper Request", \ "Send Library Mapper Response", \ "Send Library Trace Request", \ "Send Library Trace Response", \ "Send Library Projection Request", \ "Send Library Projection Response", \ + "Send Library Sharding Request", \ + "Send Library Sharding Response", \ "Send Library Task Request", \ "Send Library Task Response", \ "Send Library Redop Request", \ @@ -1343,6 +1420,113 @@ namespace Legion { TASK_SEMANTIC, }; + // Static locations for where collectives are allocated + // These are just arbitrary numbers but they should appear + // with at most one logical static collective kind + // Ones that have been commented out are free to be reused + enum CollectiveIndexLocation { + COLLECTIVE_LOC_0 = 0, + COLLECTIVE_LOC_1 = 1, + COLLECTIVE_LOC_2 = 2, + COLLECTIVE_LOC_3 = 3, + COLLECTIVE_LOC_4 = 4, + COLLECTIVE_LOC_5 = 5, + COLLECTIVE_LOC_6 = 6, + COLLECTIVE_LOC_7 = 7, + COLLECTIVE_LOC_8 = 8, + COLLECTIVE_LOC_9 = 9, + COLLECTIVE_LOC_10 = 10, + COLLECTIVE_LOC_11 = 11, + COLLECTIVE_LOC_12 = 12, + COLLECTIVE_LOC_13 = 13, + COLLECTIVE_LOC_14 = 14, + COLLECTIVE_LOC_15 = 15, + COLLECTIVE_LOC_16 = 16, + COLLECTIVE_LOC_17 = 17, + //COLLECTIVE_LOC_18 = 18, + //COLLECTIVE_LOC_19 = 19, + //COLLECTIVE_LOC_20 = 20, + //COLLECTIVE_LOC_21 = 21, + //COLLECTIVE_LOC_22 = 22, + //COLLECTIVE_LOC_23 = 23, + //COLLECTIVE_LOC_24 = 24, + //COLLECTIVE_LOC_25 = 25, + //COLLECTIVE_LOC_26 = 26, + //COLLECTIVE_LOC_27 = 27, + //COLLECTIVE_LOC_28 = 28, + //COLLECTIVE_LOC_29 = 29, + COLLECTIVE_LOC_30 = 30, + COLLECTIVE_LOC_31 = 31, + COLLECTIVE_LOC_32 = 32, + COLLECTIVE_LOC_33 = 33, + COLLECTIVE_LOC_34 = 34, + COLLECTIVE_LOC_35 = 35, + COLLECTIVE_LOC_36 = 36, + COLLECTIVE_LOC_37 = 37, + COLLECTIVE_LOC_38 = 38, + COLLECTIVE_LOC_39 = 39, + COLLECTIVE_LOC_40 = 40, + COLLECTIVE_LOC_41 = 41, + COLLECTIVE_LOC_42 = 42, + COLLECTIVE_LOC_43 = 43, + COLLECTIVE_LOC_44 = 44, + COLLECTIVE_LOC_45 = 45, + COLLECTIVE_LOC_46 = 46, + COLLECTIVE_LOC_47 = 47, + COLLECTIVE_LOC_48 = 48, + COLLECTIVE_LOC_49 = 49, + COLLECTIVE_LOC_50 = 50, + COLLECTIVE_LOC_51 = 51, + COLLECTIVE_LOC_52 = 52, + COLLECTIVE_LOC_53 = 53, + COLLECTIVE_LOC_54 = 54, + COLLECTIVE_LOC_55 = 55, + COLLECTIVE_LOC_56 = 56, + COLLECTIVE_LOC_57 = 57, + COLLECTIVE_LOC_58 = 58, + COLLECTIVE_LOC_59 = 59, + COLLECTIVE_LOC_60 = 60, + COLLECTIVE_LOC_61 = 61, + COLLECTIVE_LOC_62 = 62, + COLLECTIVE_LOC_63 = 63, + COLLECTIVE_LOC_64 = 64, + COLLECTIVE_LOC_65 = 65, + COLLECTIVE_LOC_66 = 66, + COLLECTIVE_LOC_67 = 67, + COLLECTIVE_LOC_68 = 68, + COLLECTIVE_LOC_69 = 69, + COLLECTIVE_LOC_70 = 70, + COLLECTIVE_LOC_71 = 71, + COLLECTIVE_LOC_72 = 72, + COLLECTIVE_LOC_73 = 73, + COLLECTIVE_LOC_74 = 74, + COLLECTIVE_LOC_75 = 75, + COLLECTIVE_LOC_76 = 76, + COLLECTIVE_LOC_77 = 77, + COLLECTIVE_LOC_78 = 78, + COLLECTIVE_LOC_79 = 79, + COLLECTIVE_LOC_80 = 80, + COLLECTIVE_LOC_81 = 81, + COLLECTIVE_LOC_82 = 82, + COLLECTIVE_LOC_83 = 83, + COLLECTIVE_LOC_84 = 84, + COLLECTIVE_LOC_85 = 85, + COLLECTIVE_LOC_86 = 86, + COLLECTIVE_LOC_87 = 87, + COLLECTIVE_LOC_88 = 88, + COLLECTIVE_LOC_89 = 89, + COLLECTIVE_LOC_90 = 90, + COLLECTIVE_LOC_91 = 91, + COLLECTIVE_LOC_92 = 92, + COLLECTIVE_LOC_93 = 93, + COLLECTIVE_LOC_94 = 94, + COLLECTIVE_LOC_95 = 95, + COLLECTIVE_LOC_96 = 96, + COLLECTIVE_LOC_97 = 97, + COLLECTIVE_LOC_98 = 98, + COLLECTIVE_LOC_99 = 99, + }; + // legion_types.h class LocalLock; class AutoLock; @@ -1367,6 +1551,7 @@ namespace Legion { class ArgumentMapImpl; class FutureImpl; class FutureMapImpl; + class ReplFutureMapImpl; class PhysicalRegionImpl; class PieceIteratorImpl; class GrantImpl; @@ -1381,6 +1566,7 @@ namespace Legion { class VariantImpl; class LayoutConstraints; class ProjectionFunction; + class ShardingFunction; class Runtime; // A small interface class for handling profiling responses struct ProfilingResponseBase; @@ -1454,6 +1640,7 @@ namespace Legion { class MultiTask; class IndividualTask; class PointTask; + class ShardTask; class IndexTask; class SliceTask; class RemoteTask; @@ -1462,6 +1649,7 @@ namespace Legion { class TaskContext; class InnerContext;; class TopLevelContext; + class ReplicateContext; class RemoteContext; class LeafContext; class InlineContext; @@ -1489,6 +1677,9 @@ namespace Legion { // Use this global variable to track if we're an // implicit top-level task that needs to do external waits extern __thread bool external_implicit_task; +#ifdef DEBUG_LEGION_WAITS + extern __thread int meta_task_id; +#endif /** * \class LgTaskArgs @@ -1518,6 +1709,7 @@ namespace Legion { class TraceViewSet; class TraceConditionSet; class PhysicalTemplate; + class ShardedPhysicalTemplate; class Instruction; class GetTermEvent; class CreateApUserEvent; @@ -1530,6 +1722,8 @@ namespace Legion { class SetOpSyncEvent; class SetEffects; class CompleteReplay; + class BarrierArrival; + class BarrierAdvance; #ifdef LEGION_GPU_REDUCTIONS class GPUReduction; #endif @@ -1584,6 +1778,7 @@ namespace Legion { class MaterializedView; class FillView; class PhiView; + class ShardedView; class MappingRef; class InstanceRef; class InstanceSet; @@ -1624,6 +1819,48 @@ namespace Legion { typedef Mapping::MapperEvent MapperEvent; typedef Mapping::ProfilingMeasurementID ProfilingMeasurementID; + // legion_replication.h + class ReplIndividualTask; + class ReplIndexTask; + class ReplMergeCloseOp; + class ReplFillOp; + class ReplIndexFillOp; + class ReplCopyOp; + class ReplIndexCopyOp; + class ReplDeletionOp; + class ReplPendingPartitionOp; + class ReplDependentPartitionOp; + class ReplMustEpochOp; + class ReplTimingOp; + class ReplAllReduceOp; + class ReplFenceOp; + class ReplMapOp; + class ReplAttachOp; + class ReplDetachOp; + class ReplTraceOp; + class ReplTraceCaptureOp; + class ReplTraceCompleteOp; + class ReplTraceReplayOp; + class ReplTraceBeginOp; + class ReplTraceSummaryOp; + class ShardMapping; + class ShardManager; + class ShardCollective; + class GatherCollective; + template + class AllGatherCollective; + template class BarrierExchangeCollective; + template class ValueBroadcast; + class CrossProductCollective; + class ShardingGatherCollective; + class FieldDescriptorExchange; + class FieldDescriptorGather; + class FutureBroadcast; + class FutureExchange; + class FutureNameExchange; + class MustEpochMappingBroadcast; + class MustEpochMappingExchange; + #define FRIEND_ALL_RUNTIME_CLASSES \ friend class Legion::Runtime; \ friend class Internal::Runtime; \ @@ -1673,6 +1910,22 @@ namespace Legion { friend class Internal::PointTask; \ friend class Internal::IndexTask; \ friend class Internal::SliceTask; \ + friend class Internal::ReplIndividualTask; \ + friend class Internal::ReplIndexTask; \ + friend class Internal::ReplFillOp; \ + friend class Internal::ReplIndexFillOp; \ + friend class Internal::ReplCopyOp; \ + friend class Internal::ReplIndexCopyOp; \ + friend class Internal::ReplDeletionOp; \ + friend class Internal::ReplPendingPartitionOp; \ + friend class Internal::ReplDependentPartitionOp; \ + friend class Internal::ReplMustEpochOp; \ + friend class Internal::ReplMapOp; \ + friend class Internal::ReplTimingOp; \ + friend class Internal::ReplAllReduceOp; \ + friend class Internal::ReplFenceOp; \ + friend class Internal::ReplAttachOp; \ + friend class Internal::ReplDetachOp; \ friend class Internal::RegionTreeForest; \ friend class Internal::IndexSpaceNode; \ friend class Internal::IndexPartNode; \ @@ -1696,13 +1949,18 @@ namespace Legion { friend class Internal::LegionHandshakeImpl; \ friend class Internal::ArgumentMapImpl; \ friend class Internal::FutureMapImpl; \ + friend class Internal::ReplFutureMapImpl; \ friend class Internal::TaskContext; \ friend class Internal::InnerContext; \ friend class Internal::TopLevelContext; \ friend class Internal::RemoteContext; \ friend class Internal::LeafContext; \ friend class Internal::InlineContext; \ + friend class Internal::ReplicateContext; \ friend class Internal::InstanceBuilder; \ + friend class Internal::FutureNameExchange; \ + friend class Internal::MustEpochMappingExchange; \ + friend class Internal::MustEpochMappingBroadcast; \ friend class BindingLib::Utility; \ friend class CObjectWrapper; @@ -1812,6 +2070,7 @@ namespace Legion { const LegionColor INVALID_COLOR = LLONG_MAX; // This is only needed internally typedef Realm::RegionInstance PhysicalInstance; + typedef unsigned long long CollectiveID; typedef unsigned long long IndexSpaceExprID; // Helper for encoding templates struct NT_TemplateHelper : @@ -2090,6 +2349,13 @@ namespace Legion { inline operator Realm::Barrier() const { Realm::Barrier b; b.id = id; b.timestamp = timestamp; return b; } + public: + inline bool get_result(void *value, size_t value_size) const + { Realm::Barrier b; b.id = id; + b.timestamp = timestamp; return b.get_result(value, value_size); } + inline void destroy_barrier(void) + { Realm::Barrier b; b.id = id; + b.timestamp = timestamp; b.destroy_barrier(); } public: Realm::Barrier::timestamp_t timestamp; }; @@ -2136,6 +2402,16 @@ namespace Legion { inline operator Realm::Barrier() const { Realm::Barrier b; b.id = id; b.timestamp = timestamp; return b; } + public: + inline bool get_result(void *value, size_t value_size) const + { Realm::Barrier b; b.id = id; + b.timestamp = timestamp; return b.get_result(value, value_size); } + inline RtBarrier get_previous_phase(void) + { Realm::Barrier b; b.id = id; + return RtBarrier(b.get_previous_phase()); } + inline void destroy_barrier(void) + { Realm::Barrier b; b.id = id; + b.timestamp = timestamp; b.destroy_barrier(); } public: Realm::Barrier::timestamp_t timestamp; }; @@ -2346,6 +2622,10 @@ namespace Legion { inline void LgEvent::wait(void) const //-------------------------------------------------------------------------- { +#ifdef DEBUG_LEGION_WAITS + const int local_meta_task_id = Internal::meta_task_id; + const long long start = Realm::Clock::current_time_in_microseconds(); +#endif // Save the context locally Internal::TaskContext *local_ctx = Internal::implicit_context; // Save the task provenance information @@ -2390,6 +2670,12 @@ namespace Legion { Internal::implicit_provenance = local_provenance; // Write the registration callback information back Internal::inside_registration_callback = local_callback; +#ifdef DEBUG_LEGION_WAITS + Internal::meta_task_id = local_meta_task_id; + const long long stop = Realm::Clock::current_time_in_microseconds(); + if (((stop - start) >= LIMIT) && (local_meta_task_id == BAD_TASK_ID)) + assert(false); +#endif } #ifdef LEGION_SPY diff --git a/runtime/legion/legion_utilities.h b/runtime/legion/legion_utilities.h index 1342e95fb8..9e0c9ce061 100644 --- a/runtime/legion/legion_utilities.h +++ b/runtime/legion/legion_utilities.h @@ -136,10 +136,14 @@ namespace Legion { ///////////////////////////////////////////////////////////// class Deserializer { public: - Deserializer(const void *buf, size_t buffer_size) + Deserializer(const void *buf, size_t buffer_size +#ifdef DEBUG_LEGION + , size_t ctx_bytes = 0 +#endif + ) : total_bytes(buffer_size), buffer((const char*)buf), index(0) #ifdef DEBUG_LEGION - , context_bytes(0) + , context_bytes(ctx_bytes) #endif { } Deserializer(const Deserializer &rhs) @@ -202,6 +206,8 @@ namespace Legion { size_t index; #ifdef DEBUG_LEGION size_t context_bytes; + public: + inline size_t get_context_bytes(void) const { return context_bytes; } #endif }; @@ -336,6 +342,69 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + static inline bool configure_collective_settings(const int participants, + const int local_space, + int &collective_radix, + int &collective_log_radix, + int &collective_stages, + int &participating_spaces, + int &collective_last_radix) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(collective_radix > 0); +#endif + const int MultiplyDeBruijnBitPosition[32] = + { + 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 + }; + // First adjust the radix based on the number of nodes if necessary + if (collective_radix > participants) + collective_radix = participants; + // Adjust the radix to the next smallest power of 2 + uint32_t radix_copy = collective_radix; + for (int i = 0; i < 5; i++) + radix_copy |= radix_copy >> (1 << i); + collective_log_radix = + MultiplyDeBruijnBitPosition[(uint32_t)(radix_copy * 0x07C4ACDDU) >> 27]; + if (collective_radix != (1 << collective_log_radix)) + collective_radix = (1 << collective_log_radix); + + // Compute the number of stages + uint32_t node_copy = participants; + for (int i = 0; i < 5; i++) + node_copy |= node_copy >> (1 << i); + // Now we have it log 2 + int log_nodes = + MultiplyDeBruijnBitPosition[(uint32_t)(node_copy * 0x07C4ACDDU) >> 27]; + + // Stages round up in case of incomplete stages + collective_stages = + (log_nodes + collective_log_radix - 1) / collective_log_radix; + int log_remainder = log_nodes % collective_log_radix; + if (log_remainder > 0) + { + // We have an incomplete last stage + collective_last_radix = 1 << log_remainder; + // Now we can compute the number of participating stages + participating_spaces = + 1 << ((collective_stages - 1) * collective_log_radix + + log_remainder); + } + else + { + collective_last_radix = collective_radix; + participating_spaces = 1 << (collective_stages * collective_log_radix); + } +#ifdef DEBUG_LEGION + assert((participating_spaces % collective_radix) == 0); +#endif + const bool participant = (local_space < participating_spaces); + return participant; + } + ///////////////////////////////////////////////////////////// // Semantic Info ///////////////////////////////////////////////////////////// @@ -473,6 +542,43 @@ namespace Legion { BITMASK comp[LOG2MAX]; }; + ///////////////////////////////////////////////////////////// + // Murmur3Hasher + ///////////////////////////////////////////////////////////// + + /** + * \class Murmur3Hasher + * This class implements an object-oriented version of the + * MurmurHash3 hashing algorithm for computing a 128-bit + * hash value. It is taken from the public domain here: + * https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp + */ + class Murmur3Hasher { + public: + Murmur3Hasher(uint64_t seed = 0xCC892563); + public: + template + inline void hash(const T &value); + inline void hash(const void *values, size_t size); + inline void hash(const ExecutionConstraintSet &set); + inline void hash(const TaskLayoutConstraintSet &set); + inline void finalize(uint64_t result[2]); + protected: + inline uint64_t rotl64(uint64_t x, uint8_t r); + inline uint64_t fmix64(uint64_t k); + protected: + union { + uint64_t k[2]; + uint8_t b[16]; + } blocks; + uint64_t h1, h2, len; + uint8_t bytes; + bool finalized; + public: + static const uint64_t c1 = 0x87c37b91114253d5ULL; + static const uint64_t c2 = 0x4cf5ad432745937fULL; + }; + ///////////////////////////////////////////////////////////// // Dynamic Table ///////////////////////////////////////////////////////////// @@ -1516,6 +1622,136 @@ namespace Legion { identity = true; } + //------------------------------------------------------------------------- + inline Murmur3Hasher::Murmur3Hasher(uint64_t seed) + : h1(seed), h2(seed), len(0), bytes(0), finalized(false) + //------------------------------------------------------------------------- + { + } + + //------------------------------------------------------------------------- + template + inline void Murmur3Hasher::hash(const T &value) + //------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!finalized); +#endif + const uint8_t *data = reinterpret_cast(&value); + for (unsigned idx = 0; idx < sizeof(T); idx++) + { + blocks.b[bytes++] = data[idx]; + if (bytes == 16) + { + // body + uint64_t k1 = blocks.k[0]; + uint64_t k2 = blocks.k[1]; + k1 *= c1; k1 = rotl64(k1,31); k1 *= c2; h1 ^= k1; + h1 = rotl64(h1,27); h1 += h2; h1 = h1*5+0x52dce729; + k2 *= c2; k2 = rotl64(k2,33); k2 *= c1; h2 ^= k2; + h2 = rotl64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; + len += 16; + bytes = 0; + } + } + } + + //------------------------------------------------------------------------- + inline void Murmur3Hasher::hash(const void *value, size_t size) + //------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(!finalized); +#endif + const uint8_t *data = reinterpret_cast(value); + for (unsigned idx = 0; idx < size; idx++) + { + blocks.b[bytes++] = data[idx]; + if (bytes == 16) + { + // body + uint64_t k1 = blocks.k[0]; + uint64_t k2 = blocks.k[1]; + k1 *= c1; k1 = rotl64(k1,31); k1 *= c2; h1 ^= k1; + h1 = rotl64(h1,27); h1 += h2; h1 = h1*5+0x52dce729; + k2 *= c2; k2 = rotl64(k2,33); k2 *= c1; h2 ^= k2; + h2 = rotl64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; + len += 16; + bytes = 0; + } + } + } + + //------------------------------------------------------------------------- + inline void Murmur3Hasher::finalize(uint64_t result[2]) + //------------------------------------------------------------------------- + { + if (!finalized) + { + // tail + uint64_t k1 = 0; + uint64_t k2 = 0; + switch (bytes) + { + case 15: k2 ^= ((uint64_t)blocks.b[14]) << 48; + case 14: k2 ^= ((uint64_t)blocks.b[13]) << 40; + case 13: k2 ^= ((uint64_t)blocks.b[12]) << 32; + case 12: k2 ^= ((uint64_t)blocks.b[11]) << 24; + case 11: k2 ^= ((uint64_t)blocks.b[10]) << 16; + case 10: k2 ^= ((uint64_t)blocks.b[ 9]) << 8; + case 9: k2 ^= ((uint64_t)blocks.b[ 8]) << 0; + k2 *= c2; k2 = rotl64(k2,33); k2 *= c1; h2 ^= k2; + + case 8: k1 ^= ((uint64_t)blocks.b[ 7]) << 56; + case 7: k1 ^= ((uint64_t)blocks.b[ 6]) << 48; + case 6: k1 ^= ((uint64_t)blocks.b[ 5]) << 40; + case 5: k1 ^= ((uint64_t)blocks.b[ 4]) << 32; + case 4: k1 ^= ((uint64_t)blocks.b[ 3]) << 24; + case 3: k1 ^= ((uint64_t)blocks.b[ 2]) << 16; + case 2: k1 ^= ((uint64_t)blocks.b[ 1]) << 8; + case 1: k1 ^= ((uint64_t)blocks.b[ 0]) << 0; + k1 *= c1; k1 = rotl64(k1,31); k1 *= c2; h1 ^= k1; + } + + // finalization + len += bytes; + + h1 ^= len; h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + + finalized = true; + } + result[0] = h1; + result[1] = h2; + } + + //------------------------------------------------------------------------- + inline uint64_t Murmur3Hasher::rotl64(uint64_t x, uint8_t r) + //------------------------------------------------------------------------- + { + return (x << r) | (x >> (64 - r)); + } + + //------------------------------------------------------------------------- + inline uint64_t Murmur3Hasher::fmix64(uint64_t k) + //------------------------------------------------------------------------- + { + k ^= k >> 33; + k *= 0xff51afd7ed558ccdULL; + k ^= k >> 33; + k *= 0xc4ceb9fe1a85ec53ULL; + k ^= k >> 33; + return k; + } + //------------------------------------------------------------------------- template DynamicTable::DynamicTable(void) diff --git a/runtime/legion/legion_views.cc b/runtime/legion/legion_views.cc index 781414eeb0..f143e1433f 100644 --- a/runtime/legion/legion_views.cc +++ b/runtime/legion/legion_views.cc @@ -25,6 +25,7 @@ #include "legion/legion_analysis.h" #include "legion/legion_trace.h" #include "legion/legion_context.h" +#include "legion/legion_replication.h" namespace Legion { namespace Internal { @@ -4134,36 +4135,58 @@ namespace Legion { for (LegionMap::aligned::const_iterator it = true_views.begin(); it != true_views.end(); it++) it->first->remove_nested_valid_ref(did, mutator); - for (LegionMap::aligned::const_iterator it = + for (LegionMap::aligned::const_iterator it = false_views.begin(); it != false_views.end(); it++) it->first->remove_nested_valid_ref(did, mutator); } //-------------------------------------------------------------------------- - void PhiView::record_true_view(LogicalView *view, const FieldMask &mask) + void PhiView::record_true_view(LogicalView *view, const FieldMask &mask, + ReferenceMutator *mutator) //-------------------------------------------------------------------------- { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif LegionMap::aligned::iterator finder = true_views.find(view); if (finder == true_views.end()) { true_views[view] = mask; - view->add_nested_resource_ref(did); + if (view->is_deferred_view()) + { + // Deferred views need valid and gc references + view->add_nested_gc_ref(did, mutator); + view->add_nested_valid_ref(did, mutator); + } + else // Otherwise we just need the valid reference + view->add_nested_resource_ref(did); } else finder->second |= mask; } //-------------------------------------------------------------------------- - void PhiView::record_false_view(LogicalView *view, const FieldMask &mask) + void PhiView::record_false_view(LogicalView *view, const FieldMask &mask, + ReferenceMutator *mutator) //-------------------------------------------------------------------------- { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif LegionMap::aligned::iterator finder = false_views.find(view); if (finder == false_views.end()) { false_views[view] = mask; - view->add_nested_resource_ref(did); + if (view->is_deferred_view()) + { + // Deferred views need valid and gc references + view->add_nested_gc_ref(did, mutator); + view->add_nested_valid_ref(did, mutator); + } + else // Otherwise we just need the valid reference + view->add_nested_resource_ref(did); } else finder->second |= mask; @@ -4325,6 +4348,261 @@ namespace Legion { pargs->view->register_with_runtime(NULL/*no remote registration*/); } + ///////////////////////////////////////////////////////////// + // ShardedView + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardedView::ShardedView(RegionTreeForest *forest, DistributedID did, + AddressSpaceID owner_space, bool register_now) + : DeferredView(forest, encode_sharded_did(did), owner_space, register_now) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + valid = true; +#endif + // If we're not the owner add a reference that will be removed + // by the owner when no one is valid any longer + if (!is_owner()) + add_base_gc_ref(REMOTE_DID_REF); + } + + //-------------------------------------------------------------------------- + ShardedView::ShardedView(const ShardedView &rhs) + : DeferredView(rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ShardedView::~ShardedView(void) + //-------------------------------------------------------------------------- + { + for (std::set::const_iterator it = + local_instances.begin(); it != local_instances.end(); it++) + if ((*it)->remove_nested_resource_ref(did)) + delete (*it); + } + + //-------------------------------------------------------------------------- + ShardedView& ShardedView::operator=(const ShardedView &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void ShardedView::notify_active(ReferenceMutator *mutator) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(valid); +#endif + // Do nothing + } + + //-------------------------------------------------------------------------- + void ShardedView::notify_inactive(ReferenceMutator *mutator) + //-------------------------------------------------------------------------- + { + if (!is_owner()) + { +#ifdef DEBUG_LEGION + assert(valid); + valid = false; +#endif + // Remove our valid references + for (std::set::const_iterator it = + local_instances.begin(); it != local_instances.end(); it++) + if ((*it)->remove_nested_valid_ref(did, mutator)) + delete (*it); + } + } + + //-------------------------------------------------------------------------- + void ShardedView::notify_valid(ReferenceMutator *mutator) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(valid); +#endif + if (!is_owner()) + send_remote_valid_increment(owner_space, mutator); + } + + //-------------------------------------------------------------------------- + void ShardedView::RemoteDecrementFunctor::apply(AddressSpaceID target) const + //-------------------------------------------------------------------------- + { + if (target == owner) + return; + view->send_remote_gc_decrement(target, mutator); + } + + //-------------------------------------------------------------------------- + void ShardedView::notify_invalid(ReferenceMutator *mutator) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(valid); +#endif + if (is_owner()) + { +#ifdef DEBUG_LEGION + valid = false; +#endif + if (has_remote_instances()) + { + // Send out messages to any remote copies to move the GC reference + RemoteDecrementFunctor functor(this, local_space, mutator); + map_over_remote_instances(functor); + } + // Remove our valid references + for (std::set::const_iterator it = + local_instances.begin(); it != local_instances.end(); it++) + if ((*it)->remove_nested_valid_ref(did, mutator)) + delete (*it); + } + else + send_remote_valid_decrement(owner_space, mutator); + } + + //-------------------------------------------------------------------------- + void ShardedView::send_view(AddressSpaceID target) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif + Serializer rez; + { + RezCheck z(rez); + rez.serialize(did); + AutoLock v_lock(view_lock,1,false/*exclusive*/); + rez.serialize(global_views.size()); + for (LegionMap::aligned::const_iterator it = + global_views.begin(); it != global_views.end(); it++) + { + rez.serialize(it->first); + rez.serialize(it->second); + } + } + runtime->send_sharded_view(target, rez); + update_remote_instances(target); + } + + //-------------------------------------------------------------------------- + void ShardedView::flatten(CopyFillAggregator &aggregator, + InstanceView *dst_view, const FieldMask &src_mask, + IndexSpaceExpression *expr, + CopyAcrossHelper *helper) + //-------------------------------------------------------------------------- + { + // First see if it's already valid for the fields we want + FieldMask copy_mask = src_mask; + LegionMap::aligned::const_iterator finder = + global_views.find(dst_view->did); + if (finder != global_views.end()) + { + copy_mask -= finder->second; + // If it's already valid for all the fields we're done + if (!copy_mask) + return; + } + // Request all the views and wait for them to be ready + std::set ready_events; + FieldMaskSet src_views; + for (LegionMap::aligned::const_iterator it = + global_views.begin(); it != global_views.end(); it++) + { + const FieldMask overlap = copy_mask & it->second; + if (!overlap) + continue; + RtEvent ready; + LogicalView *view = + runtime->find_or_request_logical_view(it->first, ready); + if (ready.exists()) + ready_events.insert(ready); + src_views.insert(view, overlap); + } + if (!ready_events.empty()) + { + const RtEvent wait_on = Runtime::merge_events(ready_events); + if (wait_on.exists()) + wait_on.wait(); + } + if (!src_views.empty()) + aggregator.record_updates(dst_view, src_views, copy_mask, expr, + 0/*redop*/, helper); + } + + //-------------------------------------------------------------------------- + void ShardedView::initialize( + LegionMap::aligned &views, + const InstanceSet &local_insts, std::set &applied_events) + //-------------------------------------------------------------------------- + { + WrapperReferenceMutator mutator(applied_events); + for (unsigned idx = 0; idx < local_insts.size(); idx++) + { + const InstanceRef &inst = local_insts[idx]; + PhysicalManager *manager = inst.get_instance_manager(); + std::pair::iterator,bool> result = + local_instances.insert(manager); + if (result.second) + { + manager->add_nested_valid_ref(did, &mutator); + manager->add_nested_resource_ref(did); + } + } + AutoLock v_lock(view_lock); + // If it's not empty then we already got the same thing from else where + if (global_views.empty()) + global_views.swap(views); + } + + //-------------------------------------------------------------------------- + void ShardedView::unpack_view(Deserializer &derez) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(global_views.empty()); +#endif + size_t num_views; + derez.deserialize(num_views); + for (unsigned idx = 0; idx < num_views; idx++) + { + DistributedID view_did; + derez.deserialize(view_did); + derez.deserialize(global_views[view_did]); + } + } + + //-------------------------------------------------------------------------- + /*static*/ void ShardedView::handle_send_sharded_view(Runtime *runtime, + Deserializer &derez, AddressSpaceID source) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + DistributedID did; + derez.deserialize(did); + // Make the sharded view but don't register it yet + void *location; + ShardedView *view = NULL; + if (runtime->find_pending_collectable_location(did, location)) + view = new(location) ShardedView(runtime->forest, did, source, + false/*register_now*/); + else + view = new ShardedView(runtime->forest, did, source, + false/*register now*/); + view->unpack_view(derez); + view->register_with_runtime(NULL/*remote registration not needed*/); + } + ///////////////////////////////////////////////////////////// // ReductionView ///////////////////////////////////////////////////////////// diff --git a/runtime/legion/legion_views.h b/runtime/legion/legion_views.h index 8664c17ef5..3d5a5facb3 100644 --- a/runtime/legion/legion_views.h +++ b/runtime/legion/legion_views.h @@ -19,6 +19,7 @@ #include "legion/legion_types.h" #include "legion/legion_analysis.h" #include "legion/legion_utilities.h" +#include "legion/legion_instances.h" #include "legion/legion_allocation.h" #include "legion/garbage_collection.h" @@ -60,6 +61,7 @@ namespace Legion { inline bool is_reduction_view(void) const; inline bool is_fill_view(void) const; inline bool is_phi_view(void) const; + inline bool is_sharded_view(void) const; public: inline InstanceView* as_instance_view(void) const; inline DeferredView* as_deferred_view(void) const; @@ -67,6 +69,7 @@ namespace Legion { inline ReductionView* as_reduction_view(void) const; inline FillView* as_fill_view(void) const; inline PhiView *as_phi_view(void) const; + inline ShardedView* as_sharded_view(void) const; public: virtual bool has_manager(void) const = 0; virtual PhysicalManager* get_manager(void) const = 0; @@ -85,10 +88,12 @@ namespace Legion { static inline DistributedID encode_reduction_did(DistributedID did); static inline DistributedID encode_fill_did(DistributedID did); static inline DistributedID encode_phi_did(DistributedID did); + static inline DistributedID encode_sharded_did(DistributedID did); static inline bool is_materialized_did(DistributedID did); static inline bool is_reduction_did(DistributedID did); static inline bool is_fill_did(DistributedID did); static inline bool is_phi_did(DistributedID did); + static inline bool is_sharded_did(DistributedID did); public: RegionTreeForest *const context; protected: @@ -893,13 +898,11 @@ namespace Legion { public: virtual void notify_active(ReferenceMutator *mutator) = 0; virtual void notify_inactive(ReferenceMutator *mutator) = 0; + public: virtual void notify_valid(ReferenceMutator *mutator) = 0; virtual void notify_invalid(ReferenceMutator *mutator) = 0; public: virtual void send_view(AddressSpaceID target) = 0; - // Should never be called directly - virtual InnerContext* get_context(void) const - { assert(false); return NULL; } public: virtual void flatten(CopyFillAggregator &aggregator, InstanceView *dst_view, const FieldMask &src_mask, @@ -1031,16 +1034,16 @@ namespace Legion { virtual void notify_invalid(ReferenceMutator *mutator); public: virtual void send_view(AddressSpaceID target); - virtual InnerContext* get_context(void) const - { return owner_context; } public: virtual void flatten(CopyFillAggregator &aggregator, InstanceView *dst_view, const FieldMask &src_mask, IndexSpaceExpression *expr, CopyAcrossHelper *helper); public: - void record_true_view(LogicalView *view, const FieldMask &view_mask); - void record_false_view(LogicalView *view, const FieldMask &view_mask); + void record_true_view(LogicalView *view, const FieldMask &view_mask, + ReferenceMutator *mutator); + void record_false_view(LogicalView *view, const FieldMask &view_mask, + ReferenceMutator *mutator); public: void pack_phi_view(Serializer &rez); void unpack_phi_view(Deserializer &derez,std::set &ready_events); @@ -1059,6 +1062,64 @@ namespace Legion { LegionMap::aligned false_views; }; + /** + * \class ShardedView + * A shared view is a representation of many instances all of which + * have the same data at the same version. This comes up mainly in + * control replication cases such as for inline mappings and attach + * operations where we make many local copies of the same data for + * the same logical region. It's better to store one of these in + * an equivalence set instead of a bunch of seperate invidividual views. + */ + class ShardedView : public DeferredView { + public: + class RemoteDecrementFunctor { + public: + RemoteDecrementFunctor(ShardedView *v, + AddressSpaceID own, ReferenceMutator *m) + : view(v), owner(own), mutator(m) { } + public: + void apply(AddressSpaceID space) const; + public: + ShardedView *const view; + const AddressSpaceID owner; + ReferenceMutator *const mutator; + }; + public: + ShardedView(RegionTreeForest *forest, DistributedID did, + AddressSpaceID owner_space, bool register_now); + ShardedView(const ShardedView &rhs); + virtual ~ShardedView(void); + public: + ShardedView& operator=(const ShardedView &rhs); + public: + virtual void notify_active(ReferenceMutator *mutator); + virtual void notify_inactive(ReferenceMutator *mutator); + public: + virtual void notify_valid(ReferenceMutator *mutator); + virtual void notify_invalid(ReferenceMutator *mutator); + public: + virtual void send_view(AddressSpaceID target); + public: + virtual void flatten(CopyFillAggregator &aggregator, + InstanceView *dst_view, const FieldMask &src_mask, + IndexSpaceExpression *expr,CopyAcrossHelper *helper); + public: + void initialize(LegionMap::aligned &views, + const InstanceSet &local_instances, + std::set &applied_events); + void unpack_view(Deserializer &derez); + public: + static void handle_send_sharded_view(Runtime *runtime, + Deserializer &derez, AddressSpaceID source); + protected: + std::set local_instances; + LegionMap::aligned global_views; +#ifdef DEBUG_LEGION + bool valid; +#endif + }; + //-------------------------------------------------------------------------- /*static*/ inline DistributedID LogicalView::encode_materialized_did( DistributedID did) @@ -1103,6 +1164,17 @@ namespace Legion { return LEGION_DISTRIBUTED_HELP_ENCODE(did, PHI_VIEW_DC); } + //-------------------------------------------------------------------------- + /*static*/ inline DistributedID LogicalView::encode_sharded_did( + DistributedID did) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(DIST_TYPE_LAST_DC < (1U << 7)); +#endif + return LEGION_DISTRIBUTED_HELP_ENCODE(did, SHARDED_VIEW_DC); + } + //-------------------------------------------------------------------------- /*static*/ inline bool LogicalView::is_materialized_did(DistributedID did) //-------------------------------------------------------------------------- @@ -1134,6 +1206,14 @@ namespace Legion { return ((LEGION_DISTRIBUTED_HELP_DECODE(did) & 0xFULL) == PHI_VIEW_DC); } + //-------------------------------------------------------------------------- + /*static*/ inline bool LogicalView::is_sharded_did(DistributedID did) + //-------------------------------------------------------------------------- + { + return ((LEGION_DISTRIBUTED_HELP_DECODE(did) & 0xFULL) == + SHARDED_VIEW_DC); + } + //-------------------------------------------------------------------------- inline bool LogicalView::is_instance_view(void) const //-------------------------------------------------------------------------- @@ -1145,7 +1225,7 @@ namespace Legion { inline bool LogicalView::is_deferred_view(void) const //-------------------------------------------------------------------------- { - return (is_fill_did(did) || is_phi_did(did)); + return (is_fill_did(did) || is_phi_did(did) || is_sharded_did(did)); } //-------------------------------------------------------------------------- @@ -1176,6 +1256,13 @@ namespace Legion { return is_phi_did(did); } + //-------------------------------------------------------------------------- + inline bool LogicalView::is_sharded_view(void) const + //-------------------------------------------------------------------------- + { + return is_sharded_did(did); + } + //-------------------------------------------------------------------------- inline InstanceView* LogicalView::as_instance_view(void) const //-------------------------------------------------------------------------- @@ -1236,6 +1323,16 @@ namespace Legion { return static_cast(const_cast(this)); } + //-------------------------------------------------------------------------- + inline ShardedView* LogicalView::as_sharded_view(void) const + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(is_sharded_view()); +#endif + return static_cast(const_cast(this)); + } + //-------------------------------------------------------------------------- template inline bool ExprView::has_local_precondition(PhysicalUser *user, diff --git a/runtime/legion/mapper_manager.cc b/runtime/legion/mapper_manager.cc index 55603047de..ca03e296c9 100644 --- a/runtime/legion/mapper_manager.cc +++ b/runtime/legion/mapper_manager.cc @@ -180,6 +180,35 @@ namespace Legion { finish_mapper_call(info); } + //-------------------------------------------------------------------------- + void MapperManager::invoke_map_replicate_task(TaskOp *task, + Mapper::MapTaskInput *input, + Mapper::MapTaskOutput *default_output, + Mapper::MapReplicateTaskOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(MAP_REPLICATE_TASK_CALL, + task, continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation4 + continuation(this, task, input, + default_output, output, info); + continuation.defer(runtime, continuation_precondition, task); + return; + } + } + mapper->map_replicate_task(info, *task, *input, *default_output, *output); + finish_mapper_call(info); + } + //-------------------------------------------------------------------------- void MapperManager::invoke_select_task_variant(TaskOp *task, Mapper::SelectVariantInput *input, @@ -331,6 +360,32 @@ namespace Legion { finish_mapper_call(info); } + //-------------------------------------------------------------------------- + void MapperManager::invoke_task_select_sharding_functor(TaskOp *task, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(TASK_SELECT_SHARDING_FUNCTOR_CALL, task, + continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation3 + continuation(this, task, input, output, info); + continuation.defer(runtime, continuation_precondition, task); + return; + } + } + mapper->select_sharding_functor(info, *task, *input, *output); + finish_mapper_call(info); + } + //-------------------------------------------------------------------------- void MapperManager::invoke_map_inline(MapOp *op, Mapper::MapInlineInput *input, @@ -558,6 +613,32 @@ namespace Legion { finish_mapper_call(info); } + //-------------------------------------------------------------------------- + void MapperManager::invoke_copy_select_sharding_functor(CopyOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(COPY_SELECT_SHARDING_FUNCTOR_CALL, op, + continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation3 + continuation(this, op, input, output, info); + continuation.defer(runtime, continuation_precondition, op); + return; + } + } + mapper->select_sharding_functor(info, *op, *input, *output); + finish_mapper_call(info); + } + //-------------------------------------------------------------------------- void MapperManager::invoke_select_close_sources(CloseOp *op, Mapper::SelectCloseSrcInput *input, @@ -634,6 +715,32 @@ namespace Legion { finish_mapper_call(info); } + //-------------------------------------------------------------------------- + void MapperManager::invoke_close_select_sharding_functor(CloseOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(CLOSE_SELECT_SHARDING_FUNCTOR_CALL, op, + continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation3 + continuation(this, op, input, output, info); + continuation.defer(runtime, continuation_precondition, op); + return; + } + } + mapper->select_sharding_functor(info, *op, *input, *output); + finish_mapper_call(info); + } + //-------------------------------------------------------------------------- void MapperManager::invoke_map_acquire(AcquireOp *op, Mapper::MapAcquireInput *input, @@ -707,6 +814,32 @@ namespace Legion { finish_mapper_call(info); } + //-------------------------------------------------------------------------- + void MapperManager::invoke_acquire_select_sharding_functor(AcquireOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(ACQUIRE_SELECT_SHARDING_FUNCTOR_CALL, op, + continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation3 + continuation(this, op, input, output, info); + continuation.defer(runtime, continuation_precondition, op); + return; + } + } + mapper->select_sharding_functor(info, *op, *input, *output); + finish_mapper_call(info); + } + //-------------------------------------------------------------------------- void MapperManager::invoke_map_release(ReleaseOp *op, Mapper::MapReleaseInput *input, @@ -832,6 +965,32 @@ namespace Legion { finish_mapper_call(info); } + //-------------------------------------------------------------------------- + void MapperManager::invoke_release_select_sharding_functor(ReleaseOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(RELEASE_SELECT_SHARDING_FUNCTOR_CALL, op, + continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation3 + continuation(this, op, input, output, info); + continuation.defer(runtime, continuation_precondition, op); + return; + } + } + mapper->select_sharding_functor(info, *op, *input, *output); + finish_mapper_call(info); + } + //-------------------------------------------------------------------------- void MapperManager::invoke_select_partition_projection( DependentPartitionOp *op, @@ -968,6 +1127,61 @@ namespace Legion { finish_mapper_call(info); } + //-------------------------------------------------------------------------- + void MapperManager::invoke_partition_select_sharding_functor( + DependentPartitionOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(PARTITION_SELECT_SHARDING_FUNCTOR_CALL, op, + continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation3 + continuation(this, op, input, output, info); + continuation.defer(runtime, continuation_precondition, op); + return; + } + } + mapper->select_sharding_functor(info, *op, *input, *output); + finish_mapper_call(info); + } + + //-------------------------------------------------------------------------- + void MapperManager::invoke_fill_select_sharding_functor(FillOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(FILL_SELECT_SHARDING_FUNCTOR_CALL, op, + continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation3 + continuation(this, op, input, output, info); + continuation.defer(runtime, continuation_precondition, op); + return; + } + } + mapper->select_sharding_functor(info, *op, *input, *output); + finish_mapper_call(info); + } + //-------------------------------------------------------------------------- void MapperManager::invoke_configure_context(TaskOp *task, Mapper::ContextConfigOutput *output, @@ -1018,6 +1232,34 @@ namespace Legion { finish_mapper_call(info); } + //-------------------------------------------------------------------------- + void MapperManager::invoke_must_epoch_select_sharding_functor( + MustEpochOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::MustEpochShardingFunctorOutput *output, + MappingCallInfo *info) + //-------------------------------------------------------------------------- + { + if (info == NULL) + { + RtEvent continuation_precondition; + info = begin_mapper_call(MUST_EPOCH_SELECT_SHARDING_FUNCTOR_CALL, op, + continuation_precondition); + if (continuation_precondition.exists()) + { + MapperContinuation3 + continuation(this, op, input, output, info); + continuation.defer(runtime, continuation_precondition, op); + return; + } + } + mapper->select_sharding_functor(info, *op, *input, *output); + finish_mapper_call(info); + } + //-------------------------------------------------------------------------- void MapperManager::invoke_map_must_epoch(MustEpochOp *op, Mapper::MapMustEpochInput *input, @@ -1544,6 +1786,18 @@ namespace Legion { return result; } + //-------------------------------------------------------------------------- + bool MapperManager::is_replicable_variant(MappingCallInfo *ctx, + TaskID task_id, VariantID variant_id) + //-------------------------------------------------------------------------- + { + pause_mapper_call(ctx); + VariantImpl *impl = runtime->find_variant_impl(task_id, variant_id); + bool result = impl->is_replicable(); + resume_mapper_call(ctx); + return result; + } + //-------------------------------------------------------------------------- VariantID MapperManager::register_task_variant(MappingCallInfo *ctx, const TaskVariantRegistrar ®istrar, @@ -3576,6 +3830,36 @@ namespace Legion { resume_mapper_call(ctx); } + //-------------------------------------------------------------------------- + bool MapperManager::is_MPI_interop_configured(void) + //-------------------------------------------------------------------------- + { + return runtime->is_MPI_interop_configured(); + } + + //-------------------------------------------------------------------------- + const std::map& MapperManager::find_forward_MPI_mapping( + MappingCallInfo *ctx) + //-------------------------------------------------------------------------- + { + return runtime->find_forward_MPI_mapping(); + } + + //-------------------------------------------------------------------------- + const std::map& MapperManager::find_reverse_MPI_mapping( + MappingCallInfo *ctx) + //-------------------------------------------------------------------------- + { + return runtime->find_reverse_MPI_mapping(); + } + + //-------------------------------------------------------------------------- + int MapperManager::find_local_MPI_rank(void) + //-------------------------------------------------------------------------- + { + return runtime->find_local_MPI_rank(); + } + //-------------------------------------------------------------------------- MappingCallInfo* MapperManager::allocate_call_info(MappingCallKind kind, Operation *op, bool need_lock) diff --git a/runtime/legion/mapper_manager.h b/runtime/legion/mapper_manager.h index acf60f481a..2bb91f3146 100644 --- a/runtime/legion/mapper_manager.h +++ b/runtime/legion/mapper_manager.h @@ -87,6 +87,10 @@ namespace Legion { void invoke_map_task(TaskOp *task, Mapper::MapTaskInput *input, Mapper::MapTaskOutput *output, MappingCallInfo *info = NULL); + void invoke_map_replicate_task(TaskOp *task, Mapper::MapTaskInput *input, + Mapper::MapTaskOutput *default_output, + Mapper::MapReplicateTaskOutput *output, + MappingCallInfo *info = NULL); void invoke_select_task_variant(TaskOp *task, Mapper::SelectVariantInput *input, Mapper::SelectVariantOutput *output, @@ -108,6 +112,10 @@ namespace Legion { void invoke_task_report_profiling(TaskOp *task, Mapper::TaskProfilingInfo *input, MappingCallInfo *info = NULL); + void invoke_task_select_sharding_functor(TaskOp *task, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info = NULL); public: // Inline mapper calls void invoke_map_inline(MapOp *op, Mapper::MapInlineInput *input, Mapper::MapInlineOutput *output, @@ -141,6 +149,10 @@ namespace Legion { void invoke_copy_report_profiling(CopyOp *op, Mapper::CopyProfilingInfo *input, MappingCallInfo *info = NULL); + void invoke_copy_select_sharding_functor(CopyOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info = NULL); public: // Close mapper calls void invoke_select_close_sources(CloseOp *op, Mapper::SelectCloseSrcInput *input, @@ -153,6 +165,10 @@ namespace Legion { void invoke_close_report_profiling(CloseOp *op, Mapper::CloseProfilingInfo *input, MappingCallInfo *info = NULL); + void invoke_close_select_sharding_functor(CloseOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info = NULL); public: // Acquire mapper calls void invoke_map_acquire(AcquireOp *op, Mapper::MapAcquireInput *input, @@ -164,6 +180,10 @@ namespace Legion { void invoke_acquire_report_profiling(AcquireOp *op, Mapper::AcquireProfilingInfo *input, MappingCallInfo *info = NULL); + void invoke_acquire_select_sharding_functor(AcquireOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info = NULL); public: // Release mapper calls void invoke_map_release(ReleaseOp *op, Mapper::MapReleaseInput *input, @@ -183,6 +203,10 @@ namespace Legion { void invoke_release_report_profiling(ReleaseOp *op, Mapper::ReleaseProfilingInfo *input, MappingCallInfo *info = NULL); + void invoke_release_select_sharding_functor(ReleaseOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info = NULL); public: // Partition mapper calls void invoke_select_partition_projection(DependentPartitionOp *op, Mapper::SelectPartitionProjectionInput *input, @@ -203,6 +227,15 @@ namespace Legion { void invoke_partition_report_profiling(DependentPartitionOp *op, Mapper::PartitionProfilingInfo *input, MappingCallInfo *info = NULL); + void invoke_partition_select_sharding_functor(DependentPartitionOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info = NULL); + public: // Fill mapper calls + void invoke_fill_select_sharding_functor(FillOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::SelectShardingFunctorOutput *output, + MappingCallInfo *info = NULL); public: // Task execution mapper calls void invoke_configure_context(TaskOp *task, Mapper::ContextConfigOutput *output, @@ -212,6 +245,10 @@ namespace Legion { Mapper::SelectTunableOutput *output, MappingCallInfo *info = NULL); public: // must epoch and graph mapper calls + void invoke_must_epoch_select_sharding_functor(MustEpochOp *op, + Mapper::SelectShardingFunctorInput *input, + Mapper::MustEpochShardingFunctorOutput *output, + MappingCallInfo *info = NULL); void invoke_map_must_epoch(MustEpochOp *op, Mapper::MapMustEpochInput *input, Mapper::MapMustEpochOutput *output, @@ -306,6 +343,8 @@ namespace Legion { VariantID variant_id); bool is_idempotent_variant(MappingCallInfo *ctx, TaskID task_id, VariantID variant_id); + bool is_replicable_variant(MappingCallInfo *ctx, + TaskID task_id, VariantID variant_id); public: VariantID register_task_variant(MappingCallInfo *ctx, const TaskVariantRegistrar ®istrar, @@ -537,6 +576,13 @@ namespace Legion { const char *&result); void retrieve_name(MappingCallInfo *ctx, LogicalPartition handle, const char *&result); + public: + bool is_MPI_interop_configured(void); + const std::map& find_forward_MPI_mapping( + MappingCallInfo *ctx); + const std::map& find_reverse_MPI_mapping( + MappingCallInfo *ctx); + int find_local_MPI_rank(void); protected: // Both these must be called while holding the lock MappingCallInfo* allocate_call_info(MappingCallKind kind, @@ -745,6 +791,25 @@ namespace Legion { T3 *const arg3; }; + template + class MapperContinuation4 : public MapperContinuation { + public: + MapperContinuation4(MapperManager *man, T1 *a1, T2 *a2, T3 *a3, T4 *a4, + MappingCallInfo *info) + : MapperContinuation(man, info), + arg1(a1), arg2(a2), arg3(a3), arg4(a4) { } + public: + virtual void execute(void) + { (manager->*CALL)(arg1, arg2, arg3, arg4, info); } + public: + T1 *const arg1; + T2 *const arg2; + T3 *const arg3; + T4 *const arg4; + }; + }; }; diff --git a/runtime/legion/region_tree.cc b/runtime/legion/region_tree.cc index 3b541a3cea..09750ec2b0 100644 --- a/runtime/legion/region_tree.cc +++ b/runtime/legion/region_tree.cc @@ -25,6 +25,7 @@ #include "legion/legion_views.h" #include "legion/legion_analysis.h" #include "legion/legion_trace.h" +#include "legion/legion_replication.h" // templates in legion/region_tree.inl are instantiated by region_tree_tmpl.cc @@ -92,19 +93,22 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpaceNode* RegionTreeForest::create_index_space(IndexSpace handle, - const Domain *domain, DistributedID did, + const Domain *domain, DistributedID did, + const bool notify_remote, IndexSpaceExprID expr_id, ApEvent ready /*=ApEvent::NO_AP_EVENT*/, - std::set *applied /*=NULL*/) + RtEvent init /*= RtEvent::NO_RT_EVENT*/, + std::set *applied /*= NULL*/) //-------------------------------------------------------------------------- { - return create_node(handle, domain, true/*domain*/, NULL/*parent*/, - 0/*color*/, did, RtEvent::NO_RT_EVENT, ready, 0/*expr id*/, applied); + return create_node(handle, domain, true/*is domain*/, NULL/*parent*/, + 0/*color*/, did, init, ready, expr_id, notify_remote, applied); } //-------------------------------------------------------------------------- IndexSpaceNode* RegionTreeForest::create_union_space(IndexSpace handle, DistributedID did, const std::vector &sources, - RtEvent initialized, std::set *applied /*=NULL*/) + RtEvent initialized, const bool notify_remote, + IndexSpaceExprID expr_id, std::set *applied) //-------------------------------------------------------------------------- { // Construct the set of index space expressions @@ -120,14 +124,17 @@ namespace Legion { assert(!exprs.empty()); #endif IndexSpaceExpression *expr = union_index_spaces(exprs); - return expr->create_node(handle, did, initialized, applied); + return expr->create_node(handle, did, initialized, applied, + notify_remote, expr_id); } //-------------------------------------------------------------------------- IndexSpaceNode* RegionTreeForest::create_intersection_space( - IndexSpace handle, DistributedID did, - const std::vector &sources, - RtEvent init, std::set *applied) + IndexSpace handle, DistributedID did, + const std::vector &sources, + RtEvent initialized, const bool notify_remote, + IndexSpaceExprID expr_id, + std::set *applied) //-------------------------------------------------------------------------- { // Construct the set of index space expressions @@ -143,14 +150,17 @@ namespace Legion { assert(!exprs.empty()); #endif IndexSpaceExpression *expr = intersect_index_spaces(exprs); - return expr->create_node(handle, did, init, applied); + return expr->create_node(handle, did, initialized, applied, + notify_remote, expr_id); } //-------------------------------------------------------------------------- IndexSpaceNode* RegionTreeForest::create_difference_space( - IndexSpace handle, DistributedID did, - IndexSpace left, IndexSpace right, - RtEvent init, std::set *applied) + IndexSpace handle, DistributedID did, + IndexSpace left, IndexSpace right, + RtEvent initialized, const bool notify_remote, + IndexSpaceExprID expr_id, + std::set *applied) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -158,10 +168,29 @@ namespace Legion { #endif IndexSpaceNode *lhs = get_node(left); if (!right.exists()) - return lhs->create_node(handle, did, init, applied); + return lhs->create_node(handle, did, initialized,applied,notify_remote); IndexSpaceNode *rhs = get_node(right); IndexSpaceExpression *expr = subtract_index_spaces(lhs, rhs); - return expr->create_node(handle, did, init, applied); + return expr->create_node(handle, did, initialized, applied, + notify_remote, expr_id); + } + + //-------------------------------------------------------------------------- + void RegionTreeForest::find_or_create_sharded_index_space(TaskContext *ctx, + IndexSpace handle, IndexSpace local, + DistributedID did) + //-------------------------------------------------------------------------- + { + // Quick unsafe test to see if we already have it + // in which case we can skip the rest of this + if (has_node(handle)) + return; + IndexSpaceNode *local_node = get_node(local); + local_node->create_sharded_alias(handle, did); + if (ctx != NULL) + ctx->register_index_space_creation(handle); + if (runtime->legion_spy_enabled) + LegionSpy::log_top_index_space(handle.get_id()); } //-------------------------------------------------------------------------- @@ -173,8 +202,7 @@ namespace Legion { PartitionKind part_kind, DistributedID did, ApEvent partition_ready, - ApUserEvent partial_pending, - std::set *applied) + ApBarrier partial_pending) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -204,20 +232,26 @@ namespace Legion { runtime->send_index_partition_notification(parent_owner, rez); parent_notified = notified_event; } - IndexPartNode *node = NULL; - RtUserEvent disjointness_event; + std::set applied; if ((part_kind == LEGION_COMPUTE_KIND) || (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) { + RtUserEvent disjointness_event = Runtime::create_rt_user_event(); // Use 1 if we know it's complete, 0 if it's not, // otherwise -1 since we don't know const int complete = (part_kind == LEGION_COMPUTE_COMPLETE_KIND) ? 1 : (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND) ? 0 : -1; - disjointness_event = Runtime::create_rt_user_event(); - node = create_node(pid, parent_node, color_node, partition_color, - disjointness_event, complete, did, partition_ready, - partial_pending, RtEvent::NO_RT_EVENT, applied); + IndexPartNode *node = create_node(pid, parent_node, color_node, + partition_color, disjointness_event, complete, did, partition_ready, + partial_pending, RtEvent::NO_RT_EVENT, NULL, &applied); + IndexPartNode::DisjointnessArgs args(pid, NULL, true/*owner*/); + // Get a reference for the node to hold until disjointness is computed + node->add_base_resource_ref(APPLICATION_REF); + Runtime::trigger_event(disjointness_event, + runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY, + Runtime::protect_event(partition_ready))); } else { @@ -230,9 +264,9 @@ namespace Legion { (part_kind == LEGION_ALIASED_COMPLETE_KIND)) ? 1 : ((part_kind == LEGION_DISJOINT_INCOMPLETE_KIND) || (part_kind == LEGION_ALIASED_INCOMPLETE_KIND)) ? 0 : -1; - node = create_node(pid, parent_node, color_node, partition_color, - disjoint, complete, did, partition_ready, partial_pending, - RtEvent::NO_RT_EVENT, applied); + create_node(pid, parent_node, color_node, partition_color, disjoint, + complete, did, partition_ready, partial_pending, + RtEvent::NO_RT_EVENT, NULL, &applied); if (runtime->legion_spy_enabled) LegionSpy::log_index_partition(parent.id, pid.id, disjoint, partition_color); @@ -240,24 +274,13 @@ namespace Legion { runtime->profiler->record_index_partition(parent.id,pid.id, disjoint, partition_color); } - // If we need to compute the disjointness, only do that - // after the partition is actually ready - if ((part_kind == LEGION_COMPUTE_KIND) || - (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || - (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + ctx->register_index_partition_creation(pid); + if (!applied.empty()) { -#ifdef DEBUG_LEGION - assert(disjointness_event.exists()); -#endif - // Record a reference on this node to prevent it from being - // collected until the disjointness test is done - node->add_base_resource_ref(APPLICATION_REF); - // Launch a task to compute the disjointness - DisjointnessArgs args(pid, disjointness_event); - runtime->issue_runtime_meta_task(args, LG_LATENCY_WORK_PRIORITY, - Runtime::protect_event(partition_ready)); + if (parent_notified.exists()) + applied.insert(parent_notified); + return Runtime::merge_events(applied); } - ctx->register_index_partition_creation(pid); return parent_notified; } @@ -269,7 +292,9 @@ namespace Legion { PartitionKind kind, LegionColor &part_color, ApEvent domain_ready, - std::set &safe_events) + std::set &safe_events, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *base = get_node(handle1); @@ -390,10 +415,10 @@ namespace Legion { // Iterate over all our sub-regions and generate partitions if (!children_nodes.empty()) { - for (std::vector::const_iterator it = - children_nodes.begin(); it != children_nodes.end(); it++) + for (unsigned idx = shard; + idx < children_nodes.size(); idx += total_shards) { - IndexSpaceNode *child_node = *it; + IndexSpaceNode *child_node = children_nodes[idx]; IndexPartition pid(runtime->get_unique_index_partition_id(), handle1.get_tree_id(), handle1.get_type_tag()); DistributedID did = @@ -401,9 +426,7 @@ namespace Legion { const RtEvent safe = create_pending_partition(ctx, pid, child_node->handle, source->color_space->handle, - part_color, kind, did, domain_ready, - ApUserEvent::NO_AP_USER_EVENT, - &safe_events); + part_color, kind, did, domain_ready); // If the user requested the handle for this point return it std::map::iterator finder = user_handles.find(child_node->handle); @@ -415,7 +438,8 @@ namespace Legion { } else if (base->total_children == base->max_linearized_color) { - for (LegionColor color = 0; color < base->total_children; color++) + for (LegionColor color = shard; + color < base->total_children; color += total_shards) { IndexSpaceNode *child_node = base->get_child(color); IndexPartition pid(runtime->get_unique_index_partition_id(), @@ -425,9 +449,7 @@ namespace Legion { const RtEvent safe = create_pending_partition(ctx, pid, child_node->handle, source->color_space->handle, - part_color, kind, did, domain_ready, - ApUserEvent::NO_AP_USER_EVENT, - &safe_events); + part_color, kind, did, domain_ready); // If the user requested the handle for this point return it std::map::iterator finder = user_handles.find(child_node->handle); @@ -441,6 +463,13 @@ namespace Legion { { ColorSpaceIterator *itr = base->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } while (itr->is_valid()) { const LegionColor color = itr->yield_color(); @@ -452,14 +481,19 @@ namespace Legion { const RtEvent safe = create_pending_partition(ctx, pid, child_node->handle, source->color_space->handle, - part_color, kind, did, domain_ready, - ApUserEvent::NO_AP_USER_EVENT, - &safe_events); + part_color, kind, did, domain_ready); // If the user requested the handle for this point return it std::map::iterator finder = user_handles.find(child_node->handle); if (finder != user_handles.end()) finder->second = pid; + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } if (safe.exists()) safe_events.insert(safe); } @@ -468,20 +502,201 @@ namespace Legion { } //-------------------------------------------------------------------------- - void RegionTreeForest::compute_partition_disjointness(IndexPartition handle, - RtUserEvent ready_event) + RtEvent RegionTreeForest::create_pending_partition_shard( + ShardID owner_shard, + ReplicateContext *ctx, + IndexPartition pid, + IndexSpace parent, + IndexSpace color_space, + LegionColor &partition_color, + PartitionKind part_kind, + DistributedID did, + ValueBroadcast *part_result, + ApEvent partition_ready, + ShardMapping &mapping, + RtEvent creation_ready, + ApBarrier partial_pending) //-------------------------------------------------------------------------- { - IndexPartNode *node = get_node(handle); - node->compute_disjointness(ready_event); - // Remove the reference that we added when launching this task - if (node->remove_base_resource_ref(APPLICATION_REF)) - delete node; +#ifdef DEBUG_LEGION + if (partial_pending.exists()) + assert(partition_ready == partial_pending); +#endif + if (owner_shard == ctx->owner_shard->shard_id) + { + // We're the owner so we do most of the work + IndexSpaceNode *parent_node = get_node(parent); + IndexSpaceNode *color_node = get_node(color_space); + if (partition_color == INVALID_COLOR) + partition_color = parent_node->generate_color(); + // If we are making this partition on a different node than the + // owner node of the parent index space then we have to tell that + // owner node about the existence of this partition + RtEvent parent_notified; + const AddressSpaceID parent_owner = parent_node->get_owner_space(); + if (parent_owner != runtime->address_space) + { + RtUserEvent notified_event = Runtime::create_rt_user_event(); + Serializer rez; + { + RezCheck z(rez); + rez.serialize(pid); + rez.serialize(parent); + rez.serialize(partition_color); + rez.serialize(notified_event); + } + runtime->send_index_partition_notification(parent_owner, rez); + parent_notified = notified_event; + } + RtUserEvent disjointness_event; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + { +#ifdef DEBUG_LEGION + assert(part_result != NULL); +#endif + disjointness_event = Runtime::create_rt_user_event(); + } +#ifdef DEBUG_LEGION + else + assert(part_result == NULL); +#endif + IndexPartNode *part_node; + std::set applied; + if ((part_kind != LEGION_COMPUTE_KIND) && + (part_kind != LEGION_COMPUTE_COMPLETE_KIND) && + (part_kind != LEGION_COMPUTE_INCOMPLETE_KIND)) + { + const bool disjoint = (part_kind == LEGION_DISJOINT_KIND) || + (part_kind == LEGION_DISJOINT_COMPLETE_KIND) || + (part_kind == LEGION_DISJOINT_INCOMPLETE_KIND); + // Use 1 if we know it's complete, 0 if it's not, + // otherwise -1 since we don't know + const int complete = ((part_kind == LEGION_DISJOINT_COMPLETE_KIND) || + (part_kind == LEGION_ALIASED_COMPLETE_KIND)) ? 1 : + ((part_kind == LEGION_DISJOINT_INCOMPLETE_KIND) || + (part_kind == LEGION_ALIASED_INCOMPLETE_KIND)) ? 0 :-1; + part_node = create_node(pid, parent_node, color_node, partition_color, + disjoint, complete, did, partition_ready, partial_pending, + creation_ready, &mapping, &applied); + if (runtime->legion_spy_enabled) + LegionSpy::log_index_partition(parent.id, pid.id, disjoint, + partition_color); + if (runtime->profiler != NULL) + runtime->profiler->record_index_partition(parent.id,pid.id,disjoint, + partition_color); + } + else + { + // Use 1 if we know it's complete, 0 if it's not, + // otherwise -1 since we don't know + const int complete = (part_kind == LEGION_COMPUTE_COMPLETE_KIND) ? 1 : + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND) ? 0 : -1; + part_node = create_node(pid, parent_node, color_node, partition_color, + disjointness_event, complete, did, + partition_ready, partial_pending, + creation_ready, &mapping, &applied); + } + part_node->update_creation_set(mapping); + if (disjointness_event.exists()) + { + IndexPartNode::DisjointnessArgs args(pid, part_result, true/*owner*/); + // Hold a reference on the node until disjointness is performed + part_node->add_base_resource_ref(APPLICATION_REF); + // Don't do the disjointness test until all the partition + // is ready and has been created on all the nodes + Runtime::trigger_event(disjointness_event, + runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY, + Runtime::merge_events(creation_ready, + Runtime::protect_event(partition_ready)))); + } + ctx->register_index_partition_creation(pid); + if (!applied.empty()) + { + if (parent_notified.exists()) + applied.insert(parent_notified); + return Runtime::merge_events(applied); + } + return parent_notified; + } + else + { +#ifdef DEBUG_LEGION + assert(partition_color != INVALID_COLOR); +#endif + // We're not the owner so we just do basic setup work + IndexSpaceNode *parent_node = get_node(parent); + IndexSpaceNode *color_node = get_node(color_space); + RtUserEvent disjointness_event; + if ((part_kind == LEGION_COMPUTE_KIND) || + (part_kind == LEGION_COMPUTE_COMPLETE_KIND) || + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND)) + { +#ifdef DEBUG_LEGION + assert(part_result != NULL); +#endif + disjointness_event = Runtime::create_rt_user_event(); + } +#ifdef DEBUG_LEGION + else + assert(part_result == NULL); +#endif + IndexPartNode *part_node; + std::set applied; + if ((part_kind != LEGION_COMPUTE_KIND) && + (part_kind != LEGION_COMPUTE_COMPLETE_KIND) && + (part_kind != LEGION_COMPUTE_INCOMPLETE_KIND)) + { + const bool disjoint = (part_kind == LEGION_DISJOINT_KIND) || + (part_kind == LEGION_DISJOINT_COMPLETE_KIND) || + (part_kind == LEGION_DISJOINT_INCOMPLETE_KIND); + // Use 1 if we know it's complete, 0 if it's not, + // otherwise -1 since we don't know + const int complete = ((part_kind == LEGION_DISJOINT_COMPLETE_KIND) || + (part_kind == LEGION_ALIASED_COMPLETE_KIND)) ? 1 : + ((part_kind == LEGION_DISJOINT_INCOMPLETE_KIND) || + (part_kind == LEGION_ALIASED_INCOMPLETE_KIND)) ? 0 : -1; + part_node = create_node(pid, parent_node, color_node, partition_color, + disjoint, complete, did, partition_ready, partial_pending, + creation_ready, &mapping, &applied); + } + else + { + // Use 1 if we know it's complete, 0 if it's not, + // otherwise -1 since we don't know + const int complete = (part_kind == LEGION_COMPUTE_COMPLETE_KIND) ? 1 : + (part_kind == LEGION_COMPUTE_INCOMPLETE_KIND) ? 0 : -1; + part_node = create_node(pid, parent_node, color_node, partition_color, + disjointness_event, complete, did, + partition_ready, partial_pending, + creation_ready, &mapping, &applied); + } + part_node->update_creation_set(mapping); + if (disjointness_event.exists()) + { + IndexPartNode::DisjointnessArgs args(pid, part_result,false/*owner*/); + // Hold a reference on the node until disjointness is performed + part_node->add_base_resource_ref(APPLICATION_REF); + // We only need to wait for the creation to be ready + // if we're not the owner + Runtime::trigger_event(disjointness_event, + runtime->issue_runtime_meta_task(args, + LG_THROUGHPUT_DEFERRED_PRIORITY, creation_ready)); + } + ctx->register_index_partition_creation(pid); + // We know the parent is notified or we wouldn't even have + // been given our pid + if (!applied.empty()) + return Runtime::merge_events(applied); + return RtEvent::NO_RT_EVENT; + } } //-------------------------------------------------------------------------- void RegionTreeForest::destroy_index_space(IndexSpace handle, - std::set &applied) + std::set &applied, const bool total_sharding_collective) //-------------------------------------------------------------------------- { const AddressSpaceID owner_space = @@ -493,13 +708,13 @@ namespace Legion { if (node->remove_base_valid_ref(APPLICATION_REF, &mutator)) delete node; } - else + else if (!total_sharding_collective) runtime->send_index_space_destruction(handle, owner_space, applied); } //-------------------------------------------------------------------------- void RegionTreeForest::destroy_index_partition(IndexPartition handle, - std::set &applied) + std::set &applied, const bool total_sharding_collective) //-------------------------------------------------------------------------- { const AddressSpaceID owner_space = @@ -511,110 +726,134 @@ namespace Legion { if (node->remove_base_valid_ref(APPLICATION_REF, &mutator)) delete node; } - else + else if (!total_sharding_collective) runtime->send_index_partition_destruction(handle, owner_space, applied); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_equal_partition(Operation *op, IndexPartition pid, - size_t granularity) + size_t granularity, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *new_part = get_node(pid); - return new_part->create_equal_children(op, granularity); + return new_part->create_equal_children(op, granularity, + shard, total_shards); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_partition_by_weights(Operation *op, IndexPartition pid, const FutureMap &weights, - size_t granularity) + size_t granularity, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *new_part = get_node(pid); - return new_part->create_by_weights(op, weights, granularity); + return new_part->create_by_weights(op, weights, granularity, + shard, total_shards); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_partition_by_union(Operation *op, IndexPartition pid, IndexPartition handle1, - IndexPartition handle2) + IndexPartition handle2, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *new_part = get_node(pid); IndexPartNode *node1 = get_node(handle1); IndexPartNode *node2 = get_node(handle2); - return new_part->create_by_union(op, node1, node2); + return new_part->create_by_union(op, node1, node2, shard, total_shards); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_partition_by_intersection(Operation *op, IndexPartition pid, IndexPartition handle1, - IndexPartition handle2) + IndexPartition handle2, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *new_part = get_node(pid); IndexPartNode *node1 = get_node(handle1); IndexPartNode *node2 = get_node(handle2); - return new_part->create_by_intersection(op, node1, node2); + return new_part->create_by_intersection(op, node1, node2, + shard, total_shards); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_partition_by_intersection(Operation *op, IndexPartition pid, IndexPartition part, - const bool dominates) + const bool dominates, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *new_part = get_node(pid); IndexPartNode *node = get_node(part); - return new_part->create_by_intersection(op, node, dominates); + return new_part->create_by_intersection(op, node, dominates, + shard, total_shards); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_partition_by_difference(Operation *op, IndexPartition pid, IndexPartition handle1, - IndexPartition handle2) + IndexPartition handle2, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *new_part = get_node(pid); IndexPartNode *node1 = get_node(handle1); IndexPartNode *node2 = get_node(handle2); - return new_part->create_by_difference(op, node1, node2); + return new_part->create_by_difference(op, node1, node2, + shard, total_shards); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_partition_by_restriction( IndexPartition pid, const void *transform, - const void *extent) + const void *extent, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *new_part = get_node(pid); - return new_part->create_by_restriction(transform, extent); + return new_part->create_by_restriction(transform, extent, + shard, total_shards); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_partition_by_domain(Operation *op, IndexPartition pid, const FutureMap &future_map, - bool perform_intersections) + bool perform_intersections, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *new_part = get_node(pid); return new_part->parent->create_by_domain(op, new_part, future_map.impl, - perform_intersections); + perform_intersections, shard, total_shards); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::create_cross_product_partitions(Operation *op, IndexPartition base, IndexPartition source, - LegionColor part_color) + LegionColor part_color, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *base_node = get_node(base); @@ -622,12 +861,13 @@ namespace Legion { std::set ready_events; if (base_node->total_children == base_node->max_linearized_color) { - for (LegionColor color = 0; color < base_node->total_children; color++) + for (LegionColor color = shard; + color < base_node->total_children; color+=total_shards) { IndexSpaceNode *child_node = base_node->get_child(color); IndexPartNode *part_node = child_node->get_child(part_color); - ApEvent ready = child_node->create_by_intersection(op, part_node, - source_node); + ApEvent ready = + child_node->create_by_intersection(op, part_node, source_node); ready_events.insert(ready); } } @@ -635,14 +875,28 @@ namespace Legion { { ColorSpaceIterator *itr = base_node->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } while (itr->is_valid()) { const LegionColor color = itr->yield_color(); IndexSpaceNode *child_node = base_node->get_child(color); IndexPartNode *part_node = child_node->get_child(part_color); - ApEvent ready = child_node->create_by_intersection(op, part_node, - source_node); + ApEvent ready = + child_node->create_by_intersection(op, part_node, source_node); ready_events.insert(ready); + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } } delete itr; } @@ -666,13 +920,15 @@ namespace Legion { IndexPartition pending, IndexPartition proj, const std::vector &instances, - ApEvent instances_ready) + ApEvent instances_ready, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *partition = get_node(pending); IndexPartNode *projection = get_node(proj); return partition->parent->create_by_image(op, partition, projection, - instances, instances_ready); + instances, instances_ready, shard, total_shards); } //-------------------------------------------------------------------------- @@ -680,13 +936,15 @@ namespace Legion { IndexPartition pending, IndexPartition proj, const std::vector &instances, - ApEvent instances_ready) + ApEvent instances_ready, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { IndexPartNode *partition = get_node(pending); IndexPartNode *projection = get_node(proj); return partition->parent->create_by_image_range(op, partition, projection, - instances, instances_ready); + instances, instances_ready, shard, total_shards); } //-------------------------------------------------------------------------- @@ -757,48 +1015,67 @@ namespace Legion { return get_node(is)->check_field_size(field_size, false/*is range*/); } - //-------------------------------------------------------------------------- - IndexSpace RegionTreeForest::find_pending_space(IndexPartition parent, - const void *realm_color, - TypeTag type_tag, - ApUserEvent &domain_ready) - //-------------------------------------------------------------------------- - { - IndexPartNode *parent_node = get_node(parent); - LegionColor child_color = - parent_node->color_space->linearize_color(realm_color, type_tag); - IndexSpaceNode *child_node = parent_node->get_child(child_color); - if (!parent_node->get_pending_child(child_color, domain_ready)) - REPORT_LEGION_ERROR(ERROR_INVALID_PENDING_CHILD, - "Invalid pending child!") - return child_node->handle; - } - //-------------------------------------------------------------------------- ApEvent RegionTreeForest::compute_pending_space(Operation *op, - IndexSpace target, const std::vector &handles, bool is_union) + IndexSpace target, const std::vector &handles, bool is_union, + ShardID shard, size_t total_shards) //-------------------------------------------------------------------------- { IndexSpaceNode *child_node = get_node(target); + // Check to see if we own this child or not + if ((total_shards > 1) && ((child_node->color % total_shards) != shard)) + return ApEvent::NO_AP_EVENT; + // Convert the ap event for the space into an ap user event and + // trigger it once the operation is complete + ApUserEvent space_ready = *(reinterpret_cast( + const_cast(&child_node->index_space_ready))); + if (space_ready.has_triggered()) + REPORT_LEGION_ERROR(ERROR_INVALID_PENDING_CHILD, + "Invalid pending child!") + Runtime::trigger_event(NULL, space_ready, op->get_completion_event()); return child_node->compute_pending_space(op, handles, is_union); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::compute_pending_space(Operation *op, - IndexSpace target, IndexPartition handle, bool is_union) + IndexSpace target, IndexPartition handle, bool is_union, + ShardID shard, size_t total_shards) //-------------------------------------------------------------------------- { IndexSpaceNode *child_node = get_node(target); + // Check to see if we own this child or not + if ((total_shards > 1) && ((child_node->color % total_shards) != shard)) + return ApEvent::NO_AP_EVENT; + // Convert the ap event for the space into an ap user event and + // trigger it once the operation is complete + ApUserEvent space_ready = *(reinterpret_cast( + const_cast(&child_node->index_space_ready))); + if (space_ready.has_triggered()) + REPORT_LEGION_ERROR(ERROR_INVALID_PENDING_CHILD, + "Invalid pending child!") + Runtime::trigger_event(NULL, space_ready, op->get_completion_event()); return child_node->compute_pending_space(op, handle, is_union); } //-------------------------------------------------------------------------- ApEvent RegionTreeForest::compute_pending_space(Operation *op, IndexSpace target, IndexSpace initial, - const std::vector &handles) + const std::vector &handles, + ShardID shard, size_t total_shards) //-------------------------------------------------------------------------- { IndexSpaceNode *child_node = get_node(target); + // Check to see if we own this child or not + if ((total_shards > 1) && ((child_node->color % total_shards) != shard)) + return ApEvent::NO_AP_EVENT; + // Convert the ap event for the space into an ap user event and + // trigger it once the operation is complete + ApUserEvent space_ready = *(reinterpret_cast( + const_cast(&child_node->index_space_ready))); + if (space_ready.has_triggered()) + REPORT_LEGION_ERROR(ERROR_INVALID_PENDING_CHILD, + "Invalid pending child!\n") + Runtime::trigger_event(NULL, space_ready, op->get_completion_event()); return child_node->compute_pending_difference(op, initial, handles); } @@ -1001,16 +1278,21 @@ namespace Legion { } //-------------------------------------------------------------------------- - void RegionTreeForest::create_field_space(FieldSpace handle, - DistributedID did, std::set *applied) + FieldSpaceNode* RegionTreeForest::create_field_space(FieldSpace handle, + DistributedID did, + const bool notify_remote, + RtEvent initialized, + std::set *applied, + ShardMapping *shard_mapping) //-------------------------------------------------------------------------- { - create_node(handle, did, RtEvent::NO_RT_EVENT, applied); + return create_node(handle, did, initialized, notify_remote, + applied, shard_mapping); } //-------------------------------------------------------------------------- void RegionTreeForest::destroy_field_space(FieldSpace handle, - std::set &applied) + std::set &applied, const bool total_sharding_collective) //-------------------------------------------------------------------------- { const AddressSpaceID owner_space = @@ -1022,66 +1304,75 @@ namespace Legion { if (node->remove_base_valid_ref(APPLICATION_REF, &mutator)) delete node; } - else + else if (!total_sharding_collective) runtime->send_field_space_destruction(handle, owner_space, applied); } //-------------------------------------------------------------------------- - RtEvent RegionTreeForest::create_field_space_allocator(FieldSpace handle) + RtEvent RegionTreeForest::create_field_space_allocator(FieldSpace handle, + bool sharded_owner_context, bool owner_shard) //-------------------------------------------------------------------------- { FieldSpaceNode *node = get_node(handle); - return node->create_allocator(runtime->address_space); + return node->create_allocator(runtime->address_space, + RtUserEvent::NO_RT_USER_EVENT, sharded_owner_context, owner_shard); } //-------------------------------------------------------------------------- - void RegionTreeForest::destroy_field_space_allocator(FieldSpace handle) + void RegionTreeForest::destroy_field_space_allocator(FieldSpace handle, + bool sharded_owner_context, bool owner_shard) //-------------------------------------------------------------------------- { FieldSpaceNode *node = get_node(handle); - const RtEvent ready = node->destroy_allocator(runtime->address_space); + const RtEvent ready = node->destroy_allocator(runtime->address_space, + sharded_owner_context, owner_shard); if (ready.exists() && !ready.has_triggered()) ready.wait(); } //-------------------------------------------------------------------------- - bool RegionTreeForest::allocate_field(FieldSpace handle, size_t field_size, - FieldID fid, CustomSerdezID serdez_id) + RtEvent RegionTreeForest::allocate_field(FieldSpace handle, + size_t field_size, FieldID fid, + CustomSerdezID serdez_id, + bool sharded_non_owner) //-------------------------------------------------------------------------- { FieldSpaceNode *node = get_node(handle); - RtEvent ready = node->allocate_field(fid, field_size, serdez_id); - if (ready.exists()) - ready.wait(); - return false; + RtEvent ready = + node->allocate_field(fid, field_size, serdez_id, sharded_non_owner); + return ready; } //-------------------------------------------------------------------------- FieldSpaceNode* RegionTreeForest::allocate_field(FieldSpace handle, - ApEvent size_ready, FieldID fid, CustomSerdezID serdez_id) + ApEvent size_ready, FieldID fid, CustomSerdezID serdez_id, + RtEvent &precondition, bool sharded_non_owner) //-------------------------------------------------------------------------- { FieldSpaceNode *node = get_node(handle); - RtEvent ready = node->allocate_field(fid, size_ready, serdez_id); - if (ready.exists()) - ready.wait(); + precondition = + node->allocate_field(fid, size_ready, serdez_id, sharded_non_owner); return node; } //-------------------------------------------------------------------------- void RegionTreeForest::free_field(FieldSpace handle, FieldID fid, - std::set &applied) + std::set &applied, + bool sharded_non_owner) //-------------------------------------------------------------------------- { + if (!has_node(handle)) + return; FieldSpaceNode *node = get_node(handle); - node->free_field(fid, runtime->address_space, applied); + node->free_field(fid, runtime->address_space, applied, sharded_non_owner); } //-------------------------------------------------------------------------- - void RegionTreeForest::allocate_fields(FieldSpace handle, - const std::vector &sizes, - const std::vector &fields, - CustomSerdezID serdez_id) + RtEvent RegionTreeForest::allocate_fields(FieldSpace handle, + const std::vector &sizes, + const std::vector &fields, + CustomSerdezID serdez_id, + bool sharded_non_owner) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -1089,36 +1380,39 @@ namespace Legion { #endif // We know that none of these field allocations are local FieldSpaceNode *node = get_node(handle); - RtEvent ready = node->allocate_fields(sizes, fields, serdez_id); - // Wait for this to exist - if (ready.exists()) - ready.wait(); + RtEvent ready = + node->allocate_fields(sizes, fields, serdez_id, sharded_non_owner); + return ready; } //-------------------------------------------------------------------------- FieldSpaceNode* RegionTreeForest::allocate_fields(FieldSpace handle, ApEvent sizes_ready, const std::vector &fields, - CustomSerdezID serdez_id) + CustomSerdezID serdez_id, + RtEvent &precondition, + bool sharded_non_owner) //-------------------------------------------------------------------------- { // We know that none of these field allocations are local FieldSpaceNode *node = get_node(handle); - RtEvent ready = node->allocate_fields(sizes_ready, fields, serdez_id); - // Wait for this to exist - if (ready.exists()) - ready.wait(); + precondition = + node->allocate_fields(sizes_ready, fields, serdez_id,sharded_non_owner); return node; } //-------------------------------------------------------------------------- void RegionTreeForest::free_fields(FieldSpace handle, const std::vector &to_free, - std::set &applied) + std::set &applied, + bool sharded_non_owner) //-------------------------------------------------------------------------- { + if (!has_node(handle)) + return; FieldSpaceNode *node = get_node(handle); - node->free_fields(to_free, runtime->address_space, applied); + node->free_fields(to_free, runtime->address_space, applied, + sharded_non_owner); } //-------------------------------------------------------------------------- @@ -1138,11 +1432,14 @@ namespace Legion { //-------------------------------------------------------------------------- void RegionTreeForest::free_local_fields(FieldSpace handle, const std::vector &to_free, - const std::vector &indexes) + const std::vector &indexes, + const bool collective) //-------------------------------------------------------------------------- { + if (collective && !has_node(handle)) + return; FieldSpaceNode *node = get_node(handle); - node->free_local_fields(to_free, indexes); + node->free_local_fields(to_free, indexes, collective); } //-------------------------------------------------------------------------- @@ -1187,16 +1484,19 @@ namespace Legion { } //-------------------------------------------------------------------------- - void RegionTreeForest::create_logical_region(LogicalRegion handle, - std::set *applied) + RegionNode* RegionTreeForest::create_logical_region(LogicalRegion handle, + const bool notify_remote, + RtEvent initialized, + std::set *applied) //-------------------------------------------------------------------------- { - create_node(handle, NULL/*parent*/, RtEvent::NO_RT_EVENT, applied); + return create_node(handle, NULL/*parent*/, initialized, + notify_remote, applied); } //-------------------------------------------------------------------------- void RegionTreeForest::destroy_logical_region(LogicalRegion handle, - std::set &applied) + std::set &applied, const bool total_sharding_collective) //-------------------------------------------------------------------------- { const AddressSpaceID owner_space = @@ -1208,7 +1508,7 @@ namespace Legion { if (node->remove_base_valid_ref(APPLICATION_REF, &mutator)) delete node; } - else + else if (!total_sharding_collective) runtime->send_logical_region_destruction(handle, owner_space, applied); } @@ -1823,7 +2123,8 @@ namespace Legion { const bool track_effects, const bool record_valid, const bool check_initialized, - const bool defer_copies) + const bool defer_copies, + const bool skip_output) //-------------------------------------------------------------------------- { DETAILED_PROFILER(runtime, REGION_TREE_PHYSICAL_REGISTER_ONLY_CALL); @@ -1862,8 +2163,8 @@ namespace Legion { analysis = new UpdateAnalysis(runtime, op, index, version_info, req, region_node, targets, target_views, trace_info, precondition, term_event, - track_effects, check_initialized, - record_valid); + track_effects, check_initialized, + record_valid, skip_output); analysis->add_reference(); // Iterate over all the equivalence classes and perform the analysis // Only need to check for uninitialized data for things not discarding @@ -1920,8 +2221,8 @@ namespace Legion { // Record the event as the precondition for the task targets[idx].set_ready_event(ready); if (trace_info.recording) - trace_info.record_op_view( - analysis->usage, inst_mask, analysis->target_views[idx]); + trace_info.record_op_view(analysis->usage, inst_mask, + analysis->target_views[idx], map_applied_events); } if (!user_applied.empty()) { @@ -1944,8 +2245,8 @@ namespace Legion { // Record the event as the precondition for the task targets[idx].set_ready_event(ready); if (trace_info.recording) - trace_info.record_op_view( - analysis->usage, inst_mask, analysis->target_views[idx]); + trace_info.record_op_view(analysis->usage, inst_mask, + analysis->target_views[idx], map_applied_events); } } } @@ -2875,6 +3176,55 @@ namespace Legion { return result; } + //-------------------------------------------------------------------------- + ApEvent RegionTreeForest::overwrite_sharded(Operation *op, + const unsigned index, + const RegionRequirement &req, + ShardedView *view, + VersionInfo &version_info, + const PhysicalTraceInfo &trace_info, + const ApEvent precondition, + std::set &map_applied_events, + const bool add_restriction) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(req.handle_type == LEGION_SINGULAR_PROJECTION); +#endif + if (IS_NO_ACCESS(req) || req.privilege_fields.empty()) + return ApEvent::NO_AP_EVENT; + RegionNode *region_node = get_node(req.region); + FieldMask overwrite_mask = + region_node->column_source->get_field_mask(req.privilege_fields); + const FieldMaskSet &eq_sets = + version_info.get_equivalence_sets(); + OverwriteAnalysis *analysis = new OverwriteAnalysis(runtime, op, index, + req, version_info, view, trace_info, precondition, + RtEvent::NO_RT_EVENT, PredEvent::NO_PRED_EVENT, + true/*track effects*/, add_restriction); + analysis->add_reference(); + std::set deferral_events; + for (FieldMaskSet::const_iterator it = + eq_sets.begin(); it != eq_sets.end(); it++) + analysis->traverse(it->first, it->second, deferral_events, + map_applied_events, true/*cached set*/); + const RtEvent traversal_done = deferral_events.empty() ? + RtEvent::NO_RT_EVENT : Runtime::merge_events(deferral_events); + RtEvent remote_ready; + if (traversal_done.exists() || analysis->has_remote_sets()) + remote_ready = + analysis->perform_remote(traversal_done, map_applied_events); + RtEvent output_ready; + if (traversal_done.exists() || analysis->has_output_updates()) + output_ready = + analysis->perform_updates(traversal_done, map_applied_events); + const ApEvent result = analysis->perform_output( + Runtime::merge_events(remote_ready, output_ready), map_applied_events); + if (analysis->remove_reference()) + delete analysis; + return result; + } + //-------------------------------------------------------------------------- InstanceRef RegionTreeForest::create_external_instance( AttachOp *attach_op, const RegionRequirement &req, @@ -2999,7 +3349,8 @@ namespace Legion { void RegionTreeForest::invalidate_fields(Operation *op, unsigned index, VersionInfo &version_info, const PhysicalTraceInfo &trace_info, - std::set &map_applied_events) + std::set &map_applied_events, + const bool collective) //-------------------------------------------------------------------------- { const FieldMaskSet &eq_sets = @@ -3009,10 +3360,26 @@ namespace Legion { usage, version_info, NULL/*view*/, trace_info, ApEvent::NO_AP_EVENT); analysis->add_reference(); std::set deferral_events; - for (FieldMaskSet::const_iterator it = - eq_sets.begin(); it != eq_sets.end(); it++) - analysis->traverse(it->first, it->second, deferral_events, - map_applied_events, true/*original set*/); + if (collective) + { + for (FieldMaskSet::const_iterator it = + eq_sets.begin(); it != eq_sets.end(); it++) + { + // Skip any that are not ones that we own, they will be handled + // by a a remote node + if (!it->first->is_owner()) + continue; + analysis->traverse(it->first, it->second, deferral_events, + map_applied_events, true/*cached set*/); + } + } + else + { + for (FieldMaskSet::const_iterator it = + eq_sets.begin(); it != eq_sets.end(); it++) + analysis->traverse(it->first, it->second, deferral_events, + map_applied_events, true/*cached set*/); + } const RtEvent traversal_done = deferral_events.empty() ? RtEvent::NO_RT_EVENT : Runtime::merge_events(deferral_events); if (traversal_done.exists() || analysis->has_remote_sets()) @@ -3374,6 +3741,7 @@ namespace Legion { RtEvent initialized, ApEvent is_ready, IndexSpaceExprID expr_id, + const bool notify_remote, std::set *applied) //-------------------------------------------------------------------------- { @@ -3424,7 +3792,7 @@ namespace Legion { result->add_base_valid_ref(APPLICATION_REF, &mutator); else result->add_nested_valid_ref(parent->did, &mutator); - result->register_with_runtime(&mutator); + result->register_with_runtime(&mutator, notify_remote); if (parent != NULL) parent->add_child(result); } @@ -3453,6 +3821,7 @@ namespace Legion { DistributedID did, RtEvent initialized, ApUserEvent is_ready, + const bool notify_remote, std::set *applied) //-------------------------------------------------------------------------- { @@ -3505,7 +3874,7 @@ namespace Legion { result->add_base_valid_ref(APPLICATION_REF, &mutator); else result->add_nested_valid_ref(parent->did, &mutator); - result->register_with_runtime(&mutator); + result->register_with_runtime(&mutator, notify_remote); if (parent != NULL) parent->add_child(result); } @@ -3534,8 +3903,9 @@ namespace Legion { bool disjoint, int complete, DistributedID did, ApEvent part_ready, - ApUserEvent pending, + ApBarrier pending, RtEvent initialized, + ShardMapping *shard_mapping, std::set *applied) //-------------------------------------------------------------------------- { @@ -3549,8 +3919,8 @@ namespace Legion { local_applied.insert(initialized); initialized = local_initialized; } - IndexPartCreator creator(this, p, parent, color_space, color, disjoint, - complete, did, part_ready, pending, initialized); + IndexPartCreator creator(this, p, parent, color_space, color, disjoint, + complete, did, part_ready, pending, initialized, shard_mapping); NT_TemplateHelper::demux(p.get_type_tag(), &creator); IndexPartNode *result = creator.result; #ifdef DEBUG_LEGION @@ -3586,7 +3956,10 @@ namespace Legion { result->add_base_valid_ref(APPLICATION_REF, &mutator); else result->add_base_gc_ref(REMOTE_DID_REF, &mutator); - result->register_with_runtime(&mutator); + if (shard_mapping != NULL) + result->register_with_runtime(&mutator, false/*notify remote*/); + else + result->register_with_runtime(&mutator); parent->add_child(result); } if (local_initialized.exists()) @@ -3609,8 +3982,9 @@ namespace Legion { int complete, DistributedID did, ApEvent part_ready, - ApUserEvent pending, + ApBarrier pending, RtEvent initialized, + ShardMapping *shard_mapping, std::set *applied) //-------------------------------------------------------------------------- { @@ -3625,7 +3999,8 @@ namespace Legion { initialized = local_initialized; } IndexPartCreator creator(this, p, parent, color_space, color, - disjointness_ready, complete, did, part_ready, pending, initialized); + disjointness_ready, complete, did, part_ready, + pending, initialized, shard_mapping); NT_TemplateHelper::demux(p.get_type_tag(), &creator); IndexPartNode *result = creator.result; #ifdef DEBUG_LEGION @@ -3661,7 +4036,10 @@ namespace Legion { result->add_base_valid_ref(APPLICATION_REF, &mutator); else result->add_base_gc_ref(REMOTE_DID_REF, &mutator); - result->register_with_runtime(&mutator); + if (shard_mapping != NULL) + result->register_with_runtime(&mutator, false/*notify remote*/); + else + result->register_with_runtime(&mutator); parent->add_child(result); } if (local_initialized.exists()) @@ -3677,8 +4055,11 @@ namespace Legion { //-------------------------------------------------------------------------- FieldSpaceNode* RegionTreeForest::create_node(FieldSpace space, - DistributedID did, RtEvent initialized, - std::set *applied) + DistributedID did, + RtEvent initialized, + const bool notify_remote, + std::set *applied, + ShardMapping *shard_mapping) //-------------------------------------------------------------------------- { RtUserEvent local_initialized; @@ -3691,7 +4072,8 @@ namespace Legion { local_applied.insert(initialized); initialized = local_initialized; } - FieldSpaceNode *result = new FieldSpaceNode(space, this, did,initialized); + FieldSpaceNode *result = + new FieldSpaceNode(space, this, did, initialized, shard_mapping); #ifdef DEBUG_LEGION assert(result != NULL); assert(applied != NULL); @@ -3721,7 +4103,7 @@ namespace Legion { // safely collected if (result->is_owner()) result->add_base_valid_ref(APPLICATION_REF, &mutator); - result->register_with_runtime(&mutator); + result->register_with_runtime(&mutator, notify_remote); } if (local_initialized.exists()) { @@ -3788,8 +4170,10 @@ namespace Legion { //-------------------------------------------------------------------------- RegionNode* RegionTreeForest::create_node(LogicalRegion r, - PartitionNode *parent, RtEvent initialized, - std::set *applied) + PartitionNode *parent, + RtEvent initialized, + const bool notify_remote, + std::set *applied) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -3980,7 +4364,7 @@ namespace Legion { //-------------------------------------------------------------------------- IndexSpaceNode* RegionTreeForest::get_node(IndexSpace space, - RtEvent *defer /*=NULL*/) + RtEvent *defer /*=NULL*/, bool first /*=true*/) //-------------------------------------------------------------------------- { if (!space.exists()) @@ -4016,8 +4400,39 @@ namespace Legion { // Couldn't find it, so send a request to the owner node AddressSpace owner = IndexSpaceNode::get_owner_space(space, runtime); if (owner == runtime->address_space) - REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, - "Unable to find entry for index space %x.", space.id) + { + // See if it is in the set of pending spaces in which case we + // can wait for it to be recorded + RtEvent pending_wait; + if (first) + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_index_spaces.find(space.get_id()); + if (finder != pending_index_spaces.end()) + { + if (!finder->second.exists()) + finder->second = Runtime::create_rt_user_event(); + pending_wait = finder->second; + } + } + if (pending_wait.exists()) + { + if (defer != NULL) + { + *defer = pending_wait; + return NULL; + } + else + { + pending_wait.wait(); + return get_node(space, defer, false/*first*/); + } + } + else + REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, + "Unable to find entry for index space %x.", space.id) + } // Retake the lock and get something to wait on { AutoLock l_lock(lookup_lock); @@ -4073,7 +4488,7 @@ namespace Legion { "Unable to find entry for index space %x." "This is definitely a runtime bug.", space.id) wait_on.wait(); - return get_node(space, NULL); + return get_node(space, NULL, false/*first*/); } else { @@ -4084,7 +4499,7 @@ namespace Legion { //-------------------------------------------------------------------------- IndexPartNode* RegionTreeForest::get_node(IndexPartition part, - RtEvent *defer/* = NULL*/) + RtEvent *defer/* = NULL*/, bool first/* = true*/) //-------------------------------------------------------------------------- { if (!part.exists()) @@ -4120,8 +4535,39 @@ namespace Legion { // Couldn't find it, so send a request to the owner node AddressSpace owner = IndexPartNode::get_owner_space(part, runtime); if (owner == runtime->address_space) - REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, - "Unable to find entry for index partition %x.",part.id) + { + // See if it is in the set of pending partitions in which case we + // can wait for it to be recorded + RtEvent pending_wait; + if (first) + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_partitions.find(part.get_id()); + if (finder != pending_partitions.end()) + { + if (!finder->second.exists()) + finder->second = Runtime::create_rt_user_event(); + pending_wait = finder->second; + } + } + if (pending_wait.exists()) + { + if (defer != NULL) + { + *defer = pending_wait; + return NULL; + } + else + { + pending_wait.wait(); + return get_node(part, defer, false/*first*/); + } + } + else + REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, + "Unable to find entry for index partition %x.",part.id) + } { // Retake the lock in exclusive mode and make // sure we didn't loose the race @@ -4177,7 +4623,7 @@ namespace Legion { "Unable to find entry for index partition %x. " "This is definitely a runtime bug.", part.id) wait_on.wait(); - return get_node(part, NULL); + return get_node(part, NULL, false/*first*/); } else { @@ -4188,7 +4634,7 @@ namespace Legion { //-------------------------------------------------------------------------- FieldSpaceNode* RegionTreeForest::get_node(FieldSpace space, - RtEvent *defer /*=NULL*/) + RtEvent *defer /*=NULL*/, bool first /*=true*/) //-------------------------------------------------------------------------- { if (!space.exists()) @@ -4224,8 +4670,39 @@ namespace Legion { // Couldn't find it, so send a request to the owner node AddressSpaceID owner = FieldSpaceNode::get_owner_space(space, runtime); if (owner == runtime->address_space) - REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, - "Unable to find entry for field space %x.", space.id) + { + // See if it is in the set of pending spaces in which case we + // can wait for it to be recorded + RtEvent pending_wait; + if (first) + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_field_spaces.find(space.get_id()); + if (finder != pending_field_spaces.end()) + { + if (!finder->second.exists()) + finder->second = Runtime::create_rt_user_event(); + pending_wait = finder->second; + } + } + if (pending_wait.exists()) + { + if (defer != NULL) + { + *defer = pending_wait; + return NULL; + } + else + { + pending_wait.wait(); + return get_node(space, defer, false/*first*/); + } + } + else + REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, + "Unable to find entry for field space %x.", space.id) + } { // Retake the lock in exclusive mode and // check to make sure we didn't loose the race @@ -4281,7 +4758,7 @@ namespace Legion { "Unable to find entry for field space %x. " "This is definitely a runtime bug.", space.id) wait_on.wait(); - return get_node(space, NULL); + return get_node(space, NULL, false/*first*/); } else { @@ -4292,7 +4769,7 @@ namespace Legion { //-------------------------------------------------------------------------- RegionNode* RegionTreeForest::get_node(LogicalRegion handle, - bool need_check /* = true*/) + bool need_check /* = true*/, bool first /*=true*/) //-------------------------------------------------------------------------- { if (!handle.exists()) @@ -4336,10 +4813,30 @@ namespace Legion { RegionTreeNode::get_owner_space(handle.get_tree_id(), runtime); if (owner == runtime->address_space) { - REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, - "Unable to find entry for logical region tree %d.", - handle.get_tree_id()); - assert(false); + // See if it is in the set of pending spaces in which case we + // can wait for it to be recorded + RtEvent pending_wait; + if (first) + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_region_trees.find(handle.get_tree_id()); + if (finder != pending_region_trees.end()) + { + if (!finder->second.exists()) + finder->second = Runtime::create_rt_user_event(); + pending_wait = finder->second; + } + } + if (pending_wait.exists()) + { + pending_wait.wait(); + return get_node(handle, need_check, false/*first*/); + } + else + REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, + "Unable to find entry for logical region tree %d.", + handle.get_tree_id()); } { // Retake the lock and make sure we didn't loose the race @@ -4491,7 +4988,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - RegionNode* RegionTreeForest::get_tree(RegionTreeID tid) + RegionNode* RegionTreeForest::get_tree(RegionTreeID tid,bool first/*=true*/) //-------------------------------------------------------------------------- { if (tid == 0) @@ -4521,8 +5018,31 @@ namespace Legion { // Couldn't find it, so send a request to the owner node AddressSpaceID owner = RegionTreeNode::get_owner_space(tid, runtime); if (owner == runtime->address_space) - REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, - "Unable to find entry for region tree ID %d", tid) + { + // See if it is in the set of pending spaces in which case we + // can wait for it to be recorded + RtEvent pending_wait; + if (first) + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_region_trees.find(tid); + if (finder != pending_region_trees.end()) + { + if (!finder->second.exists()) + finder->second = Runtime::create_rt_user_event(); + pending_wait = finder->second; + } + } + if (pending_wait.exists()) + { + pending_wait.wait(); + return get_tree(tid, false/*first*/); + } + else + REPORT_LEGION_ERROR(ERROR_UNABLE_FIND_ENTRY, + "Unable to find entry for region tree ID %d", tid) + } { // Retake the lock in exclusive mode and check to // make sure that we didn't lose the race @@ -4704,6 +5224,7 @@ namespace Legion { { AutoLock l_lock(lookup_lock); #ifdef DEBUG_LEGION + assert(index_part_requests.find(part) == index_part_requests.end()); std::map::iterator finder = index_parts.find(part); assert(finder != index_parts.end()); @@ -4766,6 +5287,142 @@ namespace Legion { part_nodes.erase(finder); } + //-------------------------------------------------------------------------- + void RegionTreeForest::record_pending_index_space(IndexSpaceID space) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + // We should be the owner for this space + assert((space % runtime->total_address_spaces) == runtime->address_space); +#endif + AutoLock l_lock(lookup_lock); +#ifdef DEBUG_LEGION + assert(pending_index_spaces.find(space) == pending_index_spaces.end()); +#endif + pending_index_spaces[space] = RtUserEvent::NO_RT_USER_EVENT; + } + + //-------------------------------------------------------------------------- + void RegionTreeForest::record_pending_partition(IndexPartitionID pid) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + // We should be the owner for this space + assert((pid % runtime->total_address_spaces) == runtime->address_space); +#endif + AutoLock l_lock(lookup_lock); +#ifdef DEBUG_LEGION + assert(pending_partitions.find(pid) == pending_partitions.end()); +#endif + pending_partitions[pid] = RtUserEvent::NO_RT_USER_EVENT; + } + + //-------------------------------------------------------------------------- + void RegionTreeForest::record_pending_field_space(FieldSpaceID space) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + // We should be the owner for this space + assert((space % runtime->total_address_spaces) == runtime->address_space); +#endif + AutoLock l_lock(lookup_lock); +#ifdef DEBUG_LEGION + assert(pending_field_spaces.find(space) == pending_field_spaces.end()); +#endif + pending_field_spaces[space] = RtUserEvent::NO_RT_USER_EVENT; + } + + //-------------------------------------------------------------------------- + void RegionTreeForest::record_pending_region_tree(RegionTreeID tid) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + // We should be the owner for this space + assert((tid % runtime->total_address_spaces) == runtime->address_space); +#endif + AutoLock l_lock(lookup_lock); +#ifdef DEBUG_LEGION + assert(pending_region_trees.find(tid) == pending_region_trees.end()); +#endif + pending_region_trees[tid] = RtUserEvent::NO_RT_USER_EVENT; + } + + //-------------------------------------------------------------------------- + void RegionTreeForest::revoke_pending_index_space(IndexSpaceID space) + //-------------------------------------------------------------------------- + { + RtUserEvent to_trigger; + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_index_spaces.find(space); +#ifdef DEBUG_LEGION + assert(finder != pending_index_spaces.end()); +#endif + to_trigger = finder->second; + pending_index_spaces.erase(finder); + } + if (to_trigger.exists()) + Runtime::trigger_event(to_trigger); + } + + //-------------------------------------------------------------------------- + void RegionTreeForest::revoke_pending_partition(IndexPartitionID pid) + //-------------------------------------------------------------------------- + { + RtUserEvent to_trigger; + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_partitions.find(pid); +#ifdef DEBUG_LEGION + assert(finder != pending_partitions.end()); +#endif + to_trigger = finder->second; + pending_partitions.erase(finder); + } + if (to_trigger.exists()) + Runtime::trigger_event(to_trigger); + } + + //-------------------------------------------------------------------------- + void RegionTreeForest::revoke_pending_field_space(FieldSpaceID space) + //-------------------------------------------------------------------------- + { + RtUserEvent to_trigger; + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_field_spaces.find(space); +#ifdef DEBUG_LEGION + assert(finder != pending_field_spaces.end()); +#endif + to_trigger = finder->second; + pending_field_spaces.erase(finder); + } + if (to_trigger.exists()) + Runtime::trigger_event(to_trigger); + } + + //-------------------------------------------------------------------------- + void RegionTreeForest::revoke_pending_region_tree(RegionTreeID tid) + //-------------------------------------------------------------------------- + { + RtUserEvent to_trigger; + { + AutoLock l_lock(lookup_lock); + std::map::iterator finder = + pending_region_trees.find(tid); +#ifdef DEBUG_LEGION + assert(finder != pending_region_trees.end()); +#endif + to_trigger = finder->second; + pending_region_trees.erase(finder); + } + if (to_trigger.exists()) + Runtime::trigger_event(to_trigger); + } + //-------------------------------------------------------------------------- bool RegionTreeForest::is_top_level_index_space(IndexSpace handle) //-------------------------------------------------------------------------- @@ -6198,6 +6855,27 @@ namespace Legion { parent_operations.erase(op); } + //-------------------------------------------------------------------------- + bool IndexSpaceExpression::test_intersection_nonblocking( + IndexSpaceExpression *other, RegionTreeForest *context, + ApEvent &precondition, bool second) + //-------------------------------------------------------------------------- + { + if (second) + { + // We've got two non pending expressions, so we can just test them + IndexSpaceExpression *overlap = + context->intersect_index_spaces(this, other); + return !overlap->is_empty(); + } + else + { + // First time through, we're not pending so keep going + return other->test_intersection_nonblocking(this, context, + precondition, true/*second*/); + } + } + //-------------------------------------------------------------------------- /*static*/ IndexSpaceExpression* IndexSpaceExpression::unpack_expression( Deserializer &derez, RegionTreeForest *forest, @@ -7012,6 +7690,19 @@ namespace Legion { return true; } + //-------------------------------------------------------------------------- + void IndexTreeNode::update_creation_set(const ShardMapping &mapping) + //-------------------------------------------------------------------------- + { + AutoLock n_lock(node_lock); + for (unsigned idx = 0; idx < mapping.size(); idx++) + { + const AddressSpaceID space = mapping[idx]; + if (space != context->runtime->address_space) + update_remote_instances(space, false/*need lock*/); + } + } + ///////////////////////////////////////////////////////////// // Index Space Node ///////////////////////////////////////////////////////////// @@ -7176,7 +7867,7 @@ namespace Legion { Runtime *rt) //-------------------------------------------------------------------------- { - return (handle.id % rt->runtime_stride); + return (handle.id % rt->total_address_spaces); } //-------------------------------------------------------------------------- @@ -7688,6 +8379,13 @@ namespace Legion { return; if (target == runtime->address_space) return; + if (mapping != NULL) + { + const ShardMapping &shard_mapping = *mapping; + for (unsigned idx = 0; idx < shard_mapping.size(); idx++) + if (shard_mapping[idx] == target) + return; + } runtime->send_index_space_set(target, rez); } @@ -7718,6 +8416,7 @@ namespace Legion { if (parent_handle.exists()) parent->send_node(target, true/*up*/); bool delete_parent = false; + if (!has_remote_instance(target)) { AutoLock n_lock(node_lock); { @@ -8257,13 +8956,14 @@ namespace Legion { IndexSpaceNode *par, IndexSpaceNode *color_sp, LegionColor c, bool dis, int comp, DistributedID did, ApEvent part_ready, - ApUserEvent partial, RtEvent init) - : IndexTreeNode(ctx, par->depth+1, c, did, - get_owner_space(p, ctx->runtime), init), - handle(p), parent(par), color_space(color_sp), - total_children(color_sp->get_volume()), + ApBarrier partial, RtEvent init, + ShardMapping *mapping) + : IndexTreeNode(ctx, par->depth+1, c, did, + get_owner_space(p, ctx->runtime), init), handle(p), parent(par), + color_space(color_sp), total_children(color_sp->get_volume()), max_linearized_color(color_sp->get_max_linearized_color()), - partition_ready(part_ready), partial_pending(partial), disjoint(dis), + partition_ready(part_ready), partial_pending(partial), + shard_mapping(mapping), disjoint(dis), has_complete(comp >= 0), complete(comp != 0), #ifdef DEBUG_LEGION first_valid(true), @@ -8278,6 +8978,8 @@ namespace Legion { assert(partial_pending == partition_ready); assert(handle.get_type_tag() == parent->handle.get_type_tag()); #endif + if (shard_mapping != NULL) + shard_mapping->add_reference(); #ifdef LEGION_GC log_garbage.info("GC Index Partition %lld %d %d", LEGION_DISTRIBUTED_ID_FILTER(did), local_space, handle.id); @@ -8287,15 +8989,15 @@ namespace Legion { //-------------------------------------------------------------------------- IndexPartNode::IndexPartNode(RegionTreeForest *ctx, IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *color_sp, - LegionColor c, RtEvent dis_ready, - int comp, DistributedID did,ApEvent part_ready, - ApUserEvent part, RtEvent init) - : IndexTreeNode(ctx, par->depth+1, c, did, - get_owner_space(p, ctx->runtime), init), handle(p), - parent(par), color_space(color_sp), - total_children(color_sp->get_volume()), + LegionColor c, RtEvent dis_ready, + int comp, DistributedID did, + ApEvent part_ready, ApBarrier part, + RtEvent init, ShardMapping *map) + : IndexTreeNode(ctx, par->depth+1, c, did, + get_owner_space(p, ctx->runtime), init), handle(p), parent(par), + color_space(color_sp), total_children(color_sp->get_volume()), max_linearized_color(color_sp->get_max_linearized_color()), - partition_ready(part_ready), partial_pending(part), + partition_ready(part_ready), partial_pending(part), shard_mapping(map), disjoint_ready(dis_ready), disjoint(false), has_complete(comp >= 0), complete(comp != 0), #ifdef DEBUG_LEGION @@ -8311,6 +9013,8 @@ namespace Legion { assert(partial_pending == partition_ready); assert(handle.get_type_tag() == parent->handle.get_type_tag()); #endif + if (shard_mapping != NULL) + shard_mapping->add_reference(); #ifdef LEGION_GC log_garbage.info("GC Index Partition %lld %d %d", LEGION_DISTRIBUTED_ID_FILTER(did), local_space, handle.id); @@ -8320,8 +9024,8 @@ namespace Legion { //-------------------------------------------------------------------------- IndexPartNode::IndexPartNode(const IndexPartNode &rhs) : IndexTreeNode(NULL,0,0,0,0,RtEvent::NO_RT_EVENT), - handle(IndexPartition::NO_PART), parent(NULL), color_space(NULL), - total_children(0), max_linearized_color(0) + handle(IndexPartition::NO_PART), parent(NULL), color_space(NULL), + total_children(0), max_linearized_color(0), shard_mapping(NULL) //-------------------------------------------------------------------------- { // should never be called @@ -8332,6 +9036,8 @@ namespace Legion { IndexPartNode::~IndexPartNode(void) //-------------------------------------------------------------------------- { + if ((shard_mapping != NULL) && shard_mapping->remove_reference()) + delete shard_mapping; // The reason we would be here is if we were leaked if (!partition_trackers.empty()) { @@ -8493,7 +9199,7 @@ namespace Legion { IndexPartition part, Runtime *runtime) //-------------------------------------------------------------------------- { - return (part.id % runtime->runtime_stride); + return (part.id % runtime->total_address_spaces); } //-------------------------------------------------------------------------- @@ -8661,8 +9367,14 @@ namespace Legion { REPORT_LEGION_ERROR(ERROR_INVALID_INDEX_SPACE_COLOR, "Invalid color space color for child %lld " "of partition %d", c, handle.get_id()) - AddressSpaceID owner_space = get_owner_space(); - AddressSpaceID local_space = context->runtime->address_space; + // Check to see if we're the owner space this child, in the + // common case the owner space is whichever node made the + // partition, but in the control replication case, we use the + // shard mapping to determine which node owns the given child + const AddressSpaceID owner_space = + (shard_mapping == NULL) ? get_owner_space() : + (*shard_mapping)[c % shard_mapping->size()]; + const AddressSpaceID local_space = context->runtime->address_space; // If we own the index partition, create a new subspace here if (owner_space == local_space) { @@ -8719,21 +9431,8 @@ namespace Legion { ApUserEvent partial_event = Runtime::create_ap_user_event(NULL); result = context->create_node(is, NULL/*realm is*/, this, c, did, initialized, partial_event); - add_pending_child(c, partial_event); - // Now check to see if we need to trigger our partition ready event - std::set child_ready_events; - { - AutoLock n_lock(node_lock,1,false/*exclusvie*/); - if (color_map.size() == size_t(total_children)) - { - for (std::map::const_iterator it = - color_map.begin(); it != color_map.end(); it++) - child_ready_events.insert(it->second->index_space_ready); - } - } - if (!child_ready_events.empty()) - Runtime::trigger_event(NULL, partial_pending, - Runtime::merge_events(NULL, child_ready_events)); + Runtime::phase_barrier_arrive(partial_pending, + 1/*count*/, partial_event); } else // Make a new index space node ready when the partition is ready @@ -8821,104 +9520,133 @@ namespace Legion { return color_space->get_volume(); } + //-------------------------------------------------------------------------- + IndexPartNode::RemoteDisjointnessFunctor::RemoteDisjointnessFunctor( + Serializer &r, Runtime *rt, ShardMapping *shard_mapping) + : rez(r), runtime(rt) + //-------------------------------------------------------------------------- + { + if (shard_mapping != NULL) + { + for (unsigned idx = 0; idx < shard_mapping->size(); idx++) + skip_shard_spaces.insert((*shard_mapping)[idx]); + } + } + //-------------------------------------------------------------------------- void IndexPartNode::RemoteDisjointnessFunctor::apply(AddressSpaceID target) //-------------------------------------------------------------------------- { - if (target != runtime->address_space) + if ((target != runtime->address_space) && + (skip_shard_spaces.empty() || + (skip_shard_spaces.find(target) == skip_shard_spaces.end()))) runtime->send_index_partition_disjoint_update(target, rez); } //-------------------------------------------------------------------------- - void IndexPartNode::compute_disjointness(RtUserEvent ready_event) + void IndexPartNode::compute_disjointness(ValueBroadcast *collective, + bool owner) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(disjoint_ready.exists() && !disjoint_ready.has_triggered()); - assert(ready_event == disjoint_ready); -#endif - // Now do the pairwise disjointness tests - disjoint = true; - if (total_children == max_linearized_color) + if (owner) { - for (LegionColor c1 = 0; disjoint && - (c1 < max_linearized_color); c1++) + // If we're the owner we do the disjointness test + disjoint = true; + if (total_children == max_linearized_color) { - for (LegionColor c2 = c1 + 1; disjoint && - (c2 < max_linearized_color); c2++) + for (LegionColor c1 = 0; disjoint && + (c1 < max_linearized_color); c1++) { - if (!are_disjoint(c1, c2, true/*force compute*/)) + for (LegionColor c2 = c1 + 1; disjoint && + (c2 < max_linearized_color); c2++) { - disjoint = false; - break; + if (!are_disjoint(c1, c2, true/*force compute*/)) + { + disjoint = false; + break; + } } + if (!disjoint) + break; } - if (!disjoint) - break; } - } - else - { - for (LegionColor c1 = 0; disjoint && - (c1 < max_linearized_color); c1++) + else { - if (!color_space->contains_color(c1)) - continue; - for (LegionColor c2 = c1 + 1; disjoint && - (c2 < max_linearized_color); c2++) + for (LegionColor c1 = 0; disjoint && + (c1 < max_linearized_color); c1++) { - if (!color_space->contains_color(c2)) + if (!color_space->contains_color(c1)) continue; - if (!are_disjoint(c1, c2, true/*force compute*/)) + for (LegionColor c2 = c1 + 1; disjoint && + (c2 < max_linearized_color); c2++) { - disjoint = false; - break; + if (!color_space->contains_color(c2)) + continue; + if (!are_disjoint(c1, c2, true/*force compute*/)) + { + disjoint = false; + break; + } } + if (!disjoint) + break; } - if (!disjoint) - break; } - } - // Make sure the write of disjoint propagates before - // we do the trigger of the event - __sync_synchronize(); - { - AutoLock n_lock(node_lock); + // Make sure the write of disjoint propagates before + // we do the trigger of the event + __sync_synchronize(); + { + AutoLock n_lock(node_lock); #ifdef DEBUG_LEGION - assert(disjoint_ready == ready_event); + assert(disjoint_ready.exists()); #endif - // We have to send notifications before any other remote - // requests can record themselves so we need to do it - // while we are holding the lock - if (has_remote_instances()) - { - Serializer rez; + // We have to send notifications before any other remote + // requests can record themselves so we need to do it + // while we are holding the lock + if (has_remote_instances() && + ((shard_mapping == NULL) || + (count_remote_instances() > shard_mapping->size()))) { - RezCheck z(rez); - rez.serialize(handle); - rez.serialize(disjoint); + Serializer rez; + { + RezCheck z(rez); + rez.serialize(handle); + rez.serialize(disjoint); + } + RemoteDisjointnessFunctor functor(rez, + context->runtime, shard_mapping); + map_over_remote_instances(functor); } - RemoteDisjointnessFunctor functor(rez, context->runtime); - map_over_remote_instances(functor); } + // If we have a disjointness barrier, then signal the result + if (collective != NULL) + collective->broadcast(disjoint); + // Record the result for Legion Spy + if (implicit_runtime->legion_spy_enabled) + LegionSpy::log_index_partition(parent->handle.id, handle.id, + disjoint, color); + if (implicit_runtime->profiler != NULL) + runtime->profiler->record_index_partition(parent->handle.id,handle.id, + disjoint, color); + } + else + { + // We're not the owner so we should have a barrier that tells + // us what the disjointness result is +#ifdef DEBUG_LEGION + assert(collective != NULL); +#endif + // No need to wait, we know this was a precondition for launching + // the task to compute the disjointness + disjoint = collective->get_value(); } - // Once we get here, we know the disjointness result so we can - // trigger the event saying when the disjointness value is ready - Runtime::trigger_event(ready_event); - // Record the result for Legion Spy - if (runtime->legion_spy_enabled) - LegionSpy::log_index_partition(parent->handle.id, handle.id, - disjoint, color); - if (runtime->profiler != NULL) - runtime->profiler->record_index_partition(parent->handle.id, handle.id, - disjoint, color); } //-------------------------------------------------------------------------- bool IndexPartNode::is_disjoint(bool app_query) //-------------------------------------------------------------------------- { - if (!disjoint_ready.has_triggered()) + if (disjoint_ready.exists() && !disjoint_ready.has_triggered()) disjoint_ready.wait(); return disjoint; } @@ -9141,121 +9869,116 @@ namespace Legion { color_space->instantiate_colors(colors); } - //-------------------------------------------------------------------------- - void IndexPartNode::add_pending_child(const LegionColor child_color, - ApUserEvent domain_ready) - //-------------------------------------------------------------------------- - { - bool launch_remove = false; - { - AutoLock n_lock(node_lock); - // Duplicate insertions can happen legally so avoid them - if (pending_children.find(child_color) == pending_children.end()) - { - pending_children[child_color] = domain_ready; - launch_remove = true; - } - } - if (launch_remove) - { - PendingChildArgs args(this, child_color); - // Don't remove the pending child until the handle is ready - context->runtime->issue_runtime_meta_task(args, - LG_LATENCY_WORK_PRIORITY, Runtime::protect_event(domain_ready)); - } - } - - //-------------------------------------------------------------------------- - bool IndexPartNode::get_pending_child(const LegionColor child_color, - ApUserEvent &domain_ready) - //-------------------------------------------------------------------------- - { - AutoLock n_lock(node_lock, 1, false/*exclusive*/); - std::map::const_iterator finder = - pending_children.find(child_color); - if (finder != pending_children.end()) - { - domain_ready = finder->second; - return true; - } - return false; - } - - //-------------------------------------------------------------------------- - void IndexPartNode::remove_pending_child(const LegionColor child_color) - //-------------------------------------------------------------------------- - { - AutoLock n_lock(node_lock); - pending_children.erase(child_color); - } - - //-------------------------------------------------------------------------- - /*static*/ void IndexPartNode::handle_pending_child_task(const void *args) - //-------------------------------------------------------------------------- - { - const PendingChildArgs *pargs = (const PendingChildArgs*)args; - pargs->parent->remove_pending_child(pargs->pending_child); - } - //-------------------------------------------------------------------------- ApEvent IndexPartNode::create_equal_children(Operation *op, - size_t granularity) + size_t granularity, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { - return parent->create_equal_children(op, this, granularity); + if (total_shards > 1) + return parent->create_equal_children(op, this, granularity, + shard, total_shards); + else + return parent->create_equal_children(op, this, granularity); } //-------------------------------------------------------------------------- ApEvent IndexPartNode::create_by_weights(Operation *op, - const FutureMap &weights, size_t granularity) + const FutureMap &weights, size_t granularity, + ShardID shard, size_t total_shards) //-------------------------------------------------------------------------- { - return parent->create_by_weights(op, this, weights.impl, granularity); + return parent->create_by_weights(op, this, weights.impl, granularity, + shard, total_shards); } //-------------------------------------------------------------------------- ApEvent IndexPartNode::create_by_union(Operation *op, IndexPartNode *left, - IndexPartNode *right) + IndexPartNode *right, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { - return parent->create_by_union(op, this, left, right); + if (total_shards > 1) + return parent->create_by_union(op, this, left, right, + shard, total_shards); + else + return parent->create_by_union(op, this, left, right); } //-------------------------------------------------------------------------- ApEvent IndexPartNode::create_by_intersection(Operation *op, IndexPartNode *left, - IndexPartNode *right) + IndexPartNode *right, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { - return parent->create_by_intersection(op, this, left, right); + if (total_shards > 1) + return parent->create_by_intersection(op, this, left, right, + shard, total_shards); + else + return parent->create_by_intersection(op, this, left, right); } //-------------------------------------------------------------------------- ApEvent IndexPartNode::create_by_intersection(Operation *op, IndexPartNode *original, - const bool dominates) + const bool dominates, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { - return parent->create_by_intersection(op, this, original, dominates); + if (total_shards > 1) + return parent->create_by_intersection(op, this, original, + shard, total_shards, dominates); + else + return parent->create_by_intersection(op, this, original, dominates); } //-------------------------------------------------------------------------- ApEvent IndexPartNode::create_by_difference(Operation *op, IndexPartNode *left, - IndexPartNode *right) + IndexPartNode *right, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { - return parent->create_by_difference(op, this, left, right); + if (total_shards > 1) + return parent->create_by_difference(op, this, left, right, + shard, total_shards); + else + return parent->create_by_difference(op, this, left, right); } //-------------------------------------------------------------------------- ApEvent IndexPartNode::create_by_restriction(const void *transform, - const void *extent) + const void *extent, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { return color_space->create_by_restriction(this, transform, extent, - NT_TemplateHelper::get_dim(handle.get_type_tag())); + NT_TemplateHelper::get_dim(handle.get_type_tag()), + shard, total_shards); + } + + //-------------------------------------------------------------------------- + /*static*/ void IndexPartNode::handle_disjointness_computation( + const void *args, RegionTreeForest *forest) + //-------------------------------------------------------------------------- + { + const DisjointnessArgs *dargs = (const DisjointnessArgs*)args; + IndexPartNode *node = forest->get_node(dargs->pid); + node->compute_disjointness(dargs->disjointness_collective, dargs->owner); + // We can now delete the collective + if (dargs->disjointness_collective != NULL) + delete dargs->disjointness_collective; + // Remove the reference on our node as well + if (node->remove_base_resource_ref(APPLICATION_REF)) + delete node; } //-------------------------------------------------------------------------- @@ -9439,6 +10162,8 @@ namespace Legion { parent->send_node(target, true/*up*/); // Always send the color space ahead of this color_space->send_node(target, false/*up*/); + std::map valid_copy; + if (!has_remote_instance(target)) { AutoLock n_lock(node_lock); // Check to see if we have computed the disjointness result @@ -9471,6 +10196,14 @@ namespace Legion { rez.serialize(partition_ready); rez.serialize(partial_pending); rez.serialize(initialized); + if (shard_mapping != NULL) + { + rez.serialize(shard_mapping->size()); + for (unsigned idx = 0; idx < shard_mapping->size(); idx++) + rez.serialize((*shard_mapping)[idx]); + } + else + rez.serialize(0); rez.serialize(semantic_info.size()); for (LegionMap::aligned::iterator it = semantic_info.begin(); it != semantic_info.end(); it++) @@ -9480,13 +10213,6 @@ namespace Legion { rez.serialize(it->second.buffer, it->second.size); rez.serialize(it->second.is_mutable); } - rez.serialize(pending_children.size()); - for (std::map::const_iterator it = - pending_children.begin(); it != pending_children.end(); it++) - { - rez.serialize(it->first); - rez.serialize(it->second); - } } context->runtime->send_index_partition_node(target, rez); update_remote_instances(target); @@ -9517,10 +10243,20 @@ namespace Legion { derez.deserialize(complete); ApEvent ready_event; derez.deserialize(ready_event); - ApUserEvent partial_pending; + ApBarrier partial_pending; derez.deserialize(partial_pending); RtEvent initialized; derez.deserialize(initialized); + size_t num_shard_mapping; + derez.deserialize(num_shard_mapping); + ShardMapping *mapping = NULL; + if (num_shard_mapping > 0) + { + mapping = new ShardMapping(); + mapping->resize(num_shard_mapping); + for (unsigned idx = 0; idx < num_shard_mapping; idx++) + derez.deserialize((*mapping)[idx]); + } IndexSpaceNode *parent_node = context->get_node(parent); IndexSpaceNode *color_space_node = context->get_node(color_space); #ifdef DEBUG_LEGION @@ -9532,9 +10268,11 @@ namespace Legion { dis_ready = Runtime::create_rt_user_event(); IndexPartNode *node = has_disjoint ? context->create_node(handle, parent_node, color_space_node, color, - disjoint, complete, did, ready_event, partial_pending, initialized) : + disjoint, complete, did, ready_event, partial_pending, + initialized, mapping) : context->create_node(handle, parent_node, color_space_node, color, - dis_ready, complete, did, ready_event, partial_pending, initialized); + dis_ready, complete, did, ready_event, partial_pending, + initialized, mapping); if (!has_disjoint) node->record_remote_disjoint_ready(dis_ready); #ifdef DEBUG_LEGION @@ -9555,16 +10293,6 @@ namespace Legion { node->attach_semantic_information(tag, source, buffer, buffer_size, is_mutable, false/*local only*/); } - size_t num_pending; - derez.deserialize(num_pending); - for (unsigned idx = 0; idx < num_pending; idx++) - { - LegionColor child_color; - derez.deserialize(child_color); - ApUserEvent child_ready; - derez.deserialize(child_ready); - node->add_pending_child(child_color, child_ready); - } } //-------------------------------------------------------------------------- @@ -9712,21 +10440,37 @@ namespace Legion { //-------------------------------------------------------------------------- FieldSpaceNode::FieldSpaceNode(FieldSpace sp, RegionTreeForest *ctx, - DistributedID did, RtEvent init) + DistributedID did, RtEvent init, ShardMapping *shard_mapping) : DistributedCollectable(ctx->runtime, LEGION_DISTRIBUTED_HELP_ENCODE(did, FIELD_SPACE_DC), get_owner_space(sp, ctx->runtime), false/*register with runtime*/), handle(sp), context(ctx), initialized(init), - allocation_state(FIELD_ALLOC_READ_ONLY), outstanding_allocators(0), - outstanding_invalidations(0) + allocation_state((shard_mapping != NULL) ? FIELD_ALLOC_COLLECTIVE : + is_owner() ? FIELD_ALLOC_READ_ONLY : FIELD_ALLOC_INVALID), + outstanding_allocators(0), outstanding_invalidations(0) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(is_owner()); -#endif - unallocated_indexes = FieldMask(LEGION_FIELD_MASK_FIELD_ALL_ONES); - local_index_infos.resize(runtime->max_local_fields, - std::pair(0, 0)); + if (is_owner()) + { + unallocated_indexes = FieldMask(LEGION_FIELD_MASK_FIELD_ALL_ONES); + local_index_infos.resize(runtime->max_local_fields, + std::pair(0, 0)); + if (shard_mapping != NULL) + { + const ShardMapping &mapping = *shard_mapping; + for (unsigned idx = 0; idx < mapping.size(); idx++) + { + const AddressSpaceID space = mapping[idx]; + if (space != local_space) + remote_field_infos.insert(mapping[idx]); + } + // We can have control replication inside of just a single node + if (remote_field_infos.empty()) + allocation_state = FIELD_ALLOC_READ_ONLY; + } + } + else if (allocation_state == FIELD_ALLOC_COLLECTIVE) + unallocated_indexes = FieldMask(LEGION_FIELD_MASK_FIELD_ALL_ONES); #ifdef LEGION_GC log_garbage.info("GC Field Space %lld %d %d", LEGION_DISTRIBUTED_ID_FILTER(did), local_space, handle.id); @@ -9848,7 +10592,7 @@ namespace Legion { Runtime *rt) //-------------------------------------------------------------------------- { - return (handle.id % rt->runtime_stride); + return (handle.id % rt->total_address_spaces); } //-------------------------------------------------------------------------- @@ -10474,10 +11218,28 @@ namespace Legion { //-------------------------------------------------------------------------- RtEvent FieldSpaceNode::create_allocator(AddressSpaceID source, - RtUserEvent ready_event) + RtUserEvent ready_event, bool sharded_owner_context, bool owner_shard) //-------------------------------------------------------------------------- { AutoLock n_lock(node_lock); + if (sharded_owner_context) + { + // If we were the sharded collective context that made this + // field space and we're still in collective allocation mode + // then we are trivially done + if (allocation_state == FIELD_ALLOC_COLLECTIVE) + { +#ifdef DEBUG_LEGION + assert(outstanding_allocators == 0); +#endif + outstanding_allocators = 1; + return RtEvent::NO_RT_EVENT; + } + // Otherwise if we're not the owner shard then we're also done since + // the owner shard is the only one doing the allocation + if (!owner_shard) + return RtEvent::NO_RT_EVENT; + } if (is_owner()) { switch (allocation_state) @@ -10506,6 +11268,8 @@ namespace Legion { RezCheck z(rez); rez.serialize(handle); rez.serialize(ready_event); + rez.serialize(true); // flush allocation + rez.serialize(false); // need merge } runtime->send_field_space_allocator_invalidation(remote_owner, rez); @@ -10514,6 +11278,42 @@ namespace Legion { allocation_state = FIELD_ALLOC_PENDING; break; } + case FIELD_ALLOC_COLLECTIVE: + { + // This is the case when we're still in collective mode + // and we need to switch to exclusive mode on just one node + // because someone else asked for an allocator + if (outstanding_allocators > 0) + { +#ifdef DEBUG_LEGION + assert(!remote_field_infos.empty()); + assert(outstanding_invalidations == 0); +#endif + std::set preconditions; + for (std::set::const_iterator it = + remote_field_infos.begin(); it != + remote_field_infos.end(); it++) + { + const RtUserEvent done = Runtime::create_rt_user_event(); + outstanding_invalidations++; + Serializer rez; + { + RezCheck z(rez); + rez.serialize(handle); + rez.serialize(done); + rez.serialize(true); // flush allocation + rez.serialize(true); // need merge + } + runtime->send_field_space_allocator_invalidation(*it, rez); + preconditions.insert(done); + } + remote_field_infos.clear(); + pending_field_allocation = Runtime::merge_events(preconditions); + allocation_state = FIELD_ALLOC_PENDING; + break; + } + // otherwise we fall through to the identical read-only case + } case FIELD_ALLOC_READ_ONLY: { #ifdef DEBUG_LEGION @@ -10544,6 +11344,8 @@ namespace Legion { RezCheck z(rez); rez.serialize(handle); rez.serialize(done); + rez.serialize(false); // flush allocation + rez.serialize(false); // need merge } runtime->send_field_space_allocator_invalidation(*it, rez); preconditions.insert(done); @@ -10629,13 +11431,7 @@ namespace Legion { if (ready_event.exists()) Runtime::trigger_event(ready_event, pending_field_allocation); break; - } - case FIELD_ALLOC_COLLECTIVE: - { - // TODO: implement this for control replication - assert(false); - break; - } + } default: assert(false); } @@ -10679,7 +11475,8 @@ namespace Legion { } //-------------------------------------------------------------------------- - RtEvent FieldSpaceNode::destroy_allocator(AddressSpaceID source) + RtEvent FieldSpaceNode::destroy_allocator(AddressSpaceID source, + bool sharded_owner_context, bool owner_shard) //-------------------------------------------------------------------------- { AutoLock n_lock(node_lock); @@ -10688,6 +11485,24 @@ namespace Legion { (allocation_state == FIELD_ALLOC_COLLECTIVE) || (allocation_state == FIELD_ALLOC_INVALID)); #endif + if (sharded_owner_context) + { + // If we were the sharded collective context that made this + // field space and we're still in collective allocation mode + // then we are trivially done + if (allocation_state == FIELD_ALLOC_COLLECTIVE) + { +#ifdef DEBUG_LEGION + assert(outstanding_allocators == 1); +#endif + outstanding_allocators = 0; + return RtEvent::NO_RT_EVENT; + } + // Otherwise if we're not the owner shard then we're also done since + // the owner shard is the only one doing the allocation + if (!owner_shard) + return RtEvent::NO_RT_EVENT; + } if (allocation_state == FIELD_ALLOC_INVALID) { #ifdef DEBUG_LEGION @@ -10752,10 +11567,26 @@ namespace Legion { //-------------------------------------------------------------------------- RtEvent FieldSpaceNode::allocate_field(FieldID fid, size_t size, - CustomSerdezID serdez_id) + CustomSerdezID serdez_id, + bool sharded_non_owner) //-------------------------------------------------------------------------- { AutoLock n_lock(node_lock); + // For control replication see if we've been invalidated and do not need + // to do anything because we are not the owner any longer + if (sharded_non_owner && (allocation_state != FIELD_ALLOC_COLLECTIVE)) + return RtEvent::NO_RT_EVENT; + while (allocation_state == FIELD_ALLOC_PENDING) + { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif + const RtEvent wait_on = pending_field_allocation; + n_lock.release(); + if (!wait_on.has_triggered()) + wait_on.wait(); + n_lock.reacquire(); + } // Check to see if we can do the allocation if ((allocation_state != FIELD_ALLOC_EXCLUSIVE) && (allocation_state != FIELD_ALLOC_COLLECTIVE)) @@ -10779,10 +11610,17 @@ namespace Legion { return allocated_event; } // We're the owner so do the field allocation - if (field_infos.find(fid) != field_infos.end()) + std::map::iterator finder = field_infos.find(fid); + if (finder != field_infos.end()) + { + // Handle the case of deduplicating fields that were allocated + // in a collective mode but are now merged together + if (finder->second.collective) + return RtEvent::NO_RT_EVENT; REPORT_LEGION_ERROR(ERROR_ILLEGAL_DUPLICATE_FIELD_ID, "Illegal duplicate field ID %d used by the " "application in field space %d", fid, handle.id) + } // Find an index in which to allocate this field RtEvent ready_event; int result = allocate_index(ready_event); @@ -10793,16 +11631,33 @@ namespace Legion { " related macros at the top of legion_config.h and " "recompile.", handle.id, LEGION_MAX_FIELDS) const unsigned index = result; - field_infos[fid] = FieldInfo(size, index, serdez_id); + field_infos[fid] = FieldInfo(size, index, serdez_id, false/*local*/, + (allocation_state == FIELD_ALLOC_COLLECTIVE)); return ready_event; } //-------------------------------------------------------------------------- RtEvent FieldSpaceNode::allocate_field(FieldID fid, ApEvent size_ready, - CustomSerdezID serdez_id) + CustomSerdezID serdez_id, + bool sharded_non_owner) //-------------------------------------------------------------------------- { AutoLock n_lock(node_lock); + // For control replication see if we've been invalidated and do not need + // to do anything because we are not the owner any longer + if (sharded_non_owner && (allocation_state != FIELD_ALLOC_COLLECTIVE)) + return RtEvent::NO_RT_EVENT; + while (allocation_state == FIELD_ALLOC_PENDING) + { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif + const RtEvent wait_on = pending_field_allocation; + n_lock.release(); + if (!wait_on.has_triggered()) + wait_on.wait(); + n_lock.reacquire(); + } // Check to see if we can do the allocation if ((allocation_state != FIELD_ALLOC_EXCLUSIVE) && (allocation_state != FIELD_ALLOC_COLLECTIVE)) @@ -10825,10 +11680,17 @@ namespace Legion { return allocated_event; } // We're the owner so do the field allocation - if (field_infos.find(fid) != field_infos.end()) + std::map::iterator finder = field_infos.find(fid); + if (finder != field_infos.end()) + { + // Handle the case of deduplicating fields that were allocated + // in a collective mode but are now merged together + if (finder->second.collective) + return RtEvent::NO_RT_EVENT; REPORT_LEGION_ERROR(ERROR_ILLEGAL_DUPLICATE_FIELD_ID, "Illegal duplicate field ID %d used by the " "application in field space %d", fid, handle.id) + } // Find an index in which to allocate this field RtEvent ready_event; int result = allocate_index(ready_event); @@ -10839,20 +11701,37 @@ namespace Legion { " related macros at the top of legion_config.h and " "recompile.", handle.id, LEGION_MAX_FIELDS) const unsigned index = result; - field_infos[fid] = FieldInfo(size_ready, index, serdez_id); + field_infos[fid] = FieldInfo(size_ready, index, serdez_id, false/*local*/, + (allocation_state == FIELD_ALLOC_COLLECTIVE)); return ready_event; } //-------------------------------------------------------------------------- RtEvent FieldSpaceNode::allocate_fields(const std::vector &sizes, const std::vector &fids, - CustomSerdezID serdez_id) + CustomSerdezID serdez_id, + bool sharded_non_owner) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION assert(sizes.size() == fids.size()); #endif AutoLock n_lock(node_lock); + // For control replication see if we've been invalidated and do not need + // to do anything because we are not the owner any longer + if (sharded_non_owner && (allocation_state != FIELD_ALLOC_COLLECTIVE)) + return RtEvent::NO_RT_EVENT; + while (allocation_state == FIELD_ALLOC_PENDING) + { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif + const RtEvent wait_on = pending_field_allocation; + n_lock.release(); + if (!wait_on.has_triggered()) + wait_on.wait(); + n_lock.reacquire(); + } // Check to see if we can do the allocation if ((allocation_state != FIELD_ALLOC_EXCLUSIVE) && (allocation_state != FIELD_ALLOC_COLLECTIVE)) @@ -10879,11 +11758,18 @@ namespace Legion { std::set allocated_events; for (unsigned idx = 0; idx < fids.size(); idx++) { - FieldID fid = fids[idx]; + const FieldID fid = fids[idx]; + std::map::iterator finder = field_infos.find(fid); if (field_infos.find(fid) != field_infos.end()) + { + // Handle the case of deduplicating fields that were allocated + // in a collective mode but are now merged together + if (finder->second.collective) + continue; REPORT_LEGION_ERROR(ERROR_ILLEGAL_DUPLICATE_FIELD_ID, "Illegal duplicate field ID %d used by the " "application in field space %d", fid, handle.id) + } // Find an index in which to allocate this field RtEvent ready_event; int result = allocate_index(ready_event); @@ -10896,7 +11782,8 @@ namespace Legion { if (ready_event.exists()) allocated_events.insert(ready_event); const unsigned index = result; - field_infos[fid] = FieldInfo(sizes[idx], index, serdez_id); + field_infos[fid] = FieldInfo(sizes[idx], index, serdez_id, + false/*local*/, (allocation_state == FIELD_ALLOC_COLLECTIVE)); } if (!allocated_events.empty()) return Runtime::merge_events(allocated_events); @@ -10907,10 +11794,26 @@ namespace Legion { //-------------------------------------------------------------------------- RtEvent FieldSpaceNode::allocate_fields(ApEvent sizes_ready, const std::vector &fids, - CustomSerdezID serdez_id) + CustomSerdezID serdez_id, + bool sharded_non_owner) //-------------------------------------------------------------------------- { AutoLock n_lock(node_lock); + // For control replication see if we've been invalidated and do not need + // to do anything because we are not the owner any longer + if (sharded_non_owner && (allocation_state != FIELD_ALLOC_COLLECTIVE)) + return RtEvent::NO_RT_EVENT; + while (allocation_state == FIELD_ALLOC_PENDING) + { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif + const RtEvent wait_on = pending_field_allocation; + n_lock.release(); + if (!wait_on.has_triggered()) + wait_on.wait(); + n_lock.reacquire(); + } // Check to see if we can do the allocation if ((allocation_state != FIELD_ALLOC_EXCLUSIVE) && (allocation_state != FIELD_ALLOC_COLLECTIVE)) @@ -10934,11 +11837,18 @@ namespace Legion { std::set allocated_events; for (unsigned idx = 0; idx < fids.size(); idx++) { - FieldID fid = fids[idx]; - if (field_infos.find(fid) != field_infos.end()) + const FieldID fid = fids[idx]; + std::map::iterator finder = field_infos.find(fid); + if (finder != field_infos.end()) + { + // Handle the case of deduplicating fields that were allocated + // in a collective mode but are now merged together + if (finder->second.collective) + continue; REPORT_LEGION_ERROR(ERROR_ILLEGAL_DUPLICATE_FIELD_ID, "Illegal duplicate field ID %d used by the " "application in field space %d", fid, handle.id) + } // Find an index in which to allocate this field RtEvent ready_event; int result = allocate_index(ready_event); @@ -10951,7 +11861,8 @@ namespace Legion { if (ready_event.exists()) allocated_events.insert(ready_event); const unsigned index = result; - field_infos[fid] = FieldInfo(sizes_ready, index, serdez_id); + field_infos[fid] = FieldInfo(sizes_ready, index, serdez_id, + false/*local*/, (allocation_state == FIELD_ALLOC_COLLECTIVE)); } if (!allocated_events.empty()) return Runtime::merge_events(allocated_events); @@ -11041,10 +11952,26 @@ namespace Legion { //-------------------------------------------------------------------------- void FieldSpaceNode::free_field(FieldID fid, AddressSpaceID source, - std::set &applied) + std::set &applied, + bool sharded_non_owner) //-------------------------------------------------------------------------- { AutoLock n_lock(node_lock); + // For control replication see if we've been invalidated and do not need + // to do anything because we are not the owner any longer + if (sharded_non_owner && (allocation_state != FIELD_ALLOC_COLLECTIVE)) + return; + while (allocation_state == FIELD_ALLOC_PENDING) + { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif + const RtEvent wait_on = pending_field_allocation; + n_lock.release(); + if (!wait_on.has_triggered()) + wait_on.wait(); + n_lock.reacquire(); + } if ((allocation_state != FIELD_ALLOC_EXCLUSIVE) && (allocation_state != FIELD_ALLOC_COLLECTIVE)) { @@ -11075,10 +12002,26 @@ namespace Legion { //-------------------------------------------------------------------------- void FieldSpaceNode::free_fields(const std::vector &to_free, - AddressSpaceID source, std::set &applied) + AddressSpaceID source, std::set &applied, + bool sharded_non_owner) //-------------------------------------------------------------------------- { AutoLock n_lock(node_lock); + // For control replication see if we've been invalidated and do not need + // to do anything because we are not the owner any longer + if (sharded_non_owner && (allocation_state != FIELD_ALLOC_COLLECTIVE)) + return; + while (allocation_state == FIELD_ALLOC_PENDING) + { +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif + const RtEvent wait_on = pending_field_allocation; + n_lock.release(); + if (!wait_on.has_triggered()) + wait_on.wait(); + n_lock.reacquire(); + } if ((allocation_state != FIELD_ALLOC_EXCLUSIVE) && (allocation_state != FIELD_ALLOC_COLLECTIVE)) { @@ -11187,7 +12130,8 @@ namespace Legion { //-------------------------------------------------------------------------- void FieldSpaceNode::free_local_fields(const std::vector &to_free, - const std::vector &indexes) + const std::vector &indexes, + const bool collective) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -11195,19 +12139,22 @@ namespace Legion { #endif if (!is_owner()) { - // Send a message to the owner to do the free of the fields - Serializer rez; + if (!collective) { - RezCheck z(rez); - rez.serialize(handle); - rez.serialize(to_free.size()); - for (unsigned idx = 0; idx < to_free.size(); idx++) + // Send a message to the owner to do the free of the fields + Serializer rez; { - rez.serialize(to_free[idx]); - rez.serialize(indexes[idx]); + RezCheck z(rez); + rez.serialize(handle); + rez.serialize(to_free.size()); + for (unsigned idx = 0; idx < to_free.size(); idx++) + { + rez.serialize(to_free[idx]); + rez.serialize(indexes[idx]); + } } + context->runtime->send_local_field_free(owner_space, rez); } - context->runtime->send_local_field_free(owner_space, rez); } else { @@ -11258,6 +12205,19 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + void FieldSpaceNode::update_creation_set(const ShardMapping &mapping) + //-------------------------------------------------------------------------- + { + AutoLock n_lock(node_lock); + for (unsigned idx = 0; idx < mapping.size(); idx++) + { + const AddressSpaceID space = mapping[idx]; + if (space != context->runtime->address_space) + update_remote_instances(space, false/*need lock*/); + } + } + //-------------------------------------------------------------------------- bool FieldSpaceNode::has_field(FieldID fid) //-------------------------------------------------------------------------- @@ -11976,7 +12936,8 @@ namespace Legion { } FieldSpaceNode *node = forest->get_node(handle); - node->free_local_fields(fields, indexes); + node->free_local_fields(fields, indexes, + false/*not a collective if we're here*/); } //-------------------------------------------------------------------------- @@ -12489,9 +13450,13 @@ namespace Legion { derez.deserialize(handle); RtUserEvent done_event; derez.deserialize(done_event); + bool flush_allocation; + derez.deserialize(flush_allocation); + bool merge; + derez.deserialize(merge); FieldSpaceNode *node = forest->get_node(handle); - node->process_allocator_invalidation(done_event); + node->process_allocator_invalidation(done_event, flush_allocation, merge); } //-------------------------------------------------------------------------- @@ -13059,21 +14024,28 @@ namespace Legion { } //-------------------------------------------------------------------------- - void FieldSpaceNode::process_allocator_invalidation(RtUserEvent done_event) + void FieldSpaceNode::process_allocator_invalidation(RtUserEvent done_event, + bool flush_allocation, bool need_merge) //-------------------------------------------------------------------------- { AutoLock n_lock(node_lock); #ifdef DEBUG_LEGION assert(!is_owner()); assert((allocation_state == FIELD_ALLOC_EXCLUSIVE) || + (allocation_state == FIELD_ALLOC_COLLECTIVE) || (allocation_state == FIELD_ALLOC_READ_ONLY)); #endif Serializer rez; - if (allocation_state == FIELD_ALLOC_EXCLUSIVE) + // It's possible to be in the read-only state even with a flush because + // of ships passing in the night. We get sent an invalidation, but we + // already released our allocator and sent it back to the owner so we are + // in the read-only state and the messages pass like ships in the night + if (flush_allocation && (allocation_state != FIELD_ALLOC_READ_ONLY)) { RezCheck z(rez); rez.serialize(handle); rez.serialize(true); // allocation meta data + rez.serialize(need_merge); rez.serialize(field_infos.size()); for (std::map::iterator it = field_infos.begin(); it != field_infos.end(); /*nothing*/) @@ -13104,6 +14076,10 @@ namespace Legion { } else { +#ifdef DEBUG_LEGION + assert((allocation_state == FIELD_ALLOC_READ_ONLY) || + (allocation_state == FIELD_ALLOC_COLLECTIVE)); +#endif RezCheck z(rez); rez.serialize(handle); rez.serialize(false); // allocation meta data @@ -13138,31 +14114,73 @@ namespace Legion { AutoLock n_lock(node_lock); if (allocator_meta_data) { - size_t num_infos; - derez.deserialize(num_infos); - for (unsigned idx = 0; idx < num_infos; idx++) + bool need_merge; + derez.deserialize(need_merge); + if (need_merge) { - FieldID fid; - derez.deserialize(fid); - derez.deserialize(field_infos[fid]); + size_t num_infos; + derez.deserialize(num_infos); + for (unsigned idx = 0; idx < num_infos; idx++) + { + FieldID fid; + derez.deserialize(fid); + if (field_infos.find(fid) == field_infos.end()) + derez.deserialize(field_infos[fid]); + else + derez.advance_pointer(sizeof(FieldInfo)); + } + FieldMask unallocated; + derez.deserialize(unallocated); + unallocated_indexes |= unallocated; + size_t num_available; + derez.deserialize(num_available); + for (unsigned idx = 0; idx < num_available; idx++) + { + std::pair next; + derez.deserialize(next.first); + derez.deserialize(next.second); + bool found = false; + for (std::list >::const_iterator it = + available_indexes.begin(); it != available_indexes.end(); it++) + { + if (it->first != next.first) + continue; + found = true; + break; + } + if (!found) + available_indexes.push_back(next); + } + derez.advance_pointer(sizeof(outstanding_allocators)); } + else + { + size_t num_infos; + derez.deserialize(num_infos); + for (unsigned idx = 0; idx < num_infos; idx++) + { + FieldID fid; + derez.deserialize(fid); + derez.deserialize(field_infos[fid]); + } #ifdef DEBUG_LEGION - assert(!unallocated_indexes); - assert(available_indexes.empty()); + assert(!unallocated_indexes); + assert(available_indexes.empty()); #endif - derez.deserialize(unallocated_indexes); - size_t num_available; - derez.deserialize(num_available); - for (unsigned idx = 0; idx < num_available; idx++) - { - std::pair next; - derez.deserialize(next.first); - derez.deserialize(next.second); - available_indexes.push_back(next); + derez.deserialize(unallocated_indexes); + size_t num_available; + derez.deserialize(num_available); + for (unsigned idx = 0; idx < num_available; idx++) + { + std::pair next; + derez.deserialize(next.first); + derez.deserialize(next.second); + available_indexes.push_back(next); + } + unsigned remote_allocators; + derez.deserialize(remote_allocators); + outstanding_allocators += remote_allocators; } - unsigned remote_allocators; - derez.deserialize(remote_allocators); - outstanding_allocators += remote_allocators; } #ifdef DEBUG_LEGION assert(outstanding_invalidations > 0); @@ -13326,7 +14344,7 @@ namespace Legion { Runtime *runtime) //-------------------------------------------------------------------------- { - return (tid % runtime->runtime_stride); + return (tid % runtime->total_address_spaces); } //-------------------------------------------------------------------------- @@ -13532,7 +14550,7 @@ namespace Legion { { // Close up any children which we may have dependences on below const bool captures_closes = true; - LogicalCloser closer(ctx, user, this, arrived/*validates*/); + LogicalCloser closer(ctx, user, this, arrived/*validates*/); // Special siphon operation for arrived projecting functions if (arrived && proj_info.is_projecting()) { @@ -13623,15 +14641,7 @@ namespace Legion { { // If we've arrived add ourselves as a user register_local_user(state, user, trace_info); - // Handle any projections that we might have - // If we're projecting, record any split fields and projection epochs - if (proj_info.is_projecting()) - { - // If we're writing, then record our projection info in - // the current projection epoch else do the same for non-writes - state.update_projection_epochs(user.field_mask, proj_info); - } - else if (user.usage.redop > 0) + if (!proj_info.is_projecting() && (user.usage.redop > 0)) { // Not projecting and doing a reduction of some kind so record it record_logical_reduction(state, user.usage.redop, user.field_mask); @@ -13694,7 +14704,8 @@ namespace Legion { if (arrived && proj_info.is_projecting()) { FieldState new_state(user.usage, open_mask, proj_info.projection, - proj_info.projection_space, are_all_children_disjoint()); + proj_info.projection_space, proj_info.sharding_function, + proj_info.sharding_space, this); merge_new_field_state(state, new_state); } else if (next_child != NULL) @@ -13738,7 +14749,6 @@ namespace Legion { // If this was a projection field state then we need to // advance the epoch version numbers // If this is a writing or reducing - state.advance_projection_epochs(overlap); it->filter(overlap); } else @@ -14076,8 +15086,6 @@ namespace Legion { assert(!!overlap); #endif closer.record_close_operation(overlap); - // Advance the projection epochs - state.advance_projection_epochs(overlap); } it->filter(current_mask); if (!it->valid_fields()) @@ -14090,7 +15098,6 @@ namespace Legion { break; } case OPEN_READ_WRITE_PROJ: - case OPEN_READ_WRITE_PROJ_DISJOINT_SHALLOW: { // Have to close up this sub-tree no matter what if (record_close_operations) @@ -14100,8 +15107,6 @@ namespace Legion { assert(!!overlap); #endif closer.record_close_operation(overlap); - // Advance the projection epochs - state.advance_projection_epochs(overlap); } it->filter(current_mask); if (!it->valid_fields()) @@ -14125,8 +15130,6 @@ namespace Legion { assert(!!overlap); #endif closer.record_close_operation(overlap); - // Advance the projection epochs - state.advance_projection_epochs(overlap); } it->filter(current_mask); if (!it->valid_fields()) @@ -14189,7 +15192,9 @@ namespace Legion { // dirty data can only live at the leaves of the open tree. Therefore // we must either being going into a disjoint shallow mode or any // mode which is read only that permits us to go to disjoint shallow + // Also cannot have a sharding function for control replication const bool disjoint_close = !is_region() && are_all_children_disjoint() && + (proj_info.sharding_function == NULL) && (IS_READ_ONLY(closer.user.usage) || (proj_info.projection->depth == 0)); // Now we can look at all the children for (LegionList::aligned::iterator it = @@ -14257,10 +15262,23 @@ namespace Legion { } else { + // Check to see if we have a sharding functor + // in which case we need to make sure we don't + // need a close because of different sharding functors + if (record_close_operations && + (proj_info.sharding_function != NULL)) + { + // We need a close operation here + const FieldMask overlap = current_mask & it->valid_fields(); +#ifdef DEBUG_LEGION + assert(!!overlap); +#endif + closer.record_close_operation(overlap); + } // Otherwise we are going to a different mode // no need to do a close since we're staying in - // projection mode, but we do need to advance - // the projection epochs. + // projection mode (except in the sharding function + // case described above) it->filter(current_mask); if (!it->valid_fields()) it = state.field_states.erase(it); @@ -14274,8 +15292,8 @@ namespace Legion { // Can only avoid a close operation if we have the // same projection function with the same or smaller // size domain as the original index space launch - if ((it->projection == proj_info.projection) && - it->projection_domain_dominates(proj_info.projection_space)) + if (it->can_elide_close_operation(closer.user.op, closer.user.idx, + proj_info, this, IS_REDUCE(closer.user.usage))) { // If we're a reduction we have to go into a dirty // reduction mode since we know we're already open below @@ -14288,7 +15306,8 @@ namespace Legion { // Make the new state to add FieldState new_state(closer.user.usage, overlap, proj_info.projection, proj_info.projection_space, - are_all_children_disjoint(), true/*dirty reduce*/); + proj_info.sharding_function,proj_info.sharding_space, + this, true/*dirty reduce*/); new_states.emplace(new_state); // If we are a reduction, we can go straight there it->filter(overlap); @@ -14299,8 +15318,9 @@ namespace Legion { } else { - // Update the domain - it->projection_space = proj_info.projection_space; + // If we're a write we need to update the projection space + if (IS_WRITE(closer.user.usage)) + it->record_projection_summary(proj_info, this); open_below |= (it->valid_fields() & current_mask); it++; } @@ -14314,8 +15334,6 @@ namespace Legion { #ifdef DEBUG_LEGION assert(!!overlap); #endif - // Advance the projection epochs - state.advance_projection_epochs(overlap); // If we are doing a disjoint close, update the open // states with the appropriate new state if (disjoint_close) @@ -14331,7 +15349,8 @@ namespace Legion { as_partition_node()->row_source->color_space; FieldState new_state(close_usage, overlap, context->runtime->find_projection_function(0), - color_space, true/*disjoint*/); + color_space, NULL/*sharding func*/, + NULL/*sharding space*/, this); new_states.emplace(new_state); } else @@ -14345,71 +15364,6 @@ namespace Legion { } break; } - case OPEN_READ_WRITE_PROJ_DISJOINT_SHALLOW: - { -#ifdef DEBUG_LEGION - // Can only be here if all children are disjoint - assert(are_all_children_disjoint()); -#endif - if (IS_REDUCE(closer.user.usage) && - it->projection_domain_dominates(proj_info.projection_space)) - { - const FieldMask overlap = it->valid_fields() & current_mask; - // Record that some fields are already open - open_below |= overlap; - // Make the new state to add - FieldState new_state(closer.user.usage, overlap, - proj_info.projection, proj_info.projection_space, - are_all_children_disjoint(), true/*dirty reduce*/); - new_states.emplace(new_state); - // If we are a reduction, we can go straight there - it->filter(overlap); - if (!it->valid_fields()) - it = state.field_states.erase(it); - else - it++; - } - else if (IS_READ_ONLY(closer.user.usage)) - { - // Read-only projections of any depth allow - // us to stay in disjoint shallow mode because - // they are not going to mutate the state at - // all and we can catch dependences on any - // index spaces without needing a close operation - it++; - } - else if ((proj_info.projection->depth == 0) && - !IS_REDUCE(closer.user.usage)) - { - // If we are also disjoint shallow we can stay in this mode - // Exception: reductions that are larger than the current - // domain are bad cause we can't do advances properly for - // our projection epoch - open_below |= (it->valid_fields() & current_mask); - it++; - } - else - { - // Otherwise we need a close operation - if (record_close_operations) - { - const FieldMask overlap = current_mask & it->valid_fields(); -#ifdef DEBUG_LEGION - assert(!!overlap); - //assert(!disjoint_close); -#endif - closer.record_close_operation(overlap); - // Advance the projection epochs - state.advance_projection_epochs(overlap); - } - it->filter(current_mask); - if (!it->valid_fields()) - it = state.field_states.erase(it); - else - it++; - } - break; - } case OPEN_REDUCE_PROJ: case OPEN_REDUCE_PROJ_DIRTY: { @@ -14424,8 +15378,6 @@ namespace Legion { #ifdef DEBUG_LEGION assert(!!overlap); #endif - // Advance the projection epochs - state.advance_projection_epochs(overlap); // If we're doing a disjoint close update the open // states accordingly if (disjoint_close) @@ -14441,7 +15393,8 @@ namespace Legion { as_partition_node()->row_source->color_space; FieldState new_state(close_usage, overlap, context->runtime->find_projection_function(0), - color_space, true/*disjoint*/); + color_space, NULL/*sharding func*/, + NULL/*sharding space*/, this); new_states.emplace(new_state); } else @@ -14472,7 +15425,7 @@ namespace Legion { { FieldState new_state(closer.user.usage, open_mask, proj_info.projection, proj_info.projection_space, - are_all_children_disjoint()); + proj_info.sharding_function, proj_info.sharding_space, this); new_states.emplace(new_state); } merge_new_field_states(state, new_states); @@ -14519,7 +15472,6 @@ namespace Legion { if (it->is_projection_state()) { closer.record_close_operation(overlap); - state.advance_projection_epochs(overlap); it->filter(overlap); flushed_fields |= overlap; } @@ -15374,8 +16326,6 @@ namespace Legion { { // Do a read only close here closer.record_close_operation(overlap); - // Advance the projection epochs - state.advance_projection_epochs(overlap); it->filter(current_mask); if (!it->valid_fields()) it = state.field_states.erase(it); @@ -15384,13 +16334,10 @@ namespace Legion { break; } case OPEN_READ_WRITE_PROJ: - case OPEN_READ_WRITE_PROJ_DISJOINT_SHALLOW: case OPEN_REDUCE_PROJ: { // Do the close here closer.record_close_operation(overlap); - // Advance the projection epochs - state.advance_projection_epochs(overlap); it->filter(current_mask); if (!it->valid_fields()) it = state.field_states.erase(it); @@ -15439,25 +16386,6 @@ namespace Legion { rez.serialize(it->first); rez.serialize(it->second); } - rez.serialize(state.projection_epochs.size()); - for (std::list::const_iterator pit = - state.projection_epochs.begin(); pit != - state.projection_epochs.end(); pit++) - { - rez.serialize((*pit)->epoch_id); - rez.serialize((*pit)->valid_fields); - rez.serialize((*pit)->write_projections.size()); - for (std::map >:: - const_iterator fit = (*pit)->write_projections.begin(); - fit != (*pit)->write_projections.end(); fit++) - { - rez.serialize(fit->first->projection_id); - rez.serialize(fit->second.size()); - for (std::set::const_iterator it = - fit->second.begin(); it != fit->second.end(); it++) - rez.serialize((*it)->handle); - } - } rez.serialize(state.field_states.size()); for (LegionList::aligned::const_iterator fit = state.field_states.begin(); fit != @@ -15468,15 +16396,14 @@ namespace Legion { rez.serialize(fit->redop); if (fit->open_state >= OPEN_READ_ONLY_PROJ) { -#ifdef DEBUG_LEGION - assert(fit->projection != NULL); -#endif - rez.serialize(fit->projection->projection_id); - rez.serialize(fit->projection_space->handle); + rez.serialize(fit->projections.size()); + for (std::set::const_iterator it = + fit->projections.begin(); it != fit->projections.end(); it++) + it->pack_summary(rez); } #ifdef DEBUG_LEGION else - assert(fit->projection == NULL); + assert(fit->projections.empty()); #endif rez.serialize(fit->open_children.size()); for (FieldMaskSet::const_iterator it = @@ -15517,35 +16444,6 @@ namespace Legion { derez.deserialize(redop); derez.deserialize(state.outstanding_reductions[redop]); } - size_t num_projection_epochs; - derez.deserialize(num_projection_epochs); - for (unsigned idx1 = 0; idx1 < num_projection_epochs; idx1++) - { - ProjectionEpochID epoch_id; - derez.deserialize(epoch_id); - FieldMask valid_fields; - derez.deserialize(valid_fields); - ProjectionEpoch *epoch = new ProjectionEpoch(epoch_id, valid_fields); - size_t num_projections; - derez.deserialize(num_projections); - for (unsigned idx2 = 0; idx2 < num_projections; idx2++) - { - ProjectionID proj_id; - derez.deserialize(proj_id); - ProjectionFunction *function = - context->runtime->find_projection_function(proj_id); - std::set &spaces = - epoch->write_projections[function]; - size_t num_doms; - derez.deserialize(num_doms); - for (unsigned idx3 = 0; idx3 < num_doms; idx3++) - { - IndexSpace handle; - derez.deserialize(handle); - spaces.insert(context->get_node(handle)); - } - } - } size_t num_field_states; derez.deserialize(num_field_states); state.field_states.resize(num_field_states); @@ -15559,12 +16457,11 @@ namespace Legion { derez.deserialize(fit->redop); if (fit->open_state >= OPEN_READ_ONLY_PROJ) { - ProjectionID proj_id; - derez.deserialize(proj_id); - fit->projection = context->runtime->find_projection_function(proj_id); - IndexSpace handle; - derez.deserialize(handle); - fit->projection_space = context->get_node(handle); + size_t num_summaries; + derez.deserialize(num_summaries); + for (unsigned idx = 0; idx < num_summaries; idx++) + fit->projections.insert( + ProjectionSummary::unpack_summary(derez, context)); } size_t num_open_children; derez.deserialize(num_open_children); @@ -15981,6 +16878,23 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + void RegionTreeNode::update_creation_set(const ShardMapping &mapping) + //-------------------------------------------------------------------------- + { + AutoLock n_lock(node_lock); + for (unsigned idx = 0; idx < mapping.size(); idx++) + { + const AddressSpaceID space = mapping[idx]; + if (space != context->runtime->address_space) +#ifdef LEGION_GC + update_remote_instances(space, false/*need lock*/); +#else + remote_instances.add(space); +#endif + } + } + //-------------------------------------------------------------------------- void RegionTreeNode::find_remote_instances(NodeSet &target_instances) //-------------------------------------------------------------------------- @@ -17522,7 +18436,7 @@ namespace Legion { LogicalPartition handle, Runtime *runtime) //-------------------------------------------------------------------------- { - return (handle.tree_id % runtime->runtime_stride); + return (handle.tree_id % runtime->total_address_spaces); } //-------------------------------------------------------------------------- diff --git a/runtime/legion/region_tree.h b/runtime/legion/region_tree.h index fb91b6066d..d9675fea42 100644 --- a/runtime/legion/region_tree.h +++ b/runtime/legion/region_tree.h @@ -153,23 +153,36 @@ namespace Legion { RegionTreeForest& operator=(const RegionTreeForest &rhs); public: IndexSpaceNode* create_index_space(IndexSpace handle, - const Domain *domain, DistributedID did, + const Domain *domain, + DistributedID did, + const bool notify_remote = true, + IndexSpaceExprID expr_id = 0, ApEvent ready = ApEvent::NO_AP_EVENT, + RtEvent initialized = RtEvent::NO_RT_EVENT, std::set *applied = NULL); IndexSpaceNode* create_union_space(IndexSpace handle, DistributedID did, const std::vector &sources, RtEvent initialized = RtEvent::NO_RT_EVENT, + const bool notify_remote = true, + IndexSpaceExprID expr_id = 0, std::set *applied = NULL); IndexSpaceNode* create_intersection_space(IndexSpace handle, DistributedID did, const std::vector &sources, RtEvent initialized = RtEvent::NO_RT_EVENT, + const bool noitfy_remote = true, + IndexSpaceExprID expr_id = 0, std::set *applied = NULL); IndexSpaceNode* create_difference_space(IndexSpace handle, DistributedID did, IndexSpace left, IndexSpace right, RtEvent initialized = RtEvent::NO_RT_EVENT, + const bool notify_remote = true, + IndexSpaceExprID expr_id = 0, std::set *applied = NULL); + void find_or_create_sharded_index_space(TaskContext *ctx, + IndexSpace handle, IndexSpace local, + DistributedID did); RtEvent create_pending_partition(TaskContext *ctx, IndexPartition pid, IndexSpace parent, @@ -178,8 +191,7 @@ namespace Legion { PartitionKind part_kind, DistributedID did, ApEvent partition_ready, - ApUserEvent partial_pending = ApUserEvent::NO_AP_USER_EVENT, - std::set *applied = NULL); + ApBarrier partial_pending = ApBarrier::NO_AP_BARRIER); void create_pending_cross_product(TaskContext *ctx, IndexPartition handle1, IndexPartition handle2, @@ -187,47 +199,80 @@ namespace Legion { PartitionKind kind, LegionColor &part_color, ApEvent domain_ready, - std::set &safe_events); - void compute_partition_disjointness(IndexPartition handle, - RtUserEvent ready_event); - void destroy_index_space(IndexSpace handle, - std::set &preconditions); + std::set &safe_events, + ShardID shard = 0, + size_t total_shards = 1); + // For control replication contexts + RtEvent create_pending_partition_shard(ShardID owner_shard, + ReplicateContext *ctx, + IndexPartition pid, + IndexSpace parent, + IndexSpace color_space, + LegionColor &partition_color, + PartitionKind part_kind, + DistributedID did, + ValueBroadcast *part_result, + ApEvent partition_ready, + ShardMapping &mapping, + RtEvent creation_ready, + ApBarrier partial_pending = ApBarrier::NO_AP_BARRIER); + void destroy_index_space(IndexSpace handle, std::set &applied, + const bool total_sharding_collective = false); void destroy_index_partition(IndexPartition handle, - std::set &preconditions); + std::set &applied, + const bool total_sharding_collective = false); public: ApEvent create_equal_partition(Operation *op, IndexPartition pid, - size_t granularity); + size_t granularity, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_weights(Operation *op, IndexPartition pid, const FutureMap &map, - size_t granularity); + size_t granularity, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_union(Operation *op, IndexPartition pid, IndexPartition handle1, - IndexPartition handle2); + IndexPartition handle2, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_intersection(Operation *op, IndexPartition pid, IndexPartition handle1, - IndexPartition handle2); + IndexPartition handle2, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_intersection(Operation *op, IndexPartition pid, IndexPartition part, - const bool dominates); + const bool dominates, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_difference(Operation *op, IndexPartition pid, IndexPartition handle1, - IndexPartition handle2); + IndexPartition handle2, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_restriction(IndexPartition pid, const void *transform, - const void *extent); + const void *extent, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_domain(Operation *op, IndexPartition pid, const FutureMap &future_map, - bool perform_intersections); + bool perform_intersections, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_cross_product_partitions(Operation *op, IndexPartition base, IndexPartition source, - LegionColor part_color); + LegionColor part_color, + ShardID shard = 0, + size_t total_shards = 1); public: ApEvent create_partition_by_field(Operation *op, IndexPartition pending, @@ -237,12 +282,16 @@ namespace Legion { IndexPartition pending, IndexPartition projection, const std::vector &instances, - ApEvent instances_ready); + ApEvent instances_ready, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_image_range(Operation *op, IndexPartition pending, IndexPartition projection, const std::vector &instances, - ApEvent instances_ready); + ApEvent instances_ready, + ShardID shard = 0, + size_t total_shards = 1); ApEvent create_partition_by_preimage(Operation *op, IndexPartition pending, IndexPartition projection, @@ -265,19 +314,18 @@ namespace Legion { bool check_association_field_size(IndexSpace is, FieldSpace fspace, FieldID fid); public: - IndexSpace find_pending_space(IndexPartition parent, - const void *realm_color, - TypeTag type_tag, - ApUserEvent &domain_ready); ApEvent compute_pending_space(Operation *op, IndexSpace result, const std::vector &handles, - bool is_union); + bool is_union, ShardID shard = 0, + size_t total_shards = 1); ApEvent compute_pending_space(Operation *op, IndexSpace result, IndexPartition handle, - bool is_union); + bool is_union, ShardID shard = 0, + size_t total_shards = 1); ApEvent compute_pending_space(Operation *op, IndexSpace result, IndexSpace initial, - const std::vector &handles); + const std::vector &handles, + ShardID shard = 0, size_t total_shards = 1); public: IndexPartition get_index_partition(IndexSpace parent, Color color); bool has_index_subspace(IndexPartition parent, @@ -303,30 +351,45 @@ namespace Legion { bool is_index_partition_complete(IndexPartition p); bool has_index_partition(IndexSpace parent, Color color); public: - void create_field_space(FieldSpace handle, DistributedID did, - std::set *applied = NULL); - void destroy_field_space(FieldSpace handle, - std::set &preconditions); - RtEvent create_field_space_allocator(FieldSpace handle); - void destroy_field_space_allocator(FieldSpace handle); + FieldSpaceNode* create_field_space(FieldSpace handle, DistributedID did, + const bool notify_remote = true, + RtEvent initialized = RtEvent::NO_RT_EVENT, + std::set *applied = NULL, + ShardMapping *shard_mapping = NULL); + void destroy_field_space(FieldSpace handle, std::set &applied, + const bool total_sharding_collective = false); + RtEvent create_field_space_allocator(FieldSpace handle, + bool sharded_owner_context = false, + bool owner_shard = false); + void destroy_field_space_allocator(FieldSpace handle, + bool sharded_owner_context = false, + bool owner_shard = false); // Return true if local is set to true and we actually performed the // allocation. It is an error if the field already existed and the // allocation was not local. - bool allocate_field(FieldSpace handle, size_t field_size, - FieldID fid, CustomSerdezID serdez_id); + RtEvent allocate_field(FieldSpace handle, size_t field_size, + FieldID fid, CustomSerdezID serdez_id, + bool sharded_non_owner = false); FieldSpaceNode* allocate_field(FieldSpace handle, ApEvent ready, - FieldID fid, CustomSerdezID serdez_id); - void free_field(FieldSpace handle, FieldID fid, - std::set &preconditions); - void allocate_fields(FieldSpace handle, const std::vector &sizes, + FieldID fid, CustomSerdezID serdez_id, + RtEvent &precondition, + bool sharded_non_owner = false); + void free_field(FieldSpace handle, FieldID fid, + std::set &applied, + bool sharded_non_owner = false); + RtEvent allocate_fields(FieldSpace handle, + const std::vector &sizes, const std::vector &resulting_fields, - CustomSerdezID serdez_id); + CustomSerdezID serdez_id, + bool sharded_non_owner = false); FieldSpaceNode* allocate_fields(FieldSpace handle, ApEvent ready, const std::vector &resulting_fields, - CustomSerdezID serdez_id); + CustomSerdezID serdez_id, RtEvent &precondition, + bool sharded_non_owner = false); void free_fields(FieldSpace handle, const std::vector &to_free, - std::set &preconditions); + std::set &applied, + bool sharded_non_owner = false); public: bool allocate_local_fields(FieldSpace handle, const std::vector &resulting_fields, @@ -336,7 +399,8 @@ namespace Legion { std::vector &new_indexes); void free_local_fields(FieldSpace handle, const std::vector &to_free, - const std::vector &indexes); + const std::vector &indexes, + const bool collective = false); void update_local_fields(FieldSpace handle, const std::vector &fields, const std::vector &sizes, @@ -351,10 +415,13 @@ namespace Legion { void get_field_space_fields(FieldSpace handle, std::vector &fields); public: - void create_logical_region(LogicalRegion handle, - std::set *applied = NULL); + RegionNode* create_logical_region(LogicalRegion handle, + const bool notify_remote = true, + RtEvent initialized = RtEvent::NO_RT_EVENT, + std::set *applied = NULL); void destroy_logical_region(LogicalRegion handle, - std::set &preconditions); + std::set &applied, + const bool total_sharding_collective = false); public: LogicalPartition get_logical_partition(LogicalRegion parent, IndexPartition handle); @@ -449,7 +516,8 @@ namespace Legion { const bool track_effects, const bool record_valid = true, const bool check_initialized = true, - const bool defer_copies = true); + const bool defer_copies = true, + const bool skip_output = false); // Return an event for when the copy-out effects of the // registration are done (e.g. for restricted coherence) ApEvent physical_perform_registration(UpdateAnalysis *analysis, @@ -560,6 +628,13 @@ namespace Legion { PredEvent true_guard, const PhysicalTraceInfo &trace_info, std::set &map_applied_events); + ApEvent overwrite_sharded(Operation *op, const unsigned index, + const RegionRequirement &req, + ShardedView *view, VersionInfo &version_info, + const PhysicalTraceInfo &trace_info, + const ApEvent precondition, + std::set &map_applied_events, + const bool add_restriction); InstanceRef create_external_instance(AttachOp *attach_op, const RegionRequirement &req, const std::vector &field_set); @@ -583,7 +658,8 @@ namespace Legion { void invalidate_fields(Operation *op, unsigned index, VersionInfo &version_info, const PhysicalTraceInfo &trace_info, - std::set &map_applied_events); + std::set &map_applied_events, + const bool collective = false); // Support for tracing void find_invalid_instances(Operation *op, unsigned index, VersionInfo &version_info, @@ -632,43 +708,52 @@ namespace Legion { RtEvent initialized, ApEvent is_ready = ApEvent::NO_AP_EVENT, IndexSpaceExprID expr_id = 0, + const bool notify_remote = true, std::set *applied = NULL); IndexSpaceNode* create_node(IndexSpace is, const void *realm_is, IndexPartNode *par, LegionColor color, DistributedID did, RtEvent initialized, ApUserEvent is_ready, + const bool notify_remote = true, std::set *applied = NULL); // We know the disjointness of the index partition IndexPartNode* create_node(IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *color_space, LegionColor color, bool disjoint,int complete, DistributedID did, ApEvent partition_ready, - ApUserEvent partial_pending, RtEvent init, + ApBarrier partial_pending, RtEvent init, + ShardMapping *shard_mapping = NULL, std::set *applied = NULL); // Give the event for when the disjointness information is ready IndexPartNode* create_node(IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *color_space,LegionColor color, RtEvent disjointness_ready_event,int complete, DistributedID did, ApEvent partition_ready, - ApUserEvent partial_pending, RtEvent init, + ApBarrier partial_pending, RtEvent init, + ShardMapping *shard_mapping = NULL, std::set *applied = NULL); - FieldSpaceNode* create_node(FieldSpace space, DistributedID did, - RtEvent initialized, - std::set *applied = NULL); - FieldSpaceNode* create_node(FieldSpace space, DistributedID did, + FieldSpaceNode* create_node(FieldSpace space, DistributedID did, + RtEvent init,const bool notify_remote = true, + std::set *applied = NULL, + ShardMapping *shard_mapping = NULL); + FieldSpaceNode* create_node(FieldSpace space, DistributedID did, RtEvent initialized, Deserializer &derez); - RegionNode* create_node(LogicalRegion r, PartitionNode *par, - RtEvent initialized, + RegionNode* create_node(LogicalRegion r, PartitionNode *par, + RtEvent init,const bool notify_remote = true, std::set *applied = NULL); PartitionNode* create_node(LogicalPartition p, RegionNode *par, std::set *applied = NULL); public: - IndexSpaceNode* get_node(IndexSpace space, RtEvent *defer = NULL); - IndexPartNode* get_node(IndexPartition part, RtEvent *defer = NULL); - FieldSpaceNode* get_node(FieldSpace space, RtEvent *defer = NULL); - RegionNode* get_node(LogicalRegion handle, bool need_check = true); + IndexSpaceNode* get_node(IndexSpace space, + RtEvent *defer = NULL, bool first = true); + IndexPartNode* get_node(IndexPartition part, + RtEvent *defer = NULL, bool first = true); + FieldSpaceNode* get_node(FieldSpace space, + RtEvent *defer = NULL, bool first = true); + RegionNode* get_node(LogicalRegion handle, + bool need_check = true, bool first = true); PartitionNode* get_node(LogicalPartition handle, bool need_check = true); - RegionNode* get_tree(RegionTreeID tid); + RegionNode* get_tree(RegionTreeID tid, bool first = true); // Request but don't block RtEvent request_node(IndexSpace space); // Find a local node if it exists and return it with reference @@ -689,6 +774,16 @@ namespace Legion { void remove_node(FieldSpace space); void remove_node(LogicalRegion handle, bool top); void remove_node(LogicalPartition handle); + public: + void record_pending_index_space(IndexSpaceID space); + void record_pending_partition(IndexPartitionID pid); + void record_pending_field_space(FieldSpaceID space); + void record_pending_region_tree(RegionTreeID tree); + public: + void revoke_pending_index_space(IndexSpaceID space); + void revoke_pending_partition(IndexPartitionID pid); + void revoke_pending_field_space(FieldSpaceID space); + void revoke_pending_region_tree(RegionTreeID tree); public: bool is_top_level_index_space(IndexSpace handle); bool is_top_level_region(LogicalRegion handle); @@ -859,6 +954,11 @@ namespace Legion { std::map index_part_requests; std::map field_space_requests; std::map region_tree_requests; + private: + std::map pending_index_spaces; + std::map pending_partitions; + std::map pending_field_spaces; + std::map pending_region_trees; private: // Index space operations std::map union_ops; @@ -986,9 +1086,11 @@ namespace Legion { virtual void add_expression_reference(bool expr_tree = false) = 0; virtual bool remove_expression_reference(bool expr_tree = false) = 0; virtual bool remove_operation(RegionTreeForest *forest) = 0; - virtual IndexSpaceNode* create_node(IndexSpace handle, - DistributedID did, RtEvent initialized, - std::set *applied) = 0; + virtual bool test_intersection_nonblocking(IndexSpaceExpression *expr, + RegionTreeForest *context, ApEvent &precondition, bool second = false); + virtual IndexSpaceNode* create_node(IndexSpace handle, DistributedID did, + RtEvent initialized, std::set *applied, + const bool notify_remote = true, IndexSpaceExprID expr_id = 0) = 0; virtual PieceIteratorImpl* create_piece_iterator(const void *piece_list, size_t piece_list_size, IndexSpaceNode *privilege_node) = 0; public: @@ -1187,11 +1289,14 @@ namespace Legion { virtual void add_expression_reference(bool expr_tree = false); virtual bool remove_expression_reference(bool expr_tree = false); virtual bool remove_operation(RegionTreeForest *forest) = 0; - virtual IndexSpaceNode* create_node(IndexSpace handle, - DistributedID did, RtEvent initialized, - std::set *applied) = 0; + virtual IndexSpaceNode* create_node(IndexSpace handle, DistributedID did, + RtEvent initialized, std::set *applied, + const bool notify_remote = true, IndexSpaceExprID expr_id = 0) = 0; protected: void record_remote_expression(AddressSpaceID target); + public: + static void handle_expression_invalidation(Deserializer &derez, + RegionTreeForest *forest); public: RegionTreeForest *const context; protected: @@ -1226,9 +1331,9 @@ namespace Legion { const bool top) = 0; virtual bool remove_operation(RegionTreeForest *forest) = 0; virtual bool remove_expression_reference(bool expr_tree = false); - virtual IndexSpaceNode* create_node(IndexSpace handle, - DistributedID did, RtEvent initialized, - std::set *applied) = 0; + virtual IndexSpaceNode* create_node(IndexSpace handle, DistributedID did, + RtEvent initialized, std::set *applied, + const bool notify_remote = true, IndexSpaceExprID expr_id = 0) = 0; virtual IndexSpaceExpression* find_congruence(void) = 0; virtual void activate_remote(void) = 0; public: @@ -1274,9 +1379,9 @@ namespace Legion { AddressSpaceID target, const bool top) = 0; virtual bool remove_operation(RegionTreeForest *forest) = 0; - virtual IndexSpaceNode* create_node(IndexSpace handle, - DistributedID did, RtEvent initialized, - std::set *applied); + virtual IndexSpaceNode* create_node(IndexSpace handle, DistributedID did, + RtEvent initialized, std::set *applied, + const bool notify_remote = true, IndexSpaceExprID expr_id = 0); virtual PieceIteratorImpl* create_piece_iterator(const void *piece_list, size_t piece_list_size, IndexSpaceNode *privilege_node); virtual IndexSpaceExpression* find_congruence(void) = 0; @@ -1657,6 +1762,8 @@ namespace Legion { virtual void send_semantic_info(AddressSpaceID target, SemanticTag tag, const void *buffer, size_t size, bool is_mutable, RtUserEvent ready = RtUserEvent::NO_RT_USER_EVENT) = 0; + public: + void update_creation_set(const ShardMapping &mapping); public: RegionTreeForest *const context; const unsigned depth; @@ -1726,14 +1833,16 @@ namespace Legion { }; class IndexSpaceSetFunctor { public: - IndexSpaceSetFunctor(Runtime *rt, AddressSpaceID src, Serializer &r) - : runtime(rt), source(src), rez(r) { } + IndexSpaceSetFunctor(Runtime *rt, AddressSpaceID src, + Serializer &r, ShardMapping *m) + : runtime(rt), source(src), rez(r), mapping(m) { } public: void apply(AddressSpaceID target); public: Runtime *const runtime; const AddressSpaceID source; Serializer &rez; + ShardMapping *const mapping; }; class InvalidFunctor { public: @@ -1830,7 +1939,8 @@ namespace Legion { virtual ApEvent get_expr_index_space(void *result, TypeTag tag, bool need_tight_result) = 0; virtual Domain get_domain(ApEvent &ready, bool need_tight) = 0; - virtual bool set_domain(const Domain &domain, AddressSpaceID space) = 0; + virtual bool set_domain(const Domain &domain, AddressSpaceID space, + ShardMapping *shard_mapping = NULL) = 0; virtual void tighten_index_space(void) = 0; virtual bool check_empty(void) = 0; virtual void pack_expression(Serializer &rez, AddressSpaceID target) = 0; @@ -1840,9 +1950,10 @@ namespace Legion { virtual void add_expression_reference(bool expr_tree = false); virtual bool remove_expression_reference(bool expr_tree = false); virtual bool remove_operation(RegionTreeForest *forest); - virtual IndexSpaceNode* create_node(IndexSpace handle, - DistributedID did, RtEvent initialized, - std::set *applied) = 0; + virtual IndexSpaceNode* create_node(IndexSpace handle, DistributedID did, + RtEvent initialized, std::set *applied, + const bool notify_remote = true, IndexSpaceExprID expr_id = 0) = 0; + virtual void create_sharded_alias(IndexSpace alias,DistributedID did) = 0; virtual PieceIteratorImpl* create_piece_iterator(const void *piece_list, size_t piece_list_size, IndexSpaceNode *privilege_node) = 0; public: @@ -1885,36 +1996,72 @@ namespace Legion { virtual ApEvent create_equal_children(Operation *op, IndexPartNode *partition, size_t granularity) = 0; + virtual ApEvent create_equal_children(Operation *op, + IndexPartNode *partition, + size_t granularity, + ShardID shard, + size_t total_shards) = 0; virtual ApEvent create_by_union(Operation *op, IndexPartNode *partition, IndexPartNode *left, IndexPartNode *right) = 0; + virtual ApEvent create_by_union(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards) = 0; virtual ApEvent create_by_intersection(Operation *op, IndexPartNode *partition, IndexPartNode *left, IndexPartNode *right) = 0; + virtual ApEvent create_by_intersection(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards) = 0; + virtual ApEvent create_by_intersection(Operation *op, + IndexPartNode *partition, + // Left is implicit "this" + IndexPartNode *right, + const bool dominates = false) = 0; virtual ApEvent create_by_intersection(Operation *op, IndexPartNode *partition, // Left is implicit "this" IndexPartNode *right, + ShardID shard, + size_t total_shards, const bool dominates = false) = 0; virtual ApEvent create_by_difference(Operation *op, IndexPartNode *partition, IndexPartNode *left, IndexPartNode *right) = 0; + virtual ApEvent create_by_difference(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards) = 0; // Called on color space and not parent virtual ApEvent create_by_restriction(IndexPartNode *partition, const void *transform, const void *extent, - int partition_dim) = 0; + int partition_dim, + ShardID shard, + size_t total_shards) = 0; virtual ApEvent create_by_domain(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - bool perform_intersections) = 0; + bool perform_intersections, + ShardID shard, + size_t total_shards) = 0; virtual ApEvent create_by_weights(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - size_t granularity) = 0; + size_t granularity, + ShardID shard, + size_t total_shards) = 0; virtual ApEvent create_by_field(Operation *op, IndexPartNode *partition, const std::vector &instances, @@ -1923,12 +2070,16 @@ namespace Legion { IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready) = 0; + ApEvent instances_ready, + ShardID shard, + size_t total_shards) = 0; virtual ApEvent create_by_image_range(Operation *op, IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready) = 0; + ApEvent instances_ready, + ShardID shard, + size_t total_shards) = 0; virtual ApEvent create_by_preimage(Operation *op, IndexPartNode *partition, IndexPartNode *projection, @@ -1964,6 +2115,10 @@ namespace Legion { virtual void validate_slicing(const std::vector &slice_spaces, MultiTask *task, MapperManager *mapper) = 0; virtual void log_launch_space(UniqueID op_id) = 0; + virtual IndexSpace create_shard_space(ShardingFunction *func, + ShardID shard, + IndexSpace shard_space) = 0; + virtual void destroy_shard_domain(const Domain &domain) = 0; public: const IndexSpace handle; IndexPartNode *const parent; @@ -2006,22 +2161,25 @@ namespace Legion { ApEvent get_realm_index_space(Realm::IndexSpace &result, bool need_tight_result); bool set_realm_index_space(AddressSpaceID source, - const Realm::IndexSpace &value); + const Realm::IndexSpace &value, + ShardMapping *shard_mapping = NULL); public: // From IndexSpaceExpression virtual ApEvent get_expr_index_space(void *result, TypeTag tag, bool need_tight_result); virtual Domain get_domain(ApEvent &ready, bool need_tight); - virtual bool set_domain(const Domain &domain, AddressSpaceID space); + virtual bool set_domain(const Domain &domain, AddressSpaceID space, + ShardMapping *shard_mapping = NULL); virtual void tighten_index_space(void); virtual bool check_empty(void); virtual void pack_expression(Serializer &rez, AddressSpaceID target); virtual void pack_expression_structure(Serializer &rez, AddressSpaceID target, const bool top); - virtual IndexSpaceNode* create_node(IndexSpace handle, - DistributedID did, RtEvent initialized, - std::set *applied); + virtual IndexSpaceNode* create_node(IndexSpace handle, DistributedID did, + RtEvent initialized, std::set *applied, + const bool notify_remote = true, IndexSpaceExprID expr_id = 0); + virtual void create_sharded_alias(IndexSpace alias, DistributedID did); virtual PieceIteratorImpl* create_piece_iterator(const void *piece_list, size_t piece_list_size, IndexSpaceNode *privilege_node); public: @@ -2063,50 +2221,87 @@ namespace Legion { virtual ApEvent create_equal_children(Operation *op, IndexPartNode *partition, size_t granularity); + virtual ApEvent create_equal_children(Operation *op, + IndexPartNode *partition, + size_t granularity, + ShardID shard, + size_t total_shards); virtual ApEvent create_by_union(Operation *op, IndexPartNode *partition, IndexPartNode *left, IndexPartNode *right); + virtual ApEvent create_by_union(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards); virtual ApEvent create_by_intersection(Operation *op, IndexPartNode *partition, IndexPartNode *left, IndexPartNode *right); + virtual ApEvent create_by_intersection(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards); + virtual ApEvent create_by_intersection(Operation *op, + IndexPartNode *partition, + // Left is implicit "this" + IndexPartNode *right, + const bool dominates = false); virtual ApEvent create_by_intersection(Operation *op, IndexPartNode *partition, // Left is implicit "this" IndexPartNode *right, + ShardID shard, + size_t total_shards, const bool dominates = false); virtual ApEvent create_by_difference(Operation *op, IndexPartNode *partition, IndexPartNode *left, IndexPartNode *right); + virtual ApEvent create_by_difference(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards); // Called on color space and not parent virtual ApEvent create_by_restriction(IndexPartNode *partition, const void *transform, const void *extent, - int partition_dim); + int partition_dim, + ShardID shard, + size_t total_shards); template ApEvent create_by_restriction_helper(IndexPartNode *partition, const Realm::Matrix &transform, - const Realm::Rect &extent); + const Realm::Rect &extent, + ShardID shard, size_t total_shards); virtual ApEvent create_by_domain(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - bool perform_intersections); + bool perform_intersections, + ShardID shard, size_t total_shards); template ApEvent create_by_domain_helper(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - bool perform_intersections); + bool perform_intersections, + ShardID shard, size_t total_shards); virtual ApEvent create_by_weights(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - size_t granularity); + size_t granularity, + ShardID shard, size_t total_shards); template ApEvent create_by_weight_helper(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - size_t granularity); + size_t granularity, + ShardID shard, size_t total_shards); virtual ApEvent create_by_field(Operation *op, IndexPartNode *partition, const std::vector &instances, @@ -2120,24 +2315,32 @@ namespace Legion { IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready); + ApEvent instances_ready, + ShardID shard, + size_t total_shards); template ApEvent create_by_image_helper(Operation *op, IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready); + ApEvent instances_ready, + ShardID shard, + size_t total_shards); virtual ApEvent create_by_image_range(Operation *op, IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready); + ApEvent instances_ready, + ShardID shard, + size_t total_shards); template ApEvent create_by_image_range_helper(Operation *op, IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready); + ApEvent instances_ready, + ShardID shard, + size_t total_shards); virtual ApEvent create_by_preimage(Operation *op, IndexPartNode *partition, IndexPartNode *projection, @@ -2249,6 +2452,10 @@ namespace Legion { virtual void validate_slicing(const std::vector &slice_spaces, MultiTask *task, MapperManager *mapper); virtual void log_launch_space(UniqueID op_id); + virtual IndexSpace create_shard_space(ShardingFunction *func, + ShardID shard, + IndexSpace shard_space); + virtual void destroy_shard_domain(const Domain &domain); public: bool contains_point(const Realm::Point &point); protected: @@ -2263,22 +2470,27 @@ namespace Legion { struct CreateByDomainHelper { public: CreateByDomainHelper(IndexSpaceNodeT *n, - IndexPartNode *p, Operation *o, - FutureMapImpl *fm, bool inter) - : node(n), partition(p), op(o), future_map(fm), intersect(inter) { } + IndexPartNode *p, Operation *o, + FutureMapImpl *fm, bool inter, + ShardID s, size_t total) + : node(n), partition(p), op(o), future_map(fm), + shard(s), total_shards(total), intersect(inter) { } public: template static inline void demux(CreateByDomainHelper *creator) { creator->result = creator->node->template create_by_domain_helper(creator->op, - creator->partition, creator->future_map, creator->intersect); + creator->partition, creator->future_map, creator->intersect, + creator->shard, creator->total_shards); } public: IndexSpaceNodeT *const node; IndexPartNode *const partition; Operation *const op; FutureMapImpl *const future_map; + const ShardID shard; + const size_t total_shards; const bool intersect; ApEvent result; }; @@ -2286,15 +2498,18 @@ namespace Legion { public: CreateByWeightHelper(IndexSpaceNodeT *n, IndexPartNode *p, Operation *o, - FutureMapImpl *fm, size_t g) - : node(n), partition(p), op(o), future_map(fm), granularity(g) { } + FutureMapImpl *fm, size_t g, + ShardID s, size_t total) + : node(n), partition(p), op(o), future_map(fm), + granularity(g), shard(s), total_shards(total) { } public: template static inline void demux(CreateByWeightHelper *creator) { creator->result = creator->node->template create_by_weight_helper(creator->op, - creator->partition, creator->future_map, creator->granularity); + creator->partition, creator->future_map, creator->granularity, + creator->shard, creator->total_shards); } public: IndexSpaceNodeT *const node; @@ -2302,6 +2517,8 @@ namespace Legion { Operation *const op; FutureMapImpl *const future_map; const size_t granularity; + const ShardID shard; + const size_t total_shards; ApEvent result; }; struct CreateByFieldHelper { @@ -2317,7 +2534,8 @@ namespace Legion { { creator->result = creator->node->template create_by_field_helper( - creator->op, creator->partition, creator->instances, creator->ready); + creator->op, creator->partition, creator->instances, + creator->ready); } public: IndexSpaceNodeT *node; @@ -2331,9 +2549,9 @@ namespace Legion { CreateByImageHelper(IndexSpaceNodeT *n, Operation *o, IndexPartNode *p, IndexPartNode *j, const std::vector &i, - ApEvent r) + ApEvent r, ShardID s, size_t t) : node(n), op(o), partition(p), projection(j), - instances(i), ready(r) { } + instances(i), ready(r), shard(s), total_shards(t) { } public: template static inline void demux(CreateByImageHelper *creator) @@ -2341,7 +2559,8 @@ namespace Legion { creator->result = creator->node->template create_by_image_helper( creator->op, creator->partition, creator->projection, - creator->instances, creator->ready); + creator->instances, creator->ready, creator->shard, + creator->total_shards); } public: IndexSpaceNodeT *node; @@ -2350,15 +2569,17 @@ namespace Legion { IndexPartNode *projection; const std::vector &instances; ApEvent ready, result; + ShardID shard; + size_t total_shards; }; struct CreateByImageRangeHelper { public: CreateByImageRangeHelper(IndexSpaceNodeT *n, Operation *o, IndexPartNode *p, IndexPartNode *j, const std::vector &i, - ApEvent r) + ApEvent r, ShardID s, size_t t) : node(n), op(o), partition(p), projection(j), - instances(i), ready(r) { } + instances(i), ready(r), shard(s), total_shards(t) { } public: template static inline void demux(CreateByImageRangeHelper *creator) @@ -2366,7 +2587,8 @@ namespace Legion { creator->result = creator->node->template create_by_image_range_helper( creator->op, creator->partition, creator->projection, - creator->instances, creator->ready); + creator->instances, creator->ready, creator->shard, + creator->total_shards); } public: IndexSpaceNodeT *node; @@ -2375,6 +2597,8 @@ namespace Legion { IndexPartNode *projection; const std::vector &instances; ApEvent ready, result; + ShardID shard; + size_t total_shards; }; struct CreateByPreimageHelper { public: @@ -2540,6 +2764,19 @@ namespace Legion { * A node for representing a generic index partition. */ class IndexPartNode : public IndexTreeNode { + public: + struct DisjointnessArgs : public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_DISJOINTNESS_TASK_ID; + public: + DisjointnessArgs(IndexPartition p, ValueBroadcast *c, bool own) + : LgTaskArgs(implicit_provenance), + pid(p), disjointness_collective(c), owner(own) { } + public: + const IndexPartition pid; + ValueBroadcast *const disjointness_collective; + const bool owner; + }; public: struct DynamicIndependenceArgs : public LgTaskArgs { @@ -2554,17 +2791,6 @@ namespace Legion { IndexPartNode *const parent; IndexSpaceNode *const left, *const right; }; - struct PendingChildArgs : public LgTaskArgs { - public: - static const LgTaskID TASK_ID = LG_PENDING_CHILD_TASK_ID; - public: - PendingChildArgs(IndexPartNode *par, LegionColor child) - : LgTaskArgs(implicit_provenance), - parent(par), pending_child(child) { } - public: - IndexPartNode *const parent; - const LegionColor pending_child; - }; struct SemanticRequestArgs : public LgTaskArgs { public: static const LgTaskID TASK_ID = LG_INDEX_PART_SEMANTIC_INFO_REQ_TASK_ID; @@ -2596,13 +2822,13 @@ namespace Legion { }; class RemoteDisjointnessFunctor { public: - RemoteDisjointnessFunctor(Serializer &r, Runtime *rt) - : rez(r), runtime(rt) { } + RemoteDisjointnessFunctor(Serializer &r, Runtime *rt, ShardMapping *m); public: void apply(AddressSpaceID target); public: Serializer &rez; Runtime *const runtime; + std::set skip_shard_spaces; }; class InvalidFunctor { public: @@ -2617,14 +2843,16 @@ namespace Legion { public: IndexPartNode(RegionTreeForest *ctx, IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *color_space, - LegionColor c, bool disjoint, int complete, + LegionColor c, bool disjoint, int complete, DistributedID did, ApEvent partition_ready, - ApUserEvent partial_pending, RtEvent init); + ApBarrier partial_pending, RtEvent initialized, + ShardMapping *mapping); IndexPartNode(RegionTreeForest *ctx, IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *color_space, LegionColor c, RtEvent disjointness_ready, - int complete, DistributedID did, ApEvent partition_ready, - ApUserEvent partial_pending, RtEvent init); + int complete, DistributedID did, + ApEvent partition_ready, ApBarrier partial_pending, + RtEvent initialized, ShardMapping *mapping); IndexPartNode(const IndexPartNode &rhs); virtual ~IndexPartNode(void); public: @@ -2662,9 +2890,9 @@ namespace Legion { void add_child(IndexSpaceNode *child); void add_tracker(PartitionTracker *tracker); size_t get_num_children(void) const; + void compute_disjointness(ValueBroadcast *collective, bool owner); void get_subspace_preconditions(std::set &preconditions); public: - void compute_disjointness(RtUserEvent ready_event); bool is_disjoint(bool from_app = false); bool are_disjoint(LegionColor c1, LegionColor c2, bool force_compute = false); @@ -2675,25 +2903,24 @@ namespace Legion { void record_remote_disjoint_ready(RtUserEvent ready); void record_remote_disjoint_result(const bool disjoint_result); public: - void add_pending_child(const LegionColor child_color, - ApUserEvent domain_ready); - bool get_pending_child(const LegionColor child_color, - ApUserEvent &domain_ready); - void remove_pending_child(const LegionColor child_color); - static void handle_pending_child_task(const void *args); - public: - ApEvent create_equal_children(Operation *op, size_t granularity); + ApEvent create_equal_children(Operation *op, size_t granularity, + ShardID shard, size_t total_shards); ApEvent create_by_weights(Operation *op, const FutureMap &weights, - size_t granularity); + size_t granularity, ShardID shard, size_t total_shards); ApEvent create_by_union(Operation *Op, - IndexPartNode *left, IndexPartNode *right); + IndexPartNode *left, IndexPartNode *right, + ShardID shard, size_t total_shards); ApEvent create_by_intersection(Operation *op, - IndexPartNode *left, IndexPartNode *right); + IndexPartNode *left, IndexPartNode *right, + ShardID shard, size_t total_shards); ApEvent create_by_intersection(Operation *op, IndexPartNode *original, - const bool dominates); + const bool dominates, + ShardID shard, size_t total_shards); ApEvent create_by_difference(Operation *op, - IndexPartNode *left, IndexPartNode *right); - ApEvent create_by_restriction(const void *transform, const void *extent); + IndexPartNode *left, IndexPartNode *right, + ShardID shard, size_t total_shards); + ApEvent create_by_restriction(const void *transform, const void *extent, + ShardID shard, size_t total_shards); ApEvent create_by_domain(FutureMapImpl *future_map); public: bool compute_complete(void); @@ -2702,6 +2929,8 @@ namespace Legion { bool dominates(IndexSpaceNode *other); bool dominates(IndexPartNode *other); public: + static void handle_disjointness_computation(const void *args, + RegionTreeForest *forest); static void handle_disjointness_test(IndexPartNode *parent, IndexSpaceNode *left, IndexSpaceNode *right); @@ -2731,7 +2960,8 @@ namespace Legion { const LegionColor total_children; const LegionColor max_linearized_color; const ApEvent partition_ready; - const ApUserEvent partial_pending; + const ApBarrier partial_pending; + ShardMapping *const shard_mapping; protected: RtEvent disjoint_ready; bool disjoint; @@ -2750,11 +2980,9 @@ namespace Legion { std::set > aliased_subspaces; std::vector partition_trackers; protected: - // Support for pending child spaces that still need to be computed - std::map pending_children; // Support for remote disjoint events being stored RtUserEvent remote_disjoint_ready; - }; + }; /** * \class IndexPartNodeT @@ -2769,12 +2997,14 @@ namespace Legion { IndexSpaceNode *par, IndexSpaceNode *color_space, LegionColor c, bool disjoint, int complete, DistributedID did, ApEvent partition_ready, - ApUserEvent pending, RtEvent initialized); + ApBarrier pending, RtEvent initialized, + ShardMapping *shard_mapping); IndexPartNodeT(RegionTreeForest *ctx, IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *color_space, - LegionColor c, RtEvent disjointness_ready, - int complete, DistributedID did, ApEvent partition_ready, - ApUserEvent pending, RtEvent initialized); + LegionColor c, RtEvent disjointness_ready, + int complete, DistributedID did, + ApEvent partition_ready, ApBarrier pending, + RtEvent initialized, ShardMapping *shard_mapping); IndexPartNodeT(const IndexPartNodeT &rhs); virtual ~IndexPartNodeT(void); public: @@ -2790,17 +3020,19 @@ namespace Legion { IndexPartCreator(RegionTreeForest *f, IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *cs, LegionColor c, bool d, int k, DistributedID id, - ApEvent r, ApUserEvent pend, RtEvent initialized) + ApEvent r, ApBarrier pend, RtEvent initialized, + ShardMapping *m) : forest(f), partition(p), parent(par), color_space(cs), color(c), disjoint(d), complete(k), did(id), ready(r), - pending(pend), init(initialized) { } + pending(pend), init(initialized), mapping(m) { } IndexPartCreator(RegionTreeForest *f, IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *cs, - LegionColor c, RtEvent d, int k, DistributedID id, - ApEvent r, ApUserEvent pend, RtEvent initialized) + LegionColor c, RtEvent d, int k, DistributedID id, + ApEvent r, ApBarrier pend, RtEvent initialized, + ShardMapping *m) : forest(f), partition(p), parent(par), color_space(cs), color(c), disjoint(false), complete(k), disjoint_ready(d), - did(id), ready(r), pending(pend), init(initialized) { } + did(id), ready(r), pending(pend), init(initialized), mapping(m) { } public: template static inline void demux(IndexPartCreator *creator) @@ -2808,13 +3040,15 @@ namespace Legion { if (creator->disjoint_ready.exists()) creator->result = new IndexPartNodeT(creator->forest, creator->partition, creator->parent, creator->color_space, - creator->color, creator->disjoint_ready, creator->complete, - creator->did, creator->ready, creator->pending, creator->init); + creator->color, creator->disjoint_ready, creator->complete, + creator->did, creator->ready, creator->pending, creator->init, + creator->mapping); else creator->result = new IndexPartNodeT(creator->forest, creator->partition, creator->parent, creator->color_space, - creator->color, creator->disjoint, creator->complete, - creator->did, creator->ready, creator->pending, creator->init); + creator->color, creator->disjoint, creator->complete, + creator->did, creator->ready, creator->pending, creator->init, + creator->mapping); } public: RegionTreeForest *const forest; @@ -2827,8 +3061,9 @@ namespace Legion { const RtEvent disjoint_ready; const DistributedID did; const ApEvent ready; - const ApUserEvent pending; + const ApBarrier pending; const RtEvent init; + ShardMapping *const mapping; IndexPartNode *result; }; @@ -2850,17 +3085,22 @@ namespace Legion { public: struct FieldInfo { public: - FieldInfo(void) : field_size(0), idx(0), serdez_id(0), local(false) { } - FieldInfo(size_t size, unsigned id, CustomSerdezID sid, bool loc=false) - : field_size(size), idx(id), serdez_id(sid), local(loc) { } - FieldInfo(ApEvent ready, unsigned id, CustomSerdezID sid,bool loc=false) + FieldInfo(void) : field_size(0), idx(0), serdez_id(0), + collective(false), local(false) { } + FieldInfo(size_t size, unsigned id, CustomSerdezID sid, + bool loc = false, bool collect = false) + : field_size(size), idx(id), serdez_id(sid), + collective(collect), local(loc) { } + FieldInfo(ApEvent ready, unsigned id, CustomSerdezID sid, + bool loc = false, bool collect = false) : field_size(0), size_ready(ready), idx(id), serdez_id(sid), - local(loc) { } + collective(collect), local(loc) { } public: size_t field_size; ApEvent size_ready; unsigned idx; CustomSerdezID serdez_id; + bool collective; bool local; }; struct FindTargetsFunctor { @@ -2918,8 +3158,8 @@ namespace Legion { const RtUserEvent to_trigger; }; public: - FieldSpaceNode(FieldSpace sp, RegionTreeForest *ctx, - DistributedID did, RtEvent initialized); + FieldSpaceNode(FieldSpace sp, RegionTreeForest *ctx, DistributedID did, + RtEvent initialized, ShardMapping *shard_mapping); FieldSpaceNode(FieldSpace sp, RegionTreeForest *ctx, DistributedID did, RtEvent initialized, Deserializer &derez); FieldSpaceNode(const FieldSpaceNode &rhs); @@ -2964,25 +3204,33 @@ namespace Legion { Deserializer &derez, AddressSpaceID source); public: RtEvent create_allocator(AddressSpaceID source, - RtUserEvent ready = RtUserEvent::NO_RT_USER_EVENT); - RtEvent destroy_allocator(AddressSpaceID source); + RtUserEvent ready = RtUserEvent::NO_RT_USER_EVENT, + bool sharded_owner_context = false, bool owner_shard = false); + RtEvent destroy_allocator(AddressSpaceID source, + bool sharded_owner_context = false, bool owner_shard = false); public: RtEvent allocate_field(FieldID fid, size_t size, - CustomSerdezID serdez_id); + CustomSerdezID serdez_id, + bool sharded_non_owner = false); RtEvent allocate_field(FieldID fid, ApEvent size_ready, - CustomSerdezID serdez_id); + CustomSerdezID serdez_id, + bool sharded_non_owner = false); RtEvent allocate_fields(const std::vector &sizes, const std::vector &fids, - CustomSerdezID serdez_id); + CustomSerdezID serdez_id, + bool sharded_non_owner = false); RtEvent allocate_fields(ApEvent sizes_ready, const std::vector &fids, - CustomSerdezID serdez_id); + CustomSerdezID serdez_id, + bool sharded_non_owner = false); void update_field_size(FieldID fid, size_t field_size, std::set &update_events, AddressSpaceID source); void free_field(FieldID fid, AddressSpaceID source, - std::set &applied); + std::set &applied, + bool sharded_non_owner = false); void free_fields(const std::vector &to_free, - AddressSpaceID source, std::set &applied); + AddressSpaceID source, std::set &applied, + bool sharded_non_owner = false); public: bool allocate_local_fields(const std::vector &fields, const std::vector &sizes, @@ -2990,12 +3238,15 @@ namespace Legion { const std::set &indexes, std::vector &new_indexes); void free_local_fields(const std::vector &to_free, - const std::vector &indexes); + const std::vector &indexes, + const bool collective); void update_local_fields(const std::vector &fields, const std::vector &sizes, const std::vector &serdez_ids, const std::vector &indexes); void remove_local_fields(const std::vector &to_removes); + public: + void update_creation_set(const ShardMapping &mapping); public: bool has_field(FieldID fid); size_t get_field_size(FieldID fid); @@ -3109,7 +3360,8 @@ namespace Legion { RtUserEvent to_trigger = RtUserEvent::NO_RT_USER_EVENT) const; void record_read_only_infos(const std::map &infos); void process_allocator_response(Deserializer &derez); - void process_allocator_invalidation(RtUserEvent done); + void process_allocator_invalidation(RtUserEvent done, + bool flush, bool merge); void process_allocator_flush(Deserializer &derez); void process_allocator_free(Deserializer &derez, AddressSpaceID source); protected: @@ -3370,7 +3622,8 @@ namespace Legion { const FieldMask &check_mask); public: inline FieldSpaceNode* get_column_source(void) const - { return column_source; } + { return column_source; } + void update_creation_set(const ShardMapping &mapping); void find_remote_instances(NodeSet &target_instances); public: RegionTreeForest *const context; diff --git a/runtime/legion/region_tree.inl b/runtime/legion/region_tree.inl index dba4b96d66..40695182d4 100644 --- a/runtime/legion/region_tree.inl +++ b/runtime/legion/region_tree.inl @@ -1262,18 +1262,24 @@ namespace Legion { //-------------------------------------------------------------------------- template IndexSpaceNode* IndexSpaceOperationT::create_node(IndexSpace handle, - DistributedID did, RtEvent initialized, std::set *applied) + DistributedID did, RtEvent initialized, + std::set *applied, + const bool notify_remote, IndexSpaceExprID new_expr_id) //-------------------------------------------------------------------------- { + if (new_expr_id == 0) + new_expr_id = expr_id; AutoLock i_lock(inter_lock, 1, false/*exclusive*/); if (is_index_space_tight) return context->create_node(handle, &tight_index_space, false/*domain*/, - NULL/*parent*/, 0/*color*/, did,initialized, - realm_index_space_ready, expr_id, applied); + NULL/*parent*/, 0/*color*/, did, initialized, + realm_index_space_ready, new_expr_id, + notify_remote, applied); else return context->create_node(handle, &realm_index_space, false/*domain*/, - NULL/*parent*/, 0/*color*/, did,initialized, - realm_index_space_ready, expr_id, applied); + NULL/*parent*/, 0/*color*/, did, initialized, + realm_index_space_ready, new_expr_id, + notify_remote, applied); } //-------------------------------------------------------------------------- @@ -2213,8 +2219,8 @@ namespace Legion { //-------------------------------------------------------------------------- template - bool IndexSpaceNodeT::set_realm_index_space( - AddressSpaceID source, const Realm::IndexSpace &value) + bool IndexSpaceNodeT::set_realm_index_space(AddressSpaceID source, + const Realm::IndexSpace &value, ShardMapping *mapping) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -2234,7 +2240,7 @@ namespace Legion { Runtime::trigger_event(realm_index_space_set); // We're not the owner, if this is not from the owner then // send a message there telling the owner that it is set - if (source != owner_space) + if ((source != owner_space) && (mapping == NULL)) { Serializer rez; { @@ -2261,7 +2267,7 @@ namespace Legion { rez.serialize(handle); pack_index_space(rez, false/*include size*/); } - IndexSpaceSetFunctor functor(context->runtime, source, rez); + IndexSpaceSetFunctor functor(context->runtime, source, rez, mapping); map_over_remote_instances(functor); } } @@ -2298,11 +2304,11 @@ namespace Legion { //-------------------------------------------------------------------------- template bool IndexSpaceNodeT::set_domain(const Domain &domain, - AddressSpaceID source) + AddressSpaceID source, ShardMapping *shard_mapping) //-------------------------------------------------------------------------- { const DomainT realm_space = domain; - return set_realm_index_space(source, realm_space); + return set_realm_index_space(source, realm_space, shard_mapping); } //-------------------------------------------------------------------------- @@ -2396,6 +2402,20 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + template + void IndexSpaceNodeT::create_sharded_alias(IndexSpace alias, + DistributedID alias_did) + //-------------------------------------------------------------------------- + { + // Have to wait at least until we get our index space set + if (!realm_index_space_set.has_triggered()) + realm_index_space_set.wait(); + context->create_node(alias, &realm_index_space_set, false/*is domain*/, + NULL/*parent*/, 0/*color*/, alias_did, initialized, + index_space_ready, expr_id/*alis*/,false/*notify remote*/); + } + //-------------------------------------------------------------------------- template void IndexSpaceNodeT::pack_expression_structure(Serializer &rez, @@ -2416,9 +2436,13 @@ namespace Legion { //-------------------------------------------------------------------------- template IndexSpaceNode* IndexSpaceNodeT::create_node(IndexSpace new_handle, - DistributedID did, RtEvent initialized, std::set *applied) + DistributedID did, RtEvent initialized, + std::set *applied, + const bool notify_remote, IndexSpaceExprID new_expr_id) //-------------------------------------------------------------------------- { + if (new_expr_id == 0) + new_expr_id = expr_id; #ifdef DEBUG_LEGION assert(handle.get_type_tag() == new_handle.get_type_tag()); #endif @@ -2426,7 +2450,7 @@ namespace Legion { const ApEvent ready = get_realm_index_space(local_space, false/*tight*/); return context->create_node(new_handle, &local_space, false/*domain*/, NULL/*parent*/, 0/*color*/, did, initialized, - ready, expr_id, applied); + ready, new_expr_id, notify_remote, applied); } //-------------------------------------------------------------------------- @@ -3028,6 +3052,7 @@ namespace Legion { assert(partition->parent == this); #endif const size_t count = partition->color_space->get_volume(); + // Common case is not control replication std::vector > subspaces; Realm::ProfilingRequestSet requests; if (context->runtime->profiler != NULL) @@ -3052,23 +3077,20 @@ namespace Legion { LegionSpy::log_deppart_events(op->get_unique_op_id(),handle,ready,result); #endif // Enumerate the colors and assign the spaces - unsigned subspace_index = 0; if (partition->total_children == partition->max_linearized_color) { for (LegionColor color = 0; color < partition->total_children; color++) { IndexSpaceNodeT *child = static_cast*>(partition->get_child(color)); -#ifdef DEBUG_LEGION - assert(subspace_index < subspaces.size()); -#endif if (child->set_realm_index_space(context->runtime->address_space, - subspaces[subspace_index++])) + subspaces[color])) assert(false); // should never hit this } } else { + unsigned subspace_index = 0; ColorSpaceIterator *itr = partition->color_space->create_color_space_iterator(); while (itr->is_valid()) @@ -3088,6 +3110,91 @@ namespace Legion { return result; } + //-------------------------------------------------------------------------- + template + ApEvent IndexSpaceNodeT::create_equal_children(Operation *op, + IndexPartNode *partition, size_t granularity, + ShardID shard, size_t total_shards) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(partition->parent == this); + assert(total_shards > 0); +#endif + const size_t count = partition->color_space->get_volume(); + std::set done_events; + if (!realm_index_space_set.has_triggered()) + realm_index_space_set.wait(); + // In the case of control replication we do things + // one point at a time for the subspaces owned by this shard + if (partition->total_children == partition->max_linearized_color) + { + for (LegionColor color = shard; + color < partition->max_linearized_color; color+=total_shards) + { + Realm::ProfilingRequestSet requests; + if (context->runtime->profiler != NULL) + context->runtime->profiler->add_partition_request(requests, + op, DEP_PART_EQUAL); + Realm::IndexSpace subspace; + ApEvent result(realm_index_space.create_equal_subspace(count, + granularity, color, subspace, requests, index_space_ready)); + IndexSpaceNodeT *child = + static_cast*>(partition->get_child(color)); + if (child->set_realm_index_space(context->runtime->address_space, + subspace)) + assert(false); // should never hit this + done_events.insert(result); + } + } + else + { + unsigned subspace_index = 0; + // Always use the partitions color space + ColorSpaceIterator *itr = + partition->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + subspace_index++; + itr->yield_color(); + if (!itr->is_valid()) + break; + } + while (itr->is_valid()) + { + const LegionColor color = itr->yield_color(); + Realm::ProfilingRequestSet requests; + if (context->runtime->profiler != NULL) + context->runtime->profiler->add_partition_request(requests, + op, DEP_PART_EQUAL); + Realm::IndexSpace subspace; + ApEvent result(realm_index_space.create_equal_subspace(count, + granularity, subspace_index++, subspace, requests, + index_space_ready)); + IndexSpaceNodeT *child = + static_cast*>(partition->get_child(color)); + if (child->set_realm_index_space(context->runtime->address_space, + subspace)) + assert(false); // should never hit this + done_events.insert(result); + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + subspace_index++; + itr->yield_color(); + if (!itr->is_valid()) + break; + } + } + delete itr; + } + if (!done_events.empty()) + return Runtime::merge_events(NULL, done_events); + else + return ApEvent::NO_AP_EVENT; + } + //-------------------------------------------------------------------------- template ApEvent IndexSpaceNodeT::create_by_union(Operation *op, @@ -3162,7 +3269,7 @@ namespace Legion { op, DEP_PART_UNIONS); if (op->has_execution_fence_event()) preconditions.insert(op->get_execution_fence_event()); - ApEvent precondition = Runtime::merge_events(NULL, preconditions); + const ApEvent precondition = Runtime::merge_events(NULL, preconditions); ApEvent result(Realm::IndexSpace::compute_unions( lhs_spaces, rhs_spaces, subspaces, requests, precondition)); #ifdef LEGION_DISABLE_EVENT_PRUNING @@ -3214,6 +3321,126 @@ namespace Legion { return result; } + //-------------------------------------------------------------------------- + template + ApEvent IndexSpaceNodeT::create_by_union(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(partition->parent == this); + assert(total_shards > 1); +#endif + std::vector > lhs_spaces; + std::vector > rhs_spaces; + std::vector colors; + std::set preconditions; + // First we need to fill in all the subspaces + if (partition->total_children == partition->max_linearized_color) + { + for (LegionColor color = shard; + color < partition->total_children; color += total_shards) + { + IndexSpaceNodeT *left_child = + static_cast*>(left->get_child(color)); + IndexSpaceNodeT *right_child = + static_cast*>(right->get_child(color)); + lhs_spaces.resize(lhs_spaces.size() + 1); + rhs_spaces.resize(rhs_spaces.size() + 1); + ApEvent left_ready = + left_child->get_realm_index_space(lhs_spaces.back(), + false/*tight*/); + ApEvent right_ready = + right_child->get_realm_index_space(rhs_spaces.back(), + false/*tight*/); + colors.push_back(color); + if (!left_ready.has_triggered()) + preconditions.insert(left_ready); + if (!right_ready.has_triggered()) + preconditions.insert(right_ready); + } + } + else + { + // Always use the partitions color space + ColorSpaceIterator *itr = + partition->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } + while (itr->is_valid()) + { + const LegionColor color = itr->yield_color(); + IndexSpaceNodeT *left_child = + static_cast*>(partition->get_child(color)); + IndexSpaceNodeT *right_child = + static_cast*>(right->get_child(color)); + lhs_spaces.resize(lhs_spaces.size() + 1); + rhs_spaces.resize(rhs_spaces.size() + 1); + ApEvent left_ready = + left_child->get_realm_index_space(lhs_spaces.back(), + false/*tight*/); + ApEvent right_ready = + right_child->get_realm_index_space(rhs_spaces.back(), + false/*tight*/); + colors.push_back(color); + if (!left_ready.has_triggered()) + preconditions.insert(left_ready); + if (!right_ready.has_triggered()) + preconditions.insert(right_ready); + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } + } + delete itr; + } + if (colors.empty()) + return ApEvent::NO_AP_EVENT; + std::vector > subspaces; + Realm::ProfilingRequestSet requests; + if (context->runtime->profiler != NULL) + context->runtime->profiler->add_partition_request(requests, + op, DEP_PART_UNIONS); + const ApEvent precondition = Runtime::merge_events(NULL, preconditions); + ApEvent result(Realm::IndexSpace::compute_unions( + lhs_spaces, rhs_spaces, subspaces, requests, precondition)); +#ifdef LEGION_DISABLE_EVENT_PRUNING + if (!result.exists() || (result == precondition)) + { + ApUserEvent new_result = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, new_result); + result = new_result; + } +#endif +#ifdef LEGION_SPY + LegionSpy::log_deppart_events(op->get_unique_op_id(), + handle, precondition, result); +#endif + // Now set the index spaces for the results + for (unsigned idx = 0; idx < colors.size(); idx++) + { + IndexSpaceNodeT *child = + static_cast*>( + partition->get_child(colors[idx])); + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[idx])) + assert(false); // should never hit this + } + return result; + } + //-------------------------------------------------------------------------- template ApEvent IndexSpaceNodeT::create_by_intersection(Operation *op, @@ -3288,7 +3515,7 @@ namespace Legion { op, DEP_PART_INTERSECTIONS); if (op->has_execution_fence_event()) preconditions.insert(op->get_execution_fence_event()); - ApEvent precondition = Runtime::merge_events(NULL, preconditions); + const ApEvent precondition = Runtime::merge_events(NULL, preconditions); ApEvent result(Realm::IndexSpace::compute_intersections( lhs_spaces, rhs_spaces, subspaces, requests, precondition)); #ifdef LEGION_DISABLE_EVENT_PRUNING @@ -3340,6 +3567,126 @@ namespace Legion { return result; } + //-------------------------------------------------------------------------- + template + ApEvent IndexSpaceNodeT::create_by_intersection(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(partition->parent == this); + assert(total_shards > 1); +#endif + std::vector > lhs_spaces; + std::vector > rhs_spaces; + std::vector colors; + std::set preconditions; + // First we need to fill in all the subspaces + if (partition->total_children == partition->max_linearized_color) + { + for (LegionColor color = shard; + color < partition->total_children; color += total_shards) + { + IndexSpaceNodeT *left_child = + static_cast*>(left->get_child(color)); + IndexSpaceNodeT *right_child = + static_cast*>(right->get_child(color)); + lhs_spaces.resize(lhs_spaces.size() + 1); + rhs_spaces.resize(rhs_spaces.size() + 1); + ApEvent left_ready = + left_child->get_realm_index_space(lhs_spaces.back(), + false/*tight*/); + ApEvent right_ready = + right_child->get_realm_index_space(rhs_spaces.back(), + false/*tight*/); + colors.push_back(color); + if (!left_ready.has_triggered()) + preconditions.insert(left_ready); + if (!right_ready.has_triggered()) + preconditions.insert(right_ready); + } + } + else + { + // Always use the partitions color space + ColorSpaceIterator *itr = + partition->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } + while (itr->is_valid()) + { + const LegionColor color = itr->yield_color(); + IndexSpaceNodeT *left_child = + static_cast*>(partition->get_child(color)); + IndexSpaceNodeT *right_child = + static_cast*>(right->get_child(color)); + lhs_spaces.resize(lhs_spaces.size() + 1); + rhs_spaces.resize(rhs_spaces.size() + 1); + ApEvent left_ready = + left_child->get_realm_index_space(lhs_spaces.back(), + false/*tight*/); + ApEvent right_ready = + right_child->get_realm_index_space(rhs_spaces.back(), + false/*tight*/); + colors.push_back(color); + if (!left_ready.has_triggered()) + preconditions.insert(left_ready); + if (!right_ready.has_triggered()) + preconditions.insert(right_ready); + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } + } + delete itr; + } + if (colors.empty()) + return ApEvent::NO_AP_EVENT; + std::vector > subspaces; + Realm::ProfilingRequestSet requests; + if (context->runtime->profiler != NULL) + context->runtime->profiler->add_partition_request(requests, + op, DEP_PART_INTERSECTIONS); + const ApEvent precondition = Runtime::merge_events(NULL, preconditions); + ApEvent result(Realm::IndexSpace::compute_intersections( + lhs_spaces, rhs_spaces, subspaces, requests, precondition)); +#ifdef LEGION_DISABLE_EVENT_PRUNING + if (!result.exists() || (result == precondition)) + { + ApUserEvent new_result = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, new_result); + result = new_result; + } +#endif +#ifdef LEGION_SPY + LegionSpy::log_deppart_events(op->get_unique_op_id(), + handle, precondition, result); +#endif + // Now set the index spaces for the results + for (unsigned idx = 0; idx < colors.size(); idx++) + { + IndexSpaceNodeT *child = + static_cast*>( + partition->get_child(colors[idx])); + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[idx])) + assert(false); // should never hit this + } + return result; + } + //-------------------------------------------------------------------------- template ApEvent IndexSpaceNodeT::create_by_intersection(Operation *op, @@ -3467,50 +3814,308 @@ namespace Legion { return result; } + //-------------------------------------------------------------------------- + template + ApEvent IndexSpaceNodeT::create_by_intersection(Operation *op, + IndexPartNode *partition, + // Left is implicit "this" + IndexPartNode *right, + ShardID shard, + size_t total_shards, + const bool dominates) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(partition->parent == this); + assert(total_shards > 1); +#endif + std::vector > rhs_spaces; + std::vector colors; + std::set preconditions; + // First we need to fill in all the subspaces + if (partition->total_children == partition->max_linearized_color) + { + for (LegionColor color = shard; + color < partition->total_children; color += total_shards) + { + IndexSpaceNodeT *right_child = + static_cast*>(right->get_child(color)); + rhs_spaces.resize(rhs_spaces.size() + 1); + ApEvent right_ready = + right_child->get_realm_index_space(rhs_spaces.back(), + false/*tight*/); + colors.push_back(color); + if (right_ready.exists()) + preconditions.insert(right_ready); + } + } + else + { + ColorSpaceIterator *itr = + partition->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } + while (itr->is_valid()) + { + const LegionColor color = itr->yield_color(); + + IndexSpaceNodeT *right_child = + static_cast*>(right->get_child(color)); + rhs_spaces.resize(rhs_spaces.size() + 1); + ApEvent right_ready = + right_child->get_realm_index_space(rhs_spaces.back(), + false/*tight*/); + colors.push_back(color); + if (right_ready.exists()) + preconditions.insert(right_ready); + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } + } + delete itr; + } + if (colors.empty()) + return ApEvent::NO_AP_EVENT; + ApEvent result, precondition; + std::vector > subspaces; + if (dominates) + { + // If we've been told that we dominate then there is no + // need to event do the intersection tests at all + subspaces.swap(rhs_spaces); + result = Runtime::merge_events(NULL, preconditions); + } + else + { + Realm::ProfilingRequestSet requests; + if (context->runtime->profiler != NULL) + context->runtime->profiler->add_partition_request(requests, + op, DEP_PART_INTERSECTIONS); + Realm::IndexSpace lhs_space; + ApEvent left_ready = get_realm_index_space(lhs_space, false/*tight*/); + if (left_ready.exists()) + preconditions.insert(left_ready); + if (op->has_execution_fence_event()) + preconditions.insert(op->get_execution_fence_event()); + precondition = Runtime::merge_events(NULL, preconditions); + result = ApEvent(Realm::IndexSpace::compute_intersections( + lhs_space, rhs_spaces, subspaces, requests, precondition)); + } +#ifdef LEGION_DISABLE_EVENT_PRUNING + if (!result.exists() || (result == precondition)) + { + ApUserEvent new_result = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, new_result); + result = new_result; + } +#endif +#ifdef LEGION_SPY + LegionSpy::log_deppart_events(op->get_unique_op_id(), + handle, precondition, result); +#endif + // Now set the index spaces for the results + for (unsigned idx = 0; idx < colors.size(); idx++) + { + IndexSpaceNodeT *child = + static_cast*>( + partition->get_child(colors[idx])); + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[idx])) + assert(false); // should never hit this + } + return result; + } + + //-------------------------------------------------------------------------- + template + ApEvent IndexSpaceNodeT::create_by_difference(Operation *op, + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(partition->parent == this); +#endif + const size_t count = partition->color_space->get_volume(); + std::vector > lhs_spaces(count); + std::vector > rhs_spaces(count); + std::set preconditions; + // First we need to fill in all the subspaces + unsigned subspace_index = 0; + if (partition->total_children == partition->max_linearized_color) + { + for (LegionColor color = 0; color < partition->total_children; color++) + { + IndexSpaceNodeT *left_child = + static_cast*>(left->get_child(color)); + IndexSpaceNodeT *right_child = + static_cast*>(right->get_child(color)); +#ifdef DEBUG_LEGION + assert(subspace_index < count); +#endif + ApEvent left_ready = + left_child->get_realm_index_space(lhs_spaces[subspace_index], + false/*tight*/); + ApEvent right_ready = + right_child->get_realm_index_space(rhs_spaces[subspace_index++], + false/*tight*/); + if (left_ready.exists()) + preconditions.insert(left_ready); + if (right_ready.exists()) + preconditions.insert(right_ready); + } + } + else + { + ColorSpaceIterator *itr = + partition->color_space->create_color_space_iterator(); + while (itr->is_valid()) + { + const LegionColor color = itr->yield_color(); + IndexSpaceNodeT *left_child = + static_cast*>(partition->get_child(color)); + IndexSpaceNodeT *right_child = + static_cast*>(right->get_child(color)); +#ifdef DEBUG_LEGION + assert(subspace_index < count); +#endif + ApEvent left_ready = + left_child->get_realm_index_space(lhs_spaces[subspace_index], + false/*tight*/); + ApEvent right_ready = + right_child->get_realm_index_space(rhs_spaces[subspace_index++], + false/*tight*/); + if (left_ready.exists()) + preconditions.insert(left_ready); + if (right_ready.exists()) + preconditions.insert(right_ready); + } + delete itr; + } + std::vector > subspaces; + Realm::ProfilingRequestSet requests; + if (context->runtime->profiler != NULL) + context->runtime->profiler->add_partition_request(requests, + op, DEP_PART_DIFFERENCES); + if (op->has_execution_fence_event()) + preconditions.insert(op->get_execution_fence_event()); + const ApEvent precondition = Runtime::merge_events(NULL, preconditions); + ApEvent result(Realm::IndexSpace::compute_differences( + lhs_spaces, rhs_spaces, subspaces, requests, precondition)); +#ifdef LEGION_DISABLE_EVENT_PRUNING + if (!result.exists() || (result == precondition)) + { + ApUserEvent new_result = Runtime::create_ap_user_event(NULL); + Runtime::trigger_event(NULL, new_result); + result = new_result; + } +#endif +#ifdef LEGION_SPY + LegionSpy::log_deppart_events(op->get_unique_op_id(), + handle, precondition, result); +#endif + // Now set the index spaces for the results + subspace_index = 0; + if (partition->total_children == partition->max_linearized_color) + { + for (LegionColor color = 0; color < partition->total_children; color++) + { + IndexSpaceNodeT *child = + static_cast*>(partition->get_child(color)); +#ifdef DEBUG_LEGION + assert(subspace_index < subspaces.size()); +#endif + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[subspace_index++])) + assert(false); // should never hit this + } + } + else + { + ColorSpaceIterator *itr = + partition->color_space->create_color_space_iterator(); + while (itr->is_valid()) + { + const LegionColor color = itr->yield_color(); + IndexSpaceNodeT *child = + static_cast*>(partition->get_child(color)); +#ifdef DEBUG_LEGION + assert(subspace_index < subspaces.size()); +#endif + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[subspace_index++])) + assert(false); // should never hit this + } + delete itr; + } + return result; + } + //-------------------------------------------------------------------------- template ApEvent IndexSpaceNodeT::create_by_difference(Operation *op, - IndexPartNode *partition, - IndexPartNode *left, - IndexPartNode *right) + IndexPartNode *partition, + IndexPartNode *left, + IndexPartNode *right, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION assert(partition->parent == this); + assert(total_shards > 1); #endif - const size_t count = partition->color_space->get_volume(); - std::vector > lhs_spaces(count); - std::vector > rhs_spaces(count); + std::vector > lhs_spaces; + std::vector > rhs_spaces; + std::vector colors; std::set preconditions; // First we need to fill in all the subspaces - unsigned subspace_index = 0; if (partition->total_children == partition->max_linearized_color) { - for (LegionColor color = 0; color < partition->total_children; color++) + for (LegionColor color = shard; + color < partition->total_children; color += total_shards) { IndexSpaceNodeT *left_child = static_cast*>(left->get_child(color)); IndexSpaceNodeT *right_child = static_cast*>(right->get_child(color)); -#ifdef DEBUG_LEGION - assert(subspace_index < count); -#endif + lhs_spaces.resize(lhs_spaces.size() + 1); + rhs_spaces.resize(rhs_spaces.size() + 1); ApEvent left_ready = - left_child->get_realm_index_space(lhs_spaces[subspace_index], + left_child->get_realm_index_space(lhs_spaces.back(), false/*tight*/); ApEvent right_ready = - right_child->get_realm_index_space(rhs_spaces[subspace_index++], + right_child->get_realm_index_space(rhs_spaces.back(), false/*tight*/); - if (left_ready.exists()) + colors.push_back(color); + if (!left_ready.has_triggered()) preconditions.insert(left_ready); - if (right_ready.exists()) + if (!right_ready.has_triggered()) preconditions.insert(right_ready); } } else { + // Always use the partitions color space ColorSpaceIterator *itr = partition->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } while (itr->is_valid()) { const LegionColor color = itr->yield_color(); @@ -3518,30 +4123,30 @@ namespace Legion { static_cast*>(partition->get_child(color)); IndexSpaceNodeT *right_child = static_cast*>(right->get_child(color)); -#ifdef DEBUG_LEGION - assert(subspace_index < count); -#endif + lhs_spaces.resize(lhs_spaces.size() + 1); + rhs_spaces.resize(rhs_spaces.size() + 1); ApEvent left_ready = - left_child->get_realm_index_space(lhs_spaces[subspace_index], + left_child->get_realm_index_space(lhs_spaces.back(), false/*tight*/); ApEvent right_ready = - right_child->get_realm_index_space(rhs_spaces[subspace_index++], + right_child->get_realm_index_space(rhs_spaces.back(), false/*tight*/); - if (left_ready.exists()) + colors.push_back(color); + if (!left_ready.has_triggered()) preconditions.insert(left_ready); - if (right_ready.exists()) + if (!right_ready.has_triggered()) preconditions.insert(right_ready); } delete itr; } + if (colors.empty()) + return ApEvent::NO_AP_EVENT; std::vector > subspaces; Realm::ProfilingRequestSet requests; if (context->runtime->profiler != NULL) context->runtime->profiler->add_partition_request(requests, - op, DEP_PART_DIFFERENCES); - if (op->has_execution_fence_event()) - preconditions.insert(op->get_execution_fence_event()); - ApEvent precondition = Runtime::merge_events(NULL, preconditions); + op, DEP_PART_DIFFERENCES); + const ApEvent precondition = Runtime::merge_events(NULL, preconditions); ApEvent result(Realm::IndexSpace::compute_differences( lhs_spaces, rhs_spaces, subspaces, requests, precondition)); #ifdef LEGION_DISABLE_EVENT_PRUNING @@ -3557,38 +4162,14 @@ namespace Legion { handle, precondition, result); #endif // Now set the index spaces for the results - subspace_index = 0; - if (partition->total_children == partition->max_linearized_color) - { - for (LegionColor color = 0; color < partition->total_children; color++) - { - IndexSpaceNodeT *child = - static_cast*>(partition->get_child(color)); -#ifdef DEBUG_LEGION - assert(subspace_index < subspaces.size()); -#endif - if (child->set_realm_index_space(context->runtime->address_space, - subspaces[subspace_index++])) - assert(false); // should never hit this - } - } - else + for (unsigned idx = 0; idx < colors.size(); idx++) { - ColorSpaceIterator *itr = - partition->color_space->create_color_space_iterator(); - while (itr->is_valid()) - { - const LegionColor color = itr->yield_color(); - IndexSpaceNodeT *child = - static_cast*>(partition->get_child(color)); -#ifdef DEBUG_LEGION - assert(subspace_index < subspaces.size()); -#endif - if (child->set_realm_index_space(context->runtime->address_space, - subspaces[subspace_index++])) - assert(false); // should never hit this - } - delete itr; + IndexSpaceNodeT *child = + static_cast*>( + partition->get_child(colors[idx])); + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[idx])) + assert(false); // should never hit this } return result; } @@ -3599,7 +4180,9 @@ namespace Legion { IndexPartNode *partition, const void *tran, const void *ext, - int partition_dim) + int partition_dim, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -3608,15 +4191,15 @@ namespace Legion { #endif switch (partition_dim) { -#define DIMFUNC(D2) \ - case D2: \ +#define DIMFUNC(D1) \ + case D1: \ { \ - const Realm::Matrix *transform = \ - static_cast*>(tran); \ - const Realm::Rect *extent = \ - static_cast*>(ext); \ - return create_by_restriction_helper(partition, \ - *transform, *extent); \ + const Realm::Matrix *transform = \ + static_cast*>(tran); \ + const Realm::Rect *extent = \ + static_cast*>(ext); \ + return create_by_restriction_helper(partition, *transform, \ + *extent, shard, total_shards); \ } LEGION_FOREACH_N(DIMFUNC) #undef DIMFUNC @@ -3631,7 +4214,8 @@ namespace Legion { ApEvent IndexSpaceNodeT::create_by_restriction_helper( IndexPartNode *partition, const Realm::Matrix &transform, - const Realm::Rect &extent) + const Realm::Rect &extent, + ShardID shard, size_t total_shards) //-------------------------------------------------------------------------- { // Get the parent index space in case it has a sparsity map @@ -3649,14 +4233,16 @@ namespace Legion { for (Realm::PointInRectIterator color_itr(rect_itr.rect); color_itr.valid; color_itr.step()) { + // Get the legion color + LegionColor color = linearize_color(&color_itr.p, + handle.get_type_tag()); + if ((total_shards > 1) && ((color % total_shards) != shard)) + continue; // Copy the index space from the parent Realm::IndexSpace child_is = parent_is; // Compute the new bounds and intersect it with the parent bounds child_is.bounds = parent_is.bounds.intersection( extent + transform * color_itr.p); - // Get the legion color - LegionColor color = linearize_color(&color_itr.p, - handle.get_type_tag()); // Get the appropriate child IndexSpaceNodeT *child = static_cast*>(partition->get_child(color)); @@ -3675,15 +4261,17 @@ namespace Legion { ApEvent IndexSpaceNodeT::create_by_domain(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - bool perform_intersections) + bool perform_intersections, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION assert(partition->parent == this); #endif // Demux the color space type to do the actual operations - CreateByDomainHelper creator(this, partition, op, - future_map, perform_intersections); + CreateByDomainHelper creator(this, partition, op, future_map, + perform_intersections, shard, total_shards); NT_TemplateHelper::demux( partition->color_space->handle.get_type_tag(), &creator); return creator.result; @@ -3694,14 +4282,17 @@ namespace Legion { ApEvent IndexSpaceNodeT::create_by_weights(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - size_t granularity) + size_t granularity, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION assert(partition->parent == this); #endif // Demux the color space type to do the actual operations - CreateByWeightHelper creator(this, partition, op, future_map,granularity); + CreateByWeightHelper creator(this, partition, op, future_map, + granularity, shard, total_shards); NT_TemplateHelper::demux( partition->color_space->handle.get_type_tag(), &creator); return creator.result; @@ -3719,8 +4310,7 @@ namespace Legion { assert(partition->parent == this); #endif // Demux the color space type to do the actual operations - CreateByFieldHelper creator(this, op, partition, - instances, instances_ready); + CreateByFieldHelper creator(this,op,partition,instances,instances_ready); NT_TemplateHelper::demux( partition->color_space->handle.get_type_tag(), &creator); return creator.result; @@ -3732,7 +4322,8 @@ namespace Legion { template template ApEvent IndexSpaceNodeT::create_by_domain_helper(Operation *op, IndexPartNode *partition, FutureMapImpl *future_map, - bool perform_intersections) + bool perform_intersections, + ShardID local_shard, size_t total_shards) //-------------------------------------------------------------------------- { IndexSpaceNodeT *color_space = @@ -3756,52 +4347,125 @@ namespace Legion { parent_ready = op->get_execution_fence_event(); } } - // Make all the entries for the color space - for (Realm::IndexSpaceIterator - rect_iter(realm_color_space); rect_iter.valid; rect_iter.step()) - { - for (Realm::PointInRectIterator - itr(rect_iter.rect); itr.valid; itr.step()) + DomainT future_map_space = future_map->get_domain(); + // We'll check for the case where future map space is the same as + // the color space as we can implement this much more effeciently + // and it is the most common case for + if ((future_map_space.bounds == realm_color_space.bounds) && + (future_map_space.sparsity.id == realm_color_space.sparsity.id)) + { + // Fast case for when we know that the bounds of future map + // is the same as the color space of the new partition + // Get the shard-local futures for this future map + std::map shard_local_futures; + future_map->get_shard_local_futures(shard_local_futures); + for (std::map::const_iterator it = + shard_local_futures.begin(); it != shard_local_futures.end(); it++) { - LegionColor child_color = color_space->linearize_color(&itr.p, + const Point point = it->first; + LegionColor child_color = color_space->linearize_color(&point, color_space->handle.get_type_tag()); IndexSpaceNodeT *child = static_cast*>( partition->get_child(child_color)); - Realm::IndexSpace child_space; - const DomainPoint key(Point(itr.p)); - FutureImpl *future = future_map->find_future(key); - if (future != NULL) + if (it->second->get_untyped_size(true/*internal*/) != sizeof(Domain)) + REPORT_LEGION_ERROR(ERROR_INVALID_PARTITION_BY_DOMAIN_VALUE, + "An invalid future size was found in a partition by domain " + "call. All futures must contain Domain objects.") + const Domain *domain = static_cast( + it->second->get_untyped_result(true, NULL, true/*internal*/)); + const DomainT domaint = *domain; + Realm::IndexSpace child_space = domaint; + if (perform_intersections) { - if (future->get_untyped_size(true/*internal*/) != - sizeof(Domain)) - REPORT_LEGION_ERROR(ERROR_INVALID_PARTITION_BY_DOMAIN_VALUE, - "An invalid future size was found in a partition by domain " - "call. All futures must contain Domain objects.") - const Domain *domain = static_cast( - future->get_untyped_result(true, NULL, true/*internal*/)); - const DomainT domaint = *domain; - child_space = domaint; - if (perform_intersections) - { - Realm::ProfilingRequestSet requests; - if (context->runtime->profiler != NULL) - context->runtime->profiler->add_partition_request(requests, - op, DEP_PART_INTERSECTIONS); - Realm::IndexSpace result; - ApEvent ready(Realm::IndexSpace::compute_intersection( - parent_space, child_space, result, requests, parent_ready)); - child_space = result; - if (ready.exists()) - result_events.insert(ready); - } + Realm::ProfilingRequestSet requests; + if (context->runtime->profiler != NULL) + context->runtime->profiler->add_partition_request(requests, + op, DEP_PART_INTERSECTIONS); + Realm::IndexSpace result; + ApEvent ready(Realm::IndexSpace::compute_intersection( + parent_space, child_space, result, requests, parent_ready)); + child_space = result; + if (ready.exists()) + result_events.insert(ready); } - else - child_space = Realm::IndexSpace::make_empty(); if (child->set_realm_index_space(context->runtime->address_space, child_space)) assert(false); // should never hit this } } + else + { + // This is the slow case where the color space is not the same + // as the domain of the future map + // Make all the entries for the color space + ShardID next_local_shard = 0; + const Domain &future_map_domain = future_map->get_domain(); + for (Realm::IndexSpaceIterator + rect_iter(realm_color_space); rect_iter.valid; rect_iter.step()) + { + for (Realm::PointInRectIterator + itr(rect_iter.rect); itr.valid; itr.step()) + { + const DomainPoint key(Point(itr.p)); + FutureImpl *future = NULL; + // Check to see if the future is contained in the future map + if (future_map_domain.contains(key)) + { + // If the future map can have this future, see if it is + // a local future + future = future_map->find_shard_local_future(key); + if (future == NULL) + continue; + } + else + { + // If this not a point in the future map we round-robin + // responsibility for these across the shards + const ShardID shard = next_local_shard++; + if (next_local_shard == total_shards) + next_local_shard = 0; + if (shard != local_shard) + continue; + } + LegionColor child_color = color_space->linearize_color(&itr.p, + color_space->handle.get_type_tag()); + IndexSpaceNodeT *child = + static_cast*>( + partition->get_child(child_color)); + Realm::IndexSpace child_space; + if (future != NULL) + { + if (future->get_untyped_size(true/*internal*/) != + sizeof(Domain)) + REPORT_LEGION_ERROR(ERROR_INVALID_PARTITION_BY_DOMAIN_VALUE, + "An invalid future size was found in a partition by domain " + "call. All futures must contain Domain objects.") + const Domain *domain = static_cast( + future->get_untyped_result(true, NULL, true/*internal*/)); + const DomainT domaint = *domain; + child_space = domaint; + if (perform_intersections) + { + Realm::ProfilingRequestSet requests; + if (context->runtime->profiler != NULL) + context->runtime->profiler->add_partition_request(requests, + op, DEP_PART_INTERSECTIONS); + Realm::IndexSpace result; + ApEvent ready(Realm::IndexSpace::compute_intersection( + parent_space, child_space, result, requests, parent_ready)); + child_space = result; + if (ready.exists()) + result_events.insert(ready); + } + } + else + child_space = Realm::IndexSpace::make_empty(); + if (child->set_realm_index_space(context->runtime->address_space, + child_space)) + assert(false); // should never hit this + } + } + } if (result_events.empty()) return ApEvent::NO_AP_EVENT; return Runtime::merge_events(NULL, result_events); @@ -3810,7 +4474,8 @@ namespace Legion { //-------------------------------------------------------------------------- template template ApEvent IndexSpaceNodeT::create_by_weight_helper(Operation *op, - IndexPartNode *partition, FutureMapImpl *future_map, size_t granularity) + IndexPartNode *partition, FutureMapImpl *future_map, + size_t granularity, ShardID shard, size_t total_shards) //-------------------------------------------------------------------------- { IndexSpaceNodeT *color_space = @@ -3824,6 +4489,8 @@ namespace Legion { std::vector long_weights; std::vector child_colors(count); unsigned color_index = 0; + std::map futures; + future_map->get_all_futures(futures); // Make all the entries for the color space for (Realm::IndexSpaceIterator rect_iter(realm_color_space); rect_iter.valid; rect_iter.step()) @@ -3832,11 +4499,13 @@ namespace Legion { itr(rect_iter.rect); itr.valid; itr.step()) { const DomainPoint key(Point(itr.p)); - FutureImpl *future = future_map->find_future(key); - if (future == NULL) + std::map::const_iterator finder = + futures.find(key); + if (finder == futures.end()) REPORT_LEGION_ERROR(ERROR_MISSING_PARTITION_BY_WEIGHT_COLOR, "A partition by weight call is missing an entry for a " "color in the color space. All colors must be present.") + FutureImpl *future = future_map->unpack_future(finder->second); const size_t future_size = future->get_untyped_size(true/*internal*/); if (future_size == sizeof(int)) { @@ -3900,12 +4569,17 @@ namespace Legion { #endif for (unsigned idx = 0; idx < count; idx++) { - IndexSpaceNodeT *child = - static_cast*>( - partition->get_child(child_colors[idx])); - if (child->set_realm_index_space(context->runtime->address_space, - subspaces[idx])) - assert(false); // should never hit this + if ((idx % total_shards) == shard) + { + IndexSpaceNodeT *child = + static_cast*>( + partition->get_child(child_colors[idx])); + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[idx])) + assert(false); // should never hit this + } + else // We don't need this because another shard handled it + subspaces[idx].destroy(); } return result; } @@ -3923,19 +4597,24 @@ namespace Legion { // Enumerate the color space Realm::IndexSpace realm_color_space; color_space->get_realm_index_space(realm_color_space, true/*tight*/); + std::vector > colors; + std::vector child_colors; + const TypeTag color_type = color_space->handle.get_type_tag(); const size_t num_colors = realm_color_space.volume(); - std::vector > colors(num_colors); + colors.resize(num_colors); + child_colors.resize(num_colors); unsigned index = 0; for (Realm::IndexSpaceIterator rect_iter(realm_color_space); rect_iter.valid; rect_iter.step()) { for (Realm::PointInRectIterator - itr(rect_iter.rect); itr.valid; itr.step()) + itr(rect_iter.rect); itr.valid; itr.step(), index++) { #ifdef DEBUG_LEGION assert(index < colors.size()); #endif - colors[index++] = itr.p; + colors[index] = itr.p; + child_colors[index] = color_space->linearize_color(&itr.p,color_type); } } // Translate the instances to realm field data descriptors @@ -3973,6 +4652,9 @@ namespace Legion { ApEvent precondition = Runtime::merge_events(NULL, preconditions); ApEvent result(local_space.create_subspaces_by_field( descriptors, colors, subspaces, requests, precondition)); +#ifdef DEBUG_LEGION + assert(child_colors.size() == subspaces.size()); +#endif #ifdef LEGION_DISABLE_EVENT_PRUNING if (!result.exists() || (result == precondition)) { @@ -3986,12 +4668,10 @@ namespace Legion { precondition, result); #endif // Update the children with the names of their subspaces - for (unsigned idx = 0; idx < colors.size(); idx++) + for (unsigned idx = 0; idx < child_colors.size(); idx++) { - LegionColor child_color = color_space->linearize_color(&colors[idx], - color_space->handle.get_type_tag()); IndexSpaceNodeT *child = static_cast*>( - partition->get_child(child_color)); + partition->get_child(child_colors[idx])); if (child->set_realm_index_space(context->runtime->address_space, subspaces[idx])) assert(false); // should never hit this @@ -4007,15 +4687,17 @@ namespace Legion { IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready) + ApEvent instances_ready, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION assert(partition->parent == this); #endif // Demux the projection type to do the actual operations - CreateByImageHelper creator(this, op, partition, projection, - instances, instances_ready); + CreateByImageHelper creator(this, op, partition, projection, instances, + instances_ready, shard, total_shards); NT_TemplateHelper::demux( projection->handle.get_type_tag(), &creator); return creator.result; @@ -4029,22 +4711,39 @@ namespace Legion { IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready) + ApEvent instances_ready, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { + std::vector > sources; + std::vector child_colors; + const size_t volume = projection->color_space->get_volume(); + if (total_shards > 1) + { + const size_t max_children = (volume + total_shards - 1) / total_shards; + sources.reserve(max_children); + child_colors.reserve(max_children); + } + else + { + sources.reserve(volume); + child_colors.reserve(volume); + } // Get the index spaces of the projection partition - std::vector > - sources(projection->color_space->get_volume()); std::set preconditions; if (partition->total_children == partition->max_linearized_color) { // Always use the partitions color space - for (LegionColor color = 0; color < partition->total_children; color++) + for (LegionColor color = shard; + color < partition->total_children; color+=total_shards) { + child_colors.push_back(color); // Get the child of the projection partition IndexSpaceNodeT *child = static_cast*>(projection->get_child(color)); - ApEvent ready = child->get_realm_index_space(sources[color], + sources.resize(sources.size() + 1); + ApEvent ready = child->get_realm_index_space(sources.back(), false/*tight*/); if (ready.exists()) preconditions.insert(ready); @@ -4052,23 +4751,35 @@ namespace Legion { } else { - unsigned index = 0; // Always use the partitions color space ColorSpaceIterator *itr = partition->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } while (itr->is_valid()) { const LegionColor color = itr->yield_color(); + child_colors.push_back(color); // Get the child of the projection partition IndexSpaceNodeT *child = static_cast*>(projection->get_child(color)); -#ifdef DEBUG_LEGION - assert(index < sources.size()); -#endif - ApEvent ready = child->get_realm_index_space(sources[index++], + sources.resize(sources.size() + 1); + ApEvent ready = child->get_realm_index_space(sources.back(), false/*tight*/); if (ready.exists()) preconditions.insert(ready); + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } } delete itr; } @@ -4106,6 +4817,10 @@ namespace Legion { ApEvent precondition = Runtime::merge_events(NULL, preconditions); ApEvent result(local_space.create_subspaces_by_image(descriptors, sources, subspaces, requests, precondition)); +#ifdef DEBUG_LEGION + // This should be true after the call + assert(child_colors.size() == subspaces.size()); +#endif #ifdef LEGION_DISABLE_EVENT_PRUNING if (!result.exists() || (result == precondition)) { @@ -4119,37 +4834,15 @@ namespace Legion { precondition, result); #endif // Update the child subspaces of the image - if (partition->total_children == partition->max_linearized_color) - { - for (LegionColor color = 0; color < partition->total_children; color++) - { - // Get the child of the projection partition - IndexSpaceNodeT *child = - static_cast*>(partition->get_child(color)); - if (child->set_realm_index_space(context->runtime->address_space, - subspaces[color])) - assert(false); // should never hit this - } - } - else + for (unsigned idx = 0; idx < child_colors.size(); idx++) { - unsigned index = 0; - ColorSpaceIterator *itr = - partition->color_space->create_color_space_iterator(); - while (itr->is_valid()) - { - const LegionColor color = itr->yield_color(); - // Get the child of the projection partition - IndexSpaceNodeT *child = - static_cast*>(partition->get_child(color)); -#ifdef DEBUG_LEGION - assert(index < subspaces.size()); -#endif - if (child->set_realm_index_space(context->runtime->address_space, - subspaces[index++])) - assert(false); // should never hit this - } - delete itr; + // Get the child of the projection partition + IndexSpaceNodeT *child = + static_cast*>( + partition->get_child(child_colors[idx])); + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[idx])) + assert(false); // should never hit this } return result; } @@ -4162,7 +4855,9 @@ namespace Legion { IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready) + ApEvent instances_ready, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -4170,7 +4865,7 @@ namespace Legion { #endif // Demux the projection type to do the actual operations CreateByImageRangeHelper creator(this, op, partition, projection, - instances, instances_ready); + instances, instances_ready, shard, total_shards); NT_TemplateHelper::demux( projection->handle.get_type_tag(), &creator); return creator.result; @@ -4185,22 +4880,39 @@ namespace Legion { IndexPartNode *partition, IndexPartNode *projection, const std::vector &instances, - ApEvent instances_ready) + ApEvent instances_ready, + ShardID shard, + size_t total_shards) //-------------------------------------------------------------------------- { + std::vector > sources; + std::vector child_colors; + const size_t volume = projection->color_space->get_volume(); + if (total_shards > 1) + { + const size_t max_children = (volume + total_shards - 1) / total_shards; + sources.reserve(max_children); + child_colors.reserve(max_children); + } + else + { + sources.reserve(volume); + child_colors.reserve(volume); + } // Get the index spaces of the projection partition - std::vector > - sources(projection->color_space->get_volume()); std::set preconditions; if (partition->total_children == partition->max_linearized_color) { // Always use the partitions color space - for (LegionColor color = 0; color < partition->total_children; color++) + for (LegionColor color = shard; + color < partition->total_children; color+=total_shards) { + child_colors.push_back(color); // Get the child of the projection partition IndexSpaceNodeT *child = static_cast*>(projection->get_child(color)); - ApEvent ready = child->get_realm_index_space(sources[color], + sources.resize(sources.size() + 1); + ApEvent ready = child->get_realm_index_space(sources.back(), false/*tight*/); if (ready.exists()) preconditions.insert(ready); @@ -4208,23 +4920,35 @@ namespace Legion { } else { - unsigned index = 0; ColorSpaceIterator *itr = partition->color_space->create_color_space_iterator(); + // Skip ahead if necessary for our shard + for (unsigned idx = 0; idx < shard; idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } // Always use the partitions color space while (itr->is_valid()) { const LegionColor color = itr->yield_color(); + child_colors.push_back(color); // Get the child of the projection partition IndexSpaceNodeT *child = static_cast*>(projection->get_child(color)); -#ifdef DEBUG_LEGION - assert(index < sources.size()); -#endif - ApEvent ready = child->get_realm_index_space(sources[index++], + sources.resize(sources.size() + 1); + ApEvent ready = child->get_realm_index_space(sources.back(), false/*tight*/); if (ready.exists()) preconditions.insert(ready); + // Skip ahead for the next color if necessary + for (unsigned idx = 0; idx < (total_shards-1); idx++) + { + itr->yield_color(); + if (!itr->is_valid()) + break; + } } delete itr; } @@ -4262,6 +4986,10 @@ namespace Legion { ApEvent precondition = Runtime::merge_events(NULL, preconditions); ApEvent result(local_space.create_subspaces_by_image(descriptors, sources, subspaces, requests, precondition)); +#ifdef DEBUG_LEGION + // Should be true after the call + assert(subspaces.size() == child_colors.size()); +#endif #ifdef LEGION_DISABLE_EVENT_PRUNING if (!result.exists() || (result == precondition)) { @@ -4275,37 +5003,14 @@ namespace Legion { precondition, result); #endif // Update the child subspaces of the image - if (partition->total_children == partition->max_linearized_color) - { - for (LegionColor color = 0; color < partition->total_children; color++) - { - // Get the child of the projection partition - IndexSpaceNodeT *child = - static_cast*>(partition->get_child(color)); - if (child->set_realm_index_space(context->runtime->address_space, - subspaces[color])) - assert(false); // should never hit this - } - } - else + for (unsigned idx = 0; idx < child_colors.size(); idx++) { - unsigned index = 0; - ColorSpaceIterator *itr = - partition->color_space->create_color_space_iterator(); - while (itr->is_valid()) - { - const LegionColor color = itr->yield_color(); - // Get the child of the projection partition - IndexSpaceNodeT *child = - static_cast*>(partition->get_child(color)); -#ifdef DEBUG_LEGION - assert(index < subspaces.size()); -#endif - if (child->set_realm_index_space(context->runtime->address_space, - subspaces[index++])) - assert(false); // should never hit this - } - delete itr; + IndexSpaceNodeT *child = + static_cast*>( + partition->get_child(child_colors[idx])); + if (child->set_realm_index_space(context->runtime->address_space, + subspaces[idx])) + assert(false); // should never hit this } return result; } @@ -5089,6 +5794,65 @@ namespace Legion { Rect(itr.rect)); } + //-------------------------------------------------------------------------- + template + IndexSpace IndexSpaceNodeT::create_shard_space( + ShardingFunction *func, ShardID shard, IndexSpace shard_space) + //-------------------------------------------------------------------------- + { + DomainT local_space; + get_realm_index_space(local_space, true/*tight*/); + Domain shard_domain; + if (shard_space != handle) + context->find_launch_space_domain(shard_space, shard_domain); + else + shard_domain = local_space; + std::vector > shard_points; + if (!func->functor->is_invertible()) + { + for (Realm::IndexSpaceIterator rect_itr(local_space); + rect_itr.valid; rect_itr.step()) + { + for (Realm::PointInRectIterator itr(rect_itr.rect); + itr.valid; itr.step()) + { + const ShardID point_shard = + func->find_owner(DomainPoint(Point(itr.p)), shard_domain); + if (point_shard == shard) + shard_points.push_back(itr.p); + } + } + } + else + { + std::vector domain_points; + func->functor->invert(shard, Domain(local_space), shard_domain, + func->total_shards, domain_points); + shard_points.resize(domain_points.size()); + for (unsigned idx = 0; idx < domain_points.size(); idx++) + shard_points[idx] = Point(domain_points[idx]); + } + if (shard_points.empty()) + return IndexSpace::NO_SPACE; + // Another useful case is if all the points are in the shard then + // we can return ourselves as the result + if (shard_points.size() == get_volume()) + return handle; + Realm::IndexSpace realm_is(shard_points); + const Domain domain((DomainT(realm_is))); + return context->runtime->find_or_create_index_slice_space(domain, + handle.get_type_tag()); + } + + //-------------------------------------------------------------------------- + template + void IndexSpaceNodeT::destroy_shard_domain(const Domain &domain) + //-------------------------------------------------------------------------- + { + DomainT to_destroy = domain; + to_destroy.destroy(); + } + ///////////////////////////////////////////////////////////// // Templated Color Space Iterator ///////////////////////////////////////////////////////////// @@ -5132,10 +5896,10 @@ namespace Legion { IndexSpaceNode *par, IndexSpaceNode *cs, LegionColor c, bool disjoint, int complete, DistributedID did, - ApEvent part_ready, ApUserEvent pend, - RtEvent init) - : IndexPartNode(ctx, p, par, cs, c, disjoint, complete, did, part_ready, - pend, init) + ApEvent partition_ready, ApBarrier pend, + RtEvent init, ShardMapping *map) + : IndexPartNode(ctx, p, par, cs, c, disjoint, complete, did, + partition_ready, pend, init, map) //-------------------------------------------------------------------------- { } @@ -5146,11 +5910,11 @@ namespace Legion { IndexPartition p, IndexSpaceNode *par, IndexSpaceNode *cs, LegionColor c, RtEvent disjoint_event, - int complete, DistributedID did, - ApEvent partition_ready, - ApUserEvent pending, RtEvent init) - : IndexPartNode(ctx, p, par, cs, c, disjoint_event, complete, did, - partition_ready, pending, init) + int comp, DistributedID did, + ApEvent partition_ready, ApBarrier pend, + RtEvent init, ShardMapping *map) + : IndexPartNode(ctx, p, par, cs, c, disjoint_event, comp, did, + partition_ready, pend, init, map) //-------------------------------------------------------------------------- { } diff --git a/runtime/legion/region_tree_tmpl.cc b/runtime/legion/region_tree_tmpl.cc index 0cb0573a78..a7424aba06 100644 --- a/runtime/legion/region_tree_tmpl.cc +++ b/runtime/legion/region_tree_tmpl.cc @@ -69,12 +69,12 @@ namespace Legion { create_by_domain_helper(Operation *, \ IndexPartNode *, \ FutureMapImpl *, \ - bool); \ + bool, ShardID, size_t); \ template ApEvent IndexSpaceNodeT:: \ create_by_weight_helper(Operation *, \ IndexPartNode *, \ FutureMapImpl *, \ - size_t); \ + size_t, ShardID, size_t); \ template ApEvent IndexSpaceNodeT:: \ create_by_field_helper(Operation *, \ IndexPartNode *, \ @@ -85,13 +85,13 @@ namespace Legion { IndexPartNode *, \ IndexPartNode *, \ const std::vector &, \ - ApEvent); \ + ApEvent, ShardID, size_t); \ template ApEvent IndexSpaceNodeT:: \ create_by_image_range_helper(Operation *, \ IndexPartNode *, \ IndexPartNode *, \ const std::vector &, \ - ApEvent); \ + ApEvent, ShardID, size_t); \ template ApEvent IndexSpaceNodeT:: \ create_by_preimage_helper(Operation *, \ IndexPartNode *, \ diff --git a/runtime/legion/runtime.cc b/runtime/legion/runtime.cc index fdf19b0ae4..a40740d951 100644 --- a/runtime/legion/runtime.cc +++ b/runtime/legion/runtime.cc @@ -25,6 +25,7 @@ #include "legion/legion_instances.h" #include "legion/legion_views.h" #include "legion/legion_context.h" +#include "legion/legion_replication.h" #include "legion/mapper_manager.h" #include "legion/garbage_collection.h" #include "mappers/default_mapper.h" @@ -73,6 +74,9 @@ namespace Legion { __thread UniqueID implicit_provenance = 0; __thread unsigned inside_registration_callback = NO_REGISTRATION_CALLBACK; __thread bool external_implicit_task = false; +#ifdef DEBUG_LEGION_WAITS + __thread int meta_task_id = -1; +#endif const LgEvent LgEvent::NO_LG_EVENT = LgEvent(); const ApEvent ApEvent::NO_AP_EVENT = ApEvent(); @@ -89,18 +93,31 @@ namespace Legion { //-------------------------------------------------------------------------- ArgumentMapImpl::ArgumentMapImpl(void) - : Collectable(), runtime(implicit_runtime), - dependent_futures(0), equivalent(false) + : Collectable(), runtime(implicit_runtime), + future_map(NULL), point_set(Domain::NO_DOMAIN), dimensionality(0), + dependent_futures(0), update_point_set(false), own_point_set(false), + equivalent(false) //-------------------------------------------------------------------------- { } //-------------------------------------------------------------------------- ArgumentMapImpl::ArgumentMapImpl(const FutureMap &rhs) - : Collectable(), runtime(implicit_runtime), - future_map(rhs), dependent_futures(0), equivalent(false) + : Collectable(), runtime(implicit_runtime), future_map(rhs.impl), + dependent_futures(0), update_point_set(false), own_point_set(false), + equivalent(false) //-------------------------------------------------------------------------- { + if (future_map.impl != NULL) + { + point_set = future_map.impl->get_domain(); + dimensionality = point_set.get_dim(); + } + else + { + point_set = Domain::NO_DOMAIN; + dimensionality = 0; + } } //-------------------------------------------------------------------------- @@ -116,6 +133,8 @@ namespace Legion { ArgumentMapImpl::~ArgumentMapImpl(void) //-------------------------------------------------------------------------- { + if (own_point_set) + free_point_set(); } //-------------------------------------------------------------------------- @@ -131,6 +150,18 @@ namespace Legion { bool ArgumentMapImpl::has_point(const DomainPoint &point) //-------------------------------------------------------------------------- { + if (dimensionality > 0) + { + const unsigned point_dim = point.get_dim(); + if (point_dim != dimensionality) + REPORT_LEGION_ERROR(ERROR_ARGUMENT_MAP_DIMENSIONALITY, + "Mismatch in dimensionality in 'has_point' on ArgumentMap " + "with %d dimensions and point with %d dimensions. ArgumentMaps " + "must always contain points of the same dimensionality.", + dimensionality, point_dim) + } + if (point_set.exists() && !update_point_set && point_set.contains(point)) + return true; if (future_map.impl != NULL) unfreeze(); return (arguments.find(point) != arguments.end()); @@ -142,6 +173,26 @@ namespace Legion { bool replace) //-------------------------------------------------------------------------- { + if (dimensionality > 0) + { + const unsigned point_dim = point.get_dim(); + if (point_dim != dimensionality) + REPORT_LEGION_ERROR(ERROR_ARGUMENT_MAP_DIMENSIONALITY, + "Mismatch in dimensionality in 'set_point' on ArgumentMap " + "with %d dimensions and point with %d dimensions. ArgumentMaps " + "must always contain points of the same dimensionality.", + dimensionality, point_dim) + } + else + { + dimensionality = point.get_dim(); +#ifdef DEBUG_LEGION + assert(dimensionality > 0); +#endif + } + if (!replace and point_set.exists() && !update_point_set && + point_set.contains(point)) + return; if (future_map.impl != NULL) unfreeze(); std::map::iterator finder = arguments.find(point); @@ -172,6 +223,8 @@ namespace Legion { arg.get_ptr(), arg.get_size()); else arguments[point] = Future(); + // Had to add a new point so the point set is no longer value + update_point_set = true; } // If we modified things then they are no longer equivalent if (future_map.impl != NULL) @@ -186,6 +239,26 @@ namespace Legion { const Future &f, bool replace) //-------------------------------------------------------------------------- { + if (dimensionality > 0) + { + const unsigned point_dim = point.get_dim(); + if (point_dim != dimensionality) + REPORT_LEGION_ERROR(ERROR_ARGUMENT_MAP_DIMENSIONALITY, + "Mismatch in dimensionality in 'set_point' on ArgumentMap " + "with %d dimensions and point with %d dimensions. ArgumentMaps " + "must always contain points of the same dimensionality.", + dimensionality, point_dim) + } + else + { + dimensionality = point.get_dim(); +#ifdef DEBUG_LEGION + assert(dimensionality > 0); +#endif + } + if (!replace and point_set.exists() && !update_point_set && + point_set.contains(point)) + return; if (future_map.impl != NULL) unfreeze(); std::map::iterator finder = arguments.find(point); @@ -205,7 +278,11 @@ namespace Legion { } else + { arguments[point] = f; + // Had to add a new point so the point set is no longer valid + update_point_set = true; + } if (f.impl->producer_op != NULL) dependent_futures++; // If we modified things then they are no longer equivalent @@ -220,6 +297,25 @@ namespace Legion { bool ArgumentMapImpl::remove_point(const DomainPoint &point) //-------------------------------------------------------------------------- { + if (dimensionality > 0) + { + const unsigned point_dim = point.get_dim(); + if (point_dim != dimensionality) + REPORT_LEGION_ERROR(ERROR_ARGUMENT_MAP_DIMENSIONALITY, + "Mismatch in dimensionality in 'remove_point' on ArgumentMap " + "with %d dimensions and point with %d dimensions. ArgumentMaps " + "must always contain points of the same dimensionality.", + dimensionality, point_dim) + } + else + { + dimensionality = point.get_dim(); +#ifdef DEBUG_LEGION + assert(dimensionality > 0); +#endif + } + if (point_set.exists() && !update_point_set && !point_set.contains(point)) + return false; if (future_map.impl != NULL) unfreeze(); std::map::iterator finder = arguments.find(point); @@ -239,6 +335,8 @@ namespace Legion { equivalent = false; future_map = FutureMap(); } + // We removed a point so the point set is no longer valid + update_point_set = true; return true; } return false; @@ -248,6 +346,18 @@ namespace Legion { TaskArgument ArgumentMapImpl::get_point(const DomainPoint &point) //-------------------------------------------------------------------------- { + if (dimensionality > 0) + { + const unsigned point_dim = point.get_dim(); + if (point_dim != dimensionality) + REPORT_LEGION_ERROR(ERROR_ARGUMENT_MAP_DIMENSIONALITY, + "Mismatch in dimensionality in 'get_point' on ArgumentMap " + "with %d dimensions and point with %d dimensions. ArgumentMaps " + "must always contain points of the same dimensionality.", + dimensionality, point_dim) + } + if (point_set.exists() && !update_point_set && !point_set.contains(point)) + return TaskArgument(); if (future_map.impl != NULL) unfreeze(); std::map::const_iterator finder=arguments.find(point); @@ -267,6 +377,55 @@ namespace Legion { // If we have no futures then we can return an empty map if (arguments.empty()) return FutureMap(); + // Compute the point set if needed + if (update_point_set) + { + // Free the existing point set if we're going to make a new one + if (own_point_set) + free_point_set(); + if (!arguments.empty()) + { + switch (dimensionality) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + std::vector > points(arguments.size());\ + unsigned index = 0; \ + for (std::map::const_iterator it = \ + arguments.begin(); it != arguments.end(); it++) \ + { \ + const Point point = it->first; \ + points[index++] = point; \ + } \ + const Realm::IndexSpace space(points); \ + const DomainT domaint(space); \ + point_set = domaint; \ + break; \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + // We only need to count as owning this if it is not dense + own_point_set = !point_set.dense(); + } + else + { + point_set = Domain::NO_DOMAIN; + own_point_set = false; + } + update_point_set = false; + } + RtUserEvent deletion_precondition; + // If we own the point set then we need to know when everyone is + // done using it so we can delete it + if (own_point_set) + { + deletion_precondition = Runtime::create_rt_user_event(); + point_set_deletion_preconditions.insert(deletion_precondition); + } // See if we have any dependent future points, if we do then we need // to launch an explicit creation operation to ensure we get the right // mapping dependences for this future map @@ -275,18 +434,14 @@ namespace Legion { // Otherwise we have to make a future map and set all the futures // We know that they are already completed DistributedID did = runtime->get_available_distributed_id(); - future_map = FutureMap(new FutureMapImpl(ctx, runtime, did, - runtime->address_space, RtEvent::NO_RT_EVENT)); + future_map = FutureMap(new FutureMapImpl(ctx, runtime, point_set, did, + runtime->address_space, RtEvent::NO_RT_EVENT, true/*reg now*/, + deletion_precondition)); future_map.impl->set_all_futures(arguments); } else - future_map = ctx->construct_future_map(Domain::NO_DOMAIN, - arguments, true/*internal*/); -#ifdef DEBUG_LEGION - for (std::map::const_iterator it = - arguments.begin(); it != arguments.end(); it++) - future_map.impl->add_valid_point(it->first); -#endif + future_map = ctx->construct_future_map(point_set, arguments, + deletion_precondition, true/*internal*/); equivalent = true; // mark that these are equivalent dependent_futures = 0; // reset this for the next unpack return future_map; @@ -304,6 +459,9 @@ namespace Legion { return; // Otherwise we need to make them equivalent future_map.impl->get_all_futures(arguments); + point_set = future_map.impl->get_domain(); + update_point_set = false; + own_point_set = false; // Count how many dependent futures we have #ifdef DEBUG_LEGION assert(dependent_futures == 0); @@ -315,6 +473,36 @@ namespace Legion { equivalent = true; } + //-------------------------------------------------------------------------- + void ArgumentMapImpl::free_point_set(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(own_point_set); + assert(point_set.exists()); +#endif + RtEvent precondition; + if (!point_set_deletion_preconditions.empty()) + { + precondition = Runtime::merge_events(point_set_deletion_preconditions); + point_set_deletion_preconditions.clear(); + } + switch (dimensionality) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + DomainT is = point_set; \ + is.destroy(precondition); \ + break; \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + } + ///////////////////////////////////////////////////////////// // Field Allocator Impl ///////////////////////////////////////////////////////////// @@ -463,6 +651,33 @@ namespace Legion { #endif } + //-------------------------------------------------------------------------- + FutureImpl::FutureImpl(Runtime *rt, bool register_now, DistributedID did, + AddressSpaceID own_space, ApEvent complete, + Operation *o, GenerationID gen, +#ifdef LEGION_SPY + UniqueID uid, +#endif + int depth) + : DistributedCollectable(rt, + LEGION_DISTRIBUTED_HELP_ENCODE(did, FUTURE_DC), + own_space, register_now), + producer_op(o), op_gen(gen), producer_depth(depth), +#ifdef LEGION_SPY + producer_uid(uid), +#endif + future_complete(complete), result(NULL), result_size(0), + result_set_space(local_space), empty(true), sampled(false) + //-------------------------------------------------------------------------- + { + if (producer_op != NULL) + producer_op->add_mapping_reference(op_gen); +#ifdef LEGION_GC + log_garbage.info("GC Future %lld %d", + LEGION_DISTRIBUTED_ID_FILTER(did), local_space); +#endif + } + //-------------------------------------------------------------------------- FutureImpl::FutureImpl(const FutureImpl &rhs) : DistributedCollectable(NULL, 0, 0), producer_op(NULL), op_gen(0), @@ -571,18 +786,38 @@ namespace Legion { if ((implicit_context != NULL) && !runtime->separate_runtime_instances) implicit_context->record_blocking_call(); } - const ApEvent ready_event = empty ? subscribe() : future_complete; - if (!ready_event.has_triggered()) + if (internal) { - TaskContext *context = implicit_context; - if (context != NULL) + const RtEvent ready_event = empty ? + subscribe_internal() : RtEvent::NO_RT_EVENT; + if (!ready_event.has_triggered()) { - context->begin_task_wait(false/*from runtime*/); - ready_event.wait(); - context->end_task_wait(); + TaskContext *context = implicit_context; + if (context != NULL) + { + context->begin_task_wait(false/*from runtime*/); + ready_event.wait(); + context->end_task_wait(); + } + else + ready_event.wait(); + } + } + else + { + const ApEvent ready_event = empty ? subscribe() : future_complete; + if (!ready_event.has_triggered()) + { + TaskContext *context = implicit_context; + if (context != NULL) + { + context->begin_task_wait(false/*from runtime*/); + ready_event.wait(); + context->end_task_wait(); + } + else + ready_event.wait(); } - else - ready_event.wait(); } if (check_size) { @@ -698,6 +933,13 @@ namespace Legion { broadcast_result(subscribers, future_complete, false/*need lock*/); subscribers.clear(); } + if (subscription_internal.exists()) + { + Runtime::trigger_event(subscription_internal); + if (!subscription_event.exists() && + remove_base_resource_ref(RUNTIME_REF)) + assert(false); // should always hold reference from caller + } if (subscription_event.exists()) { // Be very careful here, it might look like you can trigger the @@ -722,7 +964,7 @@ namespace Legion { AutoLock f_lock(future_lock); #ifdef DEBUG_LEGION assert(empty); - assert(subscription_event.exists()); + assert(subscription_event.exists() || subscription_internal.exists()); #endif derez.deserialize(result_size); if (result_size > 0) @@ -733,8 +975,13 @@ namespace Legion { empty = false; ApEvent complete; derez.deserialize(complete); - Runtime::trigger_event(NULL, subscription_event, complete); - subscription_event = ApUserEvent::NO_AP_USER_EVENT; + if (subscription_event.exists()) + { + Runtime::trigger_event(NULL, subscription_event, complete); + subscription_event = ApUserEvent::NO_AP_USER_EVENT; + } + if (subscription_internal.exists()) + Runtime::trigger_event(subscription_internal); if (is_owner()) { #ifdef DEBUG_LEGION @@ -769,11 +1016,15 @@ namespace Legion { { if (!empty) { - valid = future_complete.has_triggered(); + valid = !subscription_internal.exists() || + subscription_internal.has_triggered(); return *((const bool*)result); } - valid = false; - return false; + else + { + valid = false; + return false; + } } //-------------------------------------------------------------------------- @@ -789,22 +1040,28 @@ namespace Legion { if (!subscription_event.exists()) { subscription_event = Runtime::create_ap_user_event(NULL); - // Add a reference to prevent us from being collected - // until we get the result of the subscription - add_base_resource_ref(RUNTIME_REF); if (!is_owner()) { #ifdef DEBUG_LEGION assert(!future_complete.exists()); #endif future_complete = subscription_event; - // Send a request to the owner node to subscribe - Serializer rez; - rez.serialize(did); - runtime->send_future_subscription(owner_space, rez); } - else - record_subscription(local_space, false/*need lock*/); + if (!subscription_internal.exists()) + { + // Add a reference to prevent us from being collected + // until we get the result of the subscription + add_base_resource_ref(RUNTIME_REF); + if (!is_owner()) + { + // Send a request to the owner node to subscribe + Serializer rez; + rez.serialize(did); + runtime->send_future_subscription(owner_space, rez); + } + else + record_subscription(local_space, false/*need lock*/); + } } return subscription_event; } @@ -812,6 +1069,41 @@ namespace Legion { return future_complete; } + //-------------------------------------------------------------------------- + RtEvent FutureImpl::subscribe_internal(void) + //-------------------------------------------------------------------------- + { + if (!empty) + return RtEvent::NO_RT_EVENT; + AutoLock f_lock(future_lock); + // See if we lost the race + if (empty) + { + if (!subscription_internal.exists()) + { + subscription_internal = Runtime::create_rt_user_event(); + if (!subscription_event.exists()) + { + // Add a reference to prevent us from being collected + // until we get the result of the subscription + add_base_resource_ref(RUNTIME_REF); + if (!is_owner()) + { + // Send a request to the owner node to subscribe + Serializer rez; + rez.serialize(did); + runtime->send_future_subscription(owner_space, rez); + } + else + record_subscription(local_space, false/*need lock*/); + } + } + return subscription_internal; + } + else + return RtEvent::NO_RT_EVENT; + } + //-------------------------------------------------------------------------- void FutureImpl::notify_active(ReferenceMutator *mutator) //-------------------------------------------------------------------------- @@ -1153,7 +1445,9 @@ namespace Legion { //-------------------------------------------------------------------------- FutureMapImpl::FutureMapImpl(TaskContext *ctx, Operation *o, RtEvent ready, - Runtime *rt, DistributedID did, AddressSpaceID owner_space) + const Domain &domain, Runtime *rt, + DistributedID did, AddressSpaceID owner_space, + RtUserEvent deleted) : DistributedCollectable(rt, LEGION_DISTRIBUTED_HELP_ENCODE(did, FUTURE_MAP_DC), owner_space), context(ctx), op(o), op_gen(o->get_generation()), @@ -1161,9 +1455,12 @@ namespace Legion { #ifdef LEGION_SPY op_uid(o->get_unique_op_id()), #endif - ready_event(ready) + future_map_domain(domain), ready_event(ready), delete_event(deleted) //-------------------------------------------------------------------------- { +#ifdef DEBUG_LEGION + assert(future_map_domain.exists()); +#endif #ifdef LEGION_GC log_garbage.info("GC Future Map %lld %d", LEGION_DISTRIBUTED_ID_FILTER(did), local_space); @@ -1171,9 +1468,10 @@ namespace Legion { } //-------------------------------------------------------------------------- - FutureMapImpl::FutureMapImpl(TaskContext *ctx, Runtime *rt, + FutureMapImpl::FutureMapImpl(TaskContext *ctx, Runtime *rt, const Domain &d, DistributedID did, AddressSpaceID owner_space, - RtEvent ready, bool register_now) + RtEvent ready, bool register_now, + RtUserEvent deleted) : DistributedCollectable(rt, LEGION_DISTRIBUTED_HELP_ENCODE(did, FUTURE_MAP_DC), owner_space, register_now), @@ -1181,9 +1479,12 @@ namespace Legion { #ifdef LEGION_SPY op_uid(0), #endif - ready_event(ready) + future_map_domain(d), ready_event(ready), delete_event(deleted) //-------------------------------------------------------------------------- { +#ifdef DEBUG_LEGION + assert(future_map_domain.exists()); +#endif #ifdef LEGION_GC log_garbage.info("GC Future Map %lld %d", LEGION_DISTRIBUTED_ID_FILTER(did), local_space); @@ -1208,6 +1509,8 @@ namespace Legion { //-------------------------------------------------------------------------- { futures.clear(); + if (delete_event.exists()) + Runtime::trigger_event(delete_event); } //-------------------------------------------------------------------------- @@ -1254,9 +1557,20 @@ namespace Legion { } //-------------------------------------------------------------------------- - Future FutureMapImpl::get_future(const DomainPoint &point, RtEvent *wait_on) + Future FutureMapImpl::get_future(const DomainPoint &point, + bool internal, RtEvent *wait_on) //-------------------------------------------------------------------------- { +#ifndef DEBUG_LEGION + if (!internal) +#endif + { + if (!future_map_domain.contains(point)) + REPORT_LEGION_ERROR(ERROR_INVALID_FUTURE_MAP_POINT, + "Invalid request for a point not contained in the " + "domain of a future map in task %s (UID %lld).", + context->get_task_name(), context->get_unique_id()) + } if (!is_owner()) { // See if we already have it @@ -1276,6 +1590,7 @@ namespace Legion { rez.serialize(did); rez.serialize(point); rez.serialize(future_ready_event); + rez.serialize(internal); } runtime->send_future_map_request_future(owner_space, rez); if (wait_on != NULL) @@ -1295,25 +1610,6 @@ namespace Legion { } else { -#ifdef DEBUG_LEGION -#ifndef NDEBUG - // Check to make sure we are asking for something in the domain - if (valid_points.find(point) == valid_points.end()) - { - bool is_valid_point = false; - for (std::vector::const_iterator it = - valid_domains.begin(); it != valid_domains.end(); it++) - { - if (it->contains(point)) - { - is_valid_point = true; - break; - } - } - assert(is_valid_point); - } -#endif -#endif AutoLock fm_lock(future_map_lock); // Check to see if we already have a future for the point std::map::const_iterator finder = @@ -1332,26 +1628,11 @@ namespace Legion { } } - //-------------------------------------------------------------------------- - FutureImpl* FutureMapImpl::find_future(const DomainPoint &point) - //-------------------------------------------------------------------------- - { - AutoLock fm_lock(future_map_lock,1,false/*exclusive*/); - std::map::const_iterator finder = futures.find(point); - if (finder != futures.end()) - return finder->second.impl; - else - return NULL; - } - //-------------------------------------------------------------------------- void FutureMapImpl::set_future(const DomainPoint &point, FutureImpl *impl, ReferenceMutator *mutator) //-------------------------------------------------------------------------- { -#ifdef DEBUG_LEGION - assert(!is_owner()); // should never be called on the owner node -#endif // Add the reference first and then set the future impl->add_base_gc_ref(FUTURE_HANDLE_REF, mutator); AutoLock fm_lock(future_map_lock); @@ -1364,7 +1645,7 @@ namespace Legion { const char *warning_string) //-------------------------------------------------------------------------- { - Future f = get_future(point); + Future f = get_future(point, false/*internal*/); f.get_void_result(silence_warnings, warning_string); } @@ -1424,8 +1705,7 @@ namespace Legion { } //-------------------------------------------------------------------------- - void FutureMapImpl::get_all_futures( - std::map &others) const + void FutureMapImpl::get_all_futures(std::map &others) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -1460,23 +1740,33 @@ namespace Legion { futures = others; } -#ifdef DEBUG_LEGION //-------------------------------------------------------------------------- - void FutureMapImpl::add_valid_domain(const Domain &d) + FutureImpl* FutureMapImpl::find_shard_local_future(const DomainPoint &point) //-------------------------------------------------------------------------- { - assert(is_owner()); - valid_domains.push_back(d); + // Wait for all the futures to be ready + if (!ready_event.has_triggered()) + ready_event.wait(); + // No need for the lock since the map should be fixed now + std::map::const_iterator finder = futures.find(point); + if (finder != futures.end()) + return finder->second.impl; + else + return NULL; } //-------------------------------------------------------------------------- - void FutureMapImpl::add_valid_point(const DomainPoint &dp) + void FutureMapImpl::get_shard_local_futures( + std::map &others) //-------------------------------------------------------------------------- { - assert(is_owner()); - valid_points.insert(dp); + // Wait for all the futures to be ready + if (!ready_event.has_triggered()) + ready_event.wait(); + for (std::map::const_iterator it = + futures.begin(); it != futures.end(); it++) + others[it->first] = it->second.impl; } -#endif //-------------------------------------------------------------------------- void FutureMapImpl::register_dependence(Operation *consumer_op) @@ -1532,6 +1822,8 @@ namespace Legion { derez.deserialize(point); RtUserEvent done; derez.deserialize(done); + bool internal; + derez.deserialize(internal); // Should always find it since this is the owner node DistributedCollectable *dc = runtime->find_distributed_collectable(did); @@ -1541,7 +1833,7 @@ namespace Legion { #else FutureMapImpl *impl = static_cast(dc); #endif - Future f = impl->get_future(point); + Future f = impl->get_future(point, internal); Serializer rez; { RezCheck z2(rez); @@ -1589,29 +1881,37 @@ namespace Legion { } ///////////////////////////////////////////////////////////// - // Physical Region Impl + // Repl Future Map Impl ///////////////////////////////////////////////////////////// //-------------------------------------------------------------------------- - PhysicalRegionImpl::PhysicalRegionImpl(const RegionRequirement &r, - ApEvent mapped, bool m, TaskContext *ctx, - MapperID mid, MappingTagID t, - bool leaf, bool virt, Runtime *rt) - : Collectable(), runtime(rt), context(ctx), map_id(mid), tag(t), - leaf_region(leaf), virtual_mapped(virt), - replaying((ctx != NULL) ? ctx->owner_task->is_replaying() : false), - mapped_event(mapped), req(r), mapped(m), valid(false), - trigger_on_unmap(false), made_accessor(false) + ReplFutureMapImpl::ReplFutureMapImpl(ReplicateContext *ctx, Operation *op, + RtEvent ready, const Domain &domain, + const Domain &shard_dom, Runtime *rt, + DistributedID did,AddressSpaceID owner, + RtUserEvent deletion_trigger) + : FutureMapImpl(ctx, op, ready, domain, rt, did, owner, deletion_trigger), + repl_ctx(ctx), shard_domain(shard_dom), + future_map_barrier_index(ctx->peek_next_future_map_barrier_index()), + future_map_barrier(ctx->get_next_future_map_barrier()), + collective_index(ctx->get_next_collective_index(COLLECTIVE_LOC_32)), + op_depth(repl_ctx->get_depth()), op_uid(op->get_unique_op_id()), + op_ctx_index(op->get_ctx_index()), + sharding_function_ready(Runtime::create_rt_user_event()), + sharding_function(NULL), collective_performed(false), + has_non_trivial_call(false) //-------------------------------------------------------------------------- { + repl_ctx->add_reference(); + // Now register ourselves with the context + repl_ctx->register_future_map(this); } //-------------------------------------------------------------------------- - PhysicalRegionImpl::PhysicalRegionImpl(const PhysicalRegionImpl &rhs) - : Collectable(), runtime(NULL), context(NULL), map_id(0), tag(0), - leaf_region(false), virtual_mapped(false), replaying(false), - mapped_event(ApEvent::NO_AP_EVENT), mapped(false), valid(false), - trigger_on_unmap(false), made_accessor(false) + ReplFutureMapImpl::ReplFutureMapImpl(const ReplFutureMapImpl &rhs) + : FutureMapImpl(rhs), repl_ctx(NULL), shard_domain(Domain::NO_DOMAIN), + future_map_barrier_index(0), collective_index(0), op_depth(0), + op_uid(0), op_ctx_index(0) //-------------------------------------------------------------------------- { // should never be called @@ -1619,23 +1919,16 @@ namespace Legion { } //-------------------------------------------------------------------------- - PhysicalRegionImpl::~PhysicalRegionImpl(void) + ReplFutureMapImpl::~ReplFutureMapImpl(void) //-------------------------------------------------------------------------- { - // If we still have a trigger on unmap, do that before - // deleting ourselves to avoid leaking events - if (trigger_on_unmap) - { - trigger_on_unmap = false; - Runtime::trigger_event(NULL, termination_event); - } - if (!references.empty() && !replaying) - references.remove_resource_references(PHYSICAL_REGION_REF); + if (repl_ctx->remove_reference()) + delete repl_ctx; } //-------------------------------------------------------------------------- - PhysicalRegionImpl& PhysicalRegionImpl::operator=( - const PhysicalRegionImpl &rhs) + ReplFutureMapImpl& ReplFutureMapImpl::operator=( + const ReplFutureMapImpl &rhs) //-------------------------------------------------------------------------- { // should never be called @@ -1644,14 +1937,435 @@ namespace Legion { } //-------------------------------------------------------------------------- - void PhysicalRegionImpl::wait_until_valid(bool silence_warnings, - const char *warning_string, - bool warn, const char *source) + void ReplFutureMapImpl::notify_inactive(ReferenceMutator *mutator) //-------------------------------------------------------------------------- { - if (context != NULL) - context->record_blocking_call(); - if (runtime->runtime_warnings && !silence_warnings && +#ifdef DEBUG_LEGION + assert(is_owner()); +#endif + // Do the base version, then arrive on our barrier + FutureMapImpl::notify_inactive(mutator); + // Decide what to do here about our future map barrier depending + // on whether we saw any non-trivial calls on this shard. If we + // did not see any non-trivial calls then neither should any of + // the other shards and we don't have to use the barrier to guide + // reclamation of this future map + if (has_non_trivial_call) + { + if (!exchange_events.empty()) + Runtime::phase_barrier_arrive(future_map_barrier, 1/*count*/, + Runtime::merge_events(exchange_events)); + else + Runtime::phase_barrier_arrive(future_map_barrier, 1/*count*/); + if (!future_map_barrier.has_triggered()) + { + // Add a reference to this to prevent it being collected + add_base_resource_ref(DEFERRED_TASK_REF); + // Launch a task to do the reclaim once everyone is done + ReclaimFutureMapArgs args(repl_ctx, this, op_uid); + runtime->issue_runtime_meta_task(args, + LG_LATENCY_WORK_PRIORITY, future_map_barrier); + } + else + repl_ctx->unregister_future_map(this); + } + else + { + // No non-trivial call so we can unregister ourselves now + repl_ctx->unregister_future_map(this); + // If we're the owner shard of the barrier then do the arrival + // for all the shards so that the barrier generation triggers + // without needing to do any communication + const size_t total_shards = repl_ctx->total_shards; + if ((future_map_barrier_index % total_shards) == + repl_ctx->owner_shard->shard_id) + Runtime::phase_barrier_arrive(future_map_barrier, total_shards); + } + } + + //-------------------------------------------------------------------------- + Future ReplFutureMapImpl::get_future(const DomainPoint &point, + bool internal, RtEvent *wait_on) + //-------------------------------------------------------------------------- + { + if (!internal) + has_non_trivial_call = true; + // Do a quick check to see if we've already got it + { + AutoLock f_lock(future_map_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + futures.find(point); + if (finder != futures.end()) + return finder->second; + } + // Now we need to figure out which shard we're on, see if we know + // the sharding function yet, if not we have to wait + if (!sharding_function_ready.has_triggered()) + sharding_function_ready.wait(); + const ShardID owner_shard = + sharding_function->find_owner(point, shard_domain); + // If we're the owner shard we can just do the normal thing + if (owner_shard != repl_ctx->owner_shard->shard_id) + { + // We have to figure out the name of the future from the owner shard + RtUserEvent done_event = Runtime::create_rt_user_event(); + Serializer rez; + rez.serialize(repl_ctx->shard_manager->repl_id); + rez.serialize(owner_shard); + rez.serialize(future_map_barrier); + rez.serialize(point); + rez.serialize(did); + rez.serialize(done_event); + rez.serialize(internal); + repl_ctx->shard_manager->send_future_map_request(owner_shard, rez); + if (wait_on != NULL) + { + *wait_on = done_event; + return Future(); + } + // Wait for the event + done_event.wait(); + // Now we can wake up see if we found it + AutoLock f_lock(future_map_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + futures.find(point); +#ifdef DEBUG_LEGION + assert(finder != futures.end()); +#endif + return finder->second; + } + else // If we're the owner shard we can just do the normal thing + return FutureMapImpl::get_future(point, internal, wait_on); + } + + //-------------------------------------------------------------------------- + void ReplFutureMapImpl::get_all_futures( + std::map &others) + //-------------------------------------------------------------------------- + { + has_non_trivial_call = true; + // We know this call only comes from the application so we don't + // need to worry about thread safety + if (collective_performed) + { + // No need for the lock, we know we have all the futures + others = futures; + return; + } + // Wait for all the local futures to be completed + if (!ready_event.has_triggered()) + ready_event.wait(); + // Now we've got all our local futures so we can do the exchange + // Have to hold the lock when doing this as there might be + // other requests for the future map + WrapperReferenceMutator mutator(exchange_events); + AutoLock f_lock(future_map_lock); + if (!collective_performed) + { + FutureNameExchange collective(repl_ctx, collective_index,this,&mutator); + collective.exchange_future_names(futures); + // When the collective is done we can mark that we've done it + // and then copy the results + collective_performed = true; + } + others = futures; + } + + //-------------------------------------------------------------------------- + void ReplFutureMapImpl::wait_all_results(bool silence_warnings, + const char *warning_string) + //-------------------------------------------------------------------------- + { + if (runtime->runtime_warnings && !silence_warnings && + (context != NULL) && !context->is_leaf_context()) + REPORT_LEGION_WARNING(LEGION_WARNING_WAITING_ALL_FUTURES, + "Waiting for all futures in a future map in " + "non-leaf task %s (UID %lld) is a violation of Legion's deferred " + "execution model best practices. You may notice a severe " + "performance degredation. Warning string: %s", + context->get_task_name(), context->get_unique_id(), + (warning_string == NULL) ? "" : warning_string) + // As a proxy for this, we will get the names of all the futures + // needed for this future map in case we need them in the future + // The process of doing this will wait on both our ready event + // as well as on the ready events of all other shards + std::map dummy_others; + get_all_futures(dummy_others); + } + + //-------------------------------------------------------------------------- + FutureImpl* ReplFutureMapImpl::find_shard_local_future( + const DomainPoint &point) + //-------------------------------------------------------------------------- + { + if (!sharding_function_ready.has_triggered()) + sharding_function_ready.wait(); + // Check to see if we own this point or not + const ShardID shard = sharding_function->find_owner(point, shard_domain); + if (shard != repl_ctx->owner_shard->shard_id) + return NULL; + return FutureMapImpl::find_shard_local_future(point); + } + + //-------------------------------------------------------------------------- + void ReplFutureMapImpl::get_shard_local_futures( + std::map &others) + //-------------------------------------------------------------------------- + { + FutureMapImpl::get_shard_local_futures(others); + const ShardID local_shard = repl_ctx->owner_shard->shard_id; + if (!sharding_function_ready.has_triggered()) + sharding_function_ready.wait(); + for (std::map::iterator it = + others.begin(); it != others.end(); /*nothing*/) + { + const ShardID shard = + sharding_function->find_owner(it->first, shard_domain); + if (shard != local_shard) + { + std::map::iterator to_delete = it++; + others.erase(to_delete); + } + else + it++; + } + } + + //-------------------------------------------------------------------------- + void ReplFutureMapImpl::set_sharding_function(ShardingFunction *function) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(sharding_function == NULL); +#endif + std::vector to_perform; + { + AutoLock fm_lock(future_map_lock); + sharding_function = function; + if (!pending_future_map_requests.empty()) + to_perform.swap(pending_future_map_requests); + } + Runtime::trigger_event(sharding_function_ready); + if (!to_perform.empty()) + { + for (std::vector::const_iterator it = + to_perform.begin(); it != to_perform.end(); it++) + process_future_map_request(it->point, it->src_did, + it->internal, it->done_event); + } + } + + //-------------------------------------------------------------------------- + void ReplFutureMapImpl::handle_future_map_request(Deserializer &derez) + //-------------------------------------------------------------------------- + { + DomainPoint point; + derez.deserialize(point); + DistributedID src_did; + derez.deserialize(src_did); + RtUserEvent done_event; + derez.deserialize(done_event); + bool internal; + derez.deserialize(internal); + // We can't actually process this until we get our sharding function + if (sharding_function == NULL) + { + // Take the lock and see if we lost the race + AutoLock fm_lock(future_map_lock); + if (sharding_function == NULL) + { + pending_future_map_requests.push_back( + PendingRequest(point, src_did, done_event, internal)); + return; + } + // If we have a sharding function now we can fall through and continue + } + process_future_map_request(point, src_did, internal, done_event); + } + + //-------------------------------------------------------------------------- + void ReplFutureMapImpl::process_future_map_request(const DomainPoint &point, + DistributedID src_did, + const bool internal, + RtUserEvent done_event) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(sharding_function != NULL); +#endif + const AddressSpaceID source = runtime->determine_owner(src_did); + Future result = get_future(point, internal); + if (source != runtime->address_space) + { + // Remote future map so send the answer back + Serializer rez; + { + RezCheck z(rez); + rez.serialize(src_did); + rez.serialize(point); + if (result.impl != NULL) + rez.serialize(result.impl->did); + else + rez.serialize(0); + rez.serialize(done_event); + } + runtime->send_control_replicate_future_map_response(source, rez); + } + else + { + // Local future map so we should be able to find it and set it + DistributedCollectable *dc = + runtime->find_distributed_collectable(src_did); +#ifdef DEBUG_LEGION + ReplFutureMapImpl *target = dynamic_cast(dc); + assert(target != NULL); +#else + ReplFutureMapImpl *target = static_cast(dc); +#endif + std::set preconditions; + WrapperReferenceMutator mutator(preconditions); + target->set_future(point, result.impl, &mutator); + if (!preconditions.empty()) + Runtime::trigger_event(done_event, + Runtime::merge_events(preconditions)); + else + Runtime::trigger_event(done_event); + } + } + + //-------------------------------------------------------------------------- + /*static*/ void ReplFutureMapImpl::handle_future_map_response( + Deserializer &derez, Runtime *runtime) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + DistributedID map_did; + derez.deserialize(map_did); + DomainPoint point; + derez.deserialize(point); + DistributedID future_did; + derez.deserialize(future_did); + RtUserEvent done_event; + derez.deserialize(done_event); + + // It should already exist so we're just finding it + DistributedCollectable *dc = + runtime->find_distributed_collectable(map_did); +#ifdef DEBUG_LEGION + ReplFutureMapImpl *target = dynamic_cast(dc); + assert(target != NULL); +#else + ReplFutureMapImpl *target = static_cast(dc); +#endif + std::set done_events; + WrapperReferenceMutator mutator(done_events); + if (future_did > 0) + { + FutureImpl *impl = runtime->find_or_create_future(future_did, &mutator, + target->op, target->op_gen, +#ifdef LEGION_SPY + target->op_uid, +#endif + target->op_depth); + target->set_future(point, impl, &mutator); + } + else + target->set_future(point, NULL, &mutator); + if (!done_events.empty()) + Runtime::trigger_event(done_event, Runtime::merge_events(done_events)); + else + Runtime::trigger_event(done_event); + } + + //-------------------------------------------------------------------------- + /*static*/void ReplFutureMapImpl::handle_future_map_reclaim(const void *arg) + //-------------------------------------------------------------------------- + { + const ReclaimFutureMapArgs *recl_args = (const ReclaimFutureMapArgs*)arg; + recl_args->ctx->unregister_future_map(recl_args->impl); + if (recl_args->impl->remove_base_resource_ref(DEFERRED_TASK_REF)) + delete recl_args->impl; + } + + ///////////////////////////////////////////////////////////// + // Physical Region Impl + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + PhysicalRegionImpl::PhysicalRegionImpl(const RegionRequirement &r, + ApEvent mapped, bool m, TaskContext *ctx, + MapperID mid, MappingTagID t, + bool leaf, bool virt, Runtime *rt) + : Collectable(), runtime(rt), context(ctx), map_id(mid), tag(t), + leaf_region(leaf), virtual_mapped(virt), + replaying((ctx != NULL) ? ctx->owner_task->is_replaying() : false), + mapped_event(mapped), req(r), sharded_view(NULL), mapped(m), + valid(false), trigger_on_unmap(false), made_accessor(false) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + PhysicalRegionImpl::PhysicalRegionImpl(const PhysicalRegionImpl &rhs) + : Collectable(), runtime(NULL), context(NULL), map_id(0), tag(0), + leaf_region(false), virtual_mapped(false), replaying(false), + mapped_event(ApEvent::NO_AP_EVENT), mapped(false), valid(false), + trigger_on_unmap(false), made_accessor(false) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + PhysicalRegionImpl::~PhysicalRegionImpl(void) + //-------------------------------------------------------------------------- + { + // If we still have a trigger on unmap, do that before + // deleting ourselves to avoid leaking events + if (trigger_on_unmap) + { + trigger_on_unmap = false; + Runtime::trigger_event(NULL, termination_event); + } + if (!references.empty() && !replaying) + references.remove_resource_references(PHYSICAL_REGION_REF); + if ((sharded_view != NULL) && + sharded_view->remove_base_resource_ref(PHYSICAL_REGION_REF)) + delete sharded_view; + } + + //-------------------------------------------------------------------------- + PhysicalRegionImpl& PhysicalRegionImpl::operator=( + const PhysicalRegionImpl &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + void PhysicalRegionImpl::set_sharded_view(ShardedView *view) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(sharded_view == NULL); + assert(view != NULL); +#endif + sharded_view = view; + sharded_view->add_base_resource_ref(PHYSICAL_REGION_REF); + } + + //-------------------------------------------------------------------------- + void PhysicalRegionImpl::wait_until_valid(bool silence_warnings, + const char *warning_string, + bool warn, const char *source) + //-------------------------------------------------------------------------- + { + if (context != NULL) + context->record_blocking_call(); + if (runtime->runtime_warnings && !silence_warnings && (context != NULL) && !context->is_leaf_context()) { if (source != NULL) @@ -1723,7 +2437,7 @@ namespace Legion { references.update_wait_on_events(wait_on); if (wait_on.empty()) return true; - ApEvent ref_ready = Runtime::merge_events(NULL, wait_on); + const ApEvent ref_ready = Runtime::merge_events(NULL, wait_on); return ref_ready.has_triggered(); } return false; @@ -2766,21 +3480,27 @@ namespace Legion { //-------------------------------------------------------------------------- MPIRankTable::MPIRankTable(Runtime *rt) - : runtime(rt), participating(int(runtime->address_space) < - runtime->legion_collective_participating_spaces), done_triggered(false) + : runtime(rt), collective_radix(rt->legion_collective_radix), + done_triggered(false) //-------------------------------------------------------------------------- { if (runtime->total_address_spaces > 1) { + configure_collective_settings(runtime->total_address_spaces, + runtime->address_space, collective_radix, collective_log_radix, + collective_stages, collective_participating_spaces, + collective_last_radix); + participating = + (int(runtime->address_space) < collective_participating_spaces); // We already have our contributions for each stage so // we can set the inditial participants to 1 if (participating) { - sent_stages.resize(runtime->legion_collective_stages, false); + sent_stages.resize(collective_stages, false); #ifdef DEBUG_LEGION - assert(runtime->legion_collective_stages > 0); + assert(collective_stages > 0); #endif - stage_notifications.resize(runtime->legion_collective_stages, 1); + stage_notifications.resize(collective_stages, 1); // Stage 0 always starts with 0 notifications since we'll // explictcly arrive on it stage_notifications[0] = 0; @@ -2832,9 +3552,9 @@ namespace Legion { // See if we are waiting for an initial notification // if not we can just send our message now if ((int(runtime->total_address_spaces) == - runtime->legion_collective_participating_spaces) || + collective_participating_spaces) || (runtime->address_space >= (runtime->total_address_spaces - - runtime->legion_collective_participating_spaces))) + collective_participating_spaces))) { const bool all_stages_done = initiate_exchange(); if (all_stages_done) @@ -2872,11 +3592,10 @@ namespace Legion { assert(!sent_stages.empty()); assert(!sent_stages[0]); // stage 0 shouldn't be sent yet assert(!stage_notifications.empty()); - if (runtime->legion_collective_stages == 1) - assert(stage_notifications[0] < - runtime->legion_collective_last_radix); + if (collective_stages == 1) + assert(stage_notifications[0] < collective_last_radix); else - assert(stage_notifications[0] < runtime->legion_collective_radix); + assert(stage_notifications[0] < collective_radix); #endif stage_notifications[0]++; } @@ -2904,7 +3623,7 @@ namespace Legion { { // Send back to the nodes that are not participating AddressSpaceID target = runtime->address_space + - runtime->legion_collective_participating_spaces; + collective_participating_spaces; #ifdef DEBUG_LEGION assert(target < runtime->total_address_spaces); #endif @@ -2914,7 +3633,7 @@ namespace Legion { { // Sent to a node that is participating AddressSpaceID target = runtime->address_space % - runtime->legion_collective_participating_spaces; + collective_participating_spaces; runtime->send_mpi_rank_exchange(target, rez); } } @@ -2928,8 +3647,7 @@ namespace Legion { #endif // Iterate through the stages and send any that are ready // Remember that stages have to be done in order - for (int stage = start_stage; - stage < runtime->legion_collective_stages; stage++) + for (int stage = start_stage; stage < collective_stages; stage++) { Serializer rez; { @@ -2942,8 +3660,7 @@ namespace Legion { // Check to see if we're sending this stage // We need all the notifications from the previous stage before // we can send this stage - if ((stage > 0) && - (stage_notifications[stage-1] < runtime->legion_collective_radix)) + if ((stage > 0) && (stage_notifications[stage-1] < collective_radix)) return false; // If we get here then we can send the stage sent_stages[stage] = true; @@ -2951,7 +3668,7 @@ namespace Legion { { size_t expected_size = 1; for (int idx = 0; idx < stage; idx++) - expected_size *= runtime->legion_collective_radix; + expected_size *= collective_radix; assert(expected_size <= forward_mapping.size()); } #endif @@ -2964,28 +3681,26 @@ namespace Legion { } } // Now we can do the send - if (stage == (runtime->legion_collective_stages-1)) + if (stage == (collective_stages-1)) { - for (int r = 1; r < runtime->legion_collective_last_radix; r++) + for (int r = 1; r < collective_last_radix; r++) { AddressSpaceID target = runtime->address_space ^ - (r << (stage * runtime->legion_collective_log_radix)); + (r << (stage * collective_log_radix)); #ifdef DEBUG_LEGION - assert(int(target) < - runtime->legion_collective_participating_spaces); + assert(int(target) < collective_participating_spaces); #endif runtime->send_mpi_rank_exchange(target, rez); } } else { - for (int r = 1; r < runtime->legion_collective_radix; r++) + for (int r = 1; r < collective_radix; r++) { AddressSpaceID target = runtime->address_space ^ - (r << (stage * runtime->legion_collective_log_radix)); + (r << (stage * collective_log_radix)); #ifdef DEBUG_LEGION - assert(int(target) < - runtime->legion_collective_participating_spaces); + assert(int(target) < collective_participating_spaces); #endif runtime->send_mpi_rank_exchange(target, rez); } @@ -2994,7 +3709,7 @@ namespace Legion { // If we make it here, then we sent the last stage, check to see // if we've seen all the notifications for it AutoLock r_lock(reservation); - if ((stage_notifications.back() == runtime->legion_collective_last_radix) + if ((stage_notifications.back() == collective_last_radix) && !done_triggered) { done_triggered = true; @@ -3054,12 +3769,10 @@ namespace Legion { { #ifdef DEBUG_LEGION assert(stage < int(stage_notifications.size())); - if (stage < (runtime->legion_collective_stages-1)) - assert(stage_notifications[stage] < - runtime->legion_collective_radix); + if (stage < (collective_stages-1)) + assert(stage_notifications[stage] < collective_radix); else - assert(stage_notifications[stage] < - runtime->legion_collective_last_radix); + assert(stage_notifications[stage] < collective_last_radix); #endif stage_notifications[stage]++; } @@ -3075,34 +3788,318 @@ namespace Legion { // See if we have to send a message back to a // non-participating node if ((int(runtime->total_address_spaces) > - runtime->legion_collective_participating_spaces) && + collective_participating_spaces) && (int(runtime->address_space) < int(runtime->total_address_spaces - - runtime->legion_collective_participating_spaces))) + collective_participating_spaces))) send_remainder_stage(); // We are done Runtime::trigger_event(done_event); } ///////////////////////////////////////////////////////////// - // Processor Manager + // Implicit Shard Manager ///////////////////////////////////////////////////////////// //-------------------------------------------------------------------------- - ProcessorManager::ProcessorManager(Processor proc, Processor::Kind kind, - Runtime *rt, unsigned def_mappers, - bool no_steal, bool replay) - : runtime(rt), local_proc(proc), proc_kind(kind), - stealing_disabled(no_steal), replay_execution(replay), - next_local_index(0), task_scheduler_enabled(false), - outstanding_task_scheduler(false), - total_active_contexts(0), total_active_mappers(0) + ImplicitShardManager::ImplicitShardManager(Runtime *rt, TaskID tid, + MapperID mid, Processor::Kind k, unsigned shards_per_space) + : Collectable(), runtime(rt), task_id(tid), mapper_id(mid), kind(k), + shards_per_address_space(shards_per_space), + expected_local_arrivals(shards_per_space), expected_remote_arrivals(0), + local_shard_id(0), top_context(NULL), shard_manager(NULL) //-------------------------------------------------------------------------- { - context_states.resize(LEGION_DEFAULT_CONTEXTS); - // Find our set of visible memories - Machine::MemoryQuery vis_mems(runtime->machine); - vis_mems.has_affinity_to(proc); - for (Machine::MemoryQuery::iterator it = vis_mems.begin(); + // If we're the owner node, we also expect one arrival from + // every remote node as well + if (runtime->address_space == 0) + expected_remote_arrivals = (runtime->total_address_spaces - 1); + } + + //-------------------------------------------------------------------------- + ImplicitShardManager::ImplicitShardManager(const ImplicitShardManager &rhs) + : Collectable(), runtime(rhs.runtime), task_id(rhs.task_id), + mapper_id(rhs.mapper_id), kind(rhs.kind), + shards_per_address_space(rhs.shards_per_address_space) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ImplicitShardManager::~ImplicitShardManager(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ImplicitShardManager& ImplicitShardManager::operator=( + const ImplicitShardManager &rhs) + //-------------------------------------------------------------------------- + { + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + bool ImplicitShardManager::record_arrival(bool local) + //-------------------------------------------------------------------------- + { + // No need for the lock here, we're always protected by the shard_lock + // when this is called + if (local) + { +#ifdef DEBUG_LEGION + assert(expected_local_arrivals > 0); +#endif + return ((--expected_local_arrivals == 0) && + (expected_remote_arrivals == 0)); + } + else + { +#ifdef DEBUG_LEGION + assert(expected_remote_arrivals > 0); +#endif + return ((--expected_remote_arrivals == 0) && + (expected_local_arrivals == 0)); + } + } + + //-------------------------------------------------------------------------- + ShardTask* ImplicitShardManager::create_shard(int shard_id, Processor proxy, + const char *task_name) + //-------------------------------------------------------------------------- + { + ShardTask *result = NULL; + if (runtime->address_space == 0) + { + AutoLock m_lock(manager_lock); + if (shard_manager == NULL) + create_shard_manager(proxy, task_name); +#ifdef DEBUG_LEGION + assert(local_shard_id < shards_per_address_space); +#endif + const ShardID shard = (shard_id < 0) ? local_shard_id++ : shard_id; + result = shard_manager->create_shard(shard, proxy); + } + else + { + RtEvent wait_on; + if (shard_manager == NULL) + { + AutoLock m_lock(manager_lock); + if (shard_manager == NULL) + { + if (!manager_ready.exists()) + request_shard_manager(); + wait_on = manager_ready; + } + } + if (wait_on.exists()) + wait_on.wait(); + AutoLock m_lock(manager_lock); +#ifdef DEBUG_LEGION + assert(local_shard_id < shards_per_address_space); +#endif + const ShardID shard = (shard_id < 0) ? (runtime->address_space * + shards_per_address_space + local_shard_id++) : shard_id; + result = shard_manager->create_shard(shard, proxy); + } +#ifdef DEBUG_LEGION + assert(top_context != NULL); +#endif + top_context->increment_pending(); + result->initialize_implicit_task(top_context, task_id, mapper_id, proxy); + result->complete_mapping(); + result->resolve_speculation(); + return result; + } + + //-------------------------------------------------------------------------- + void ImplicitShardManager::create_shard_manager(Processor proxy, + const char *task_name) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(top_context == NULL); + assert(shard_manager == NULL); +#endif + IndividualTask *implicit_top = + runtime->create_implicit_top_level(task_id, mapper_id, proxy, task_name); + top_context = implicit_top->get_context(); + // Now we need to make the shard manager + const ReplicationID repl_context = runtime->get_unique_replication_id(); + const size_t total_shards = + runtime->total_address_spaces * shards_per_address_space; + // We also need a shard + shard_manager = new ShardManager(runtime, repl_context, true/*cr*/, + true/*top level*/, total_shards, runtime->address_space, implicit_top); + implicit_top->set_shard_manager(shard_manager); + // This is a dummy shard_mapping for now since we won't actually need + // a real one, this just needs to make sure all the checks pass + std::vector shard_mapping(total_shards, Processor::NO_PROC); + shard_manager->set_shard_mapping(shard_mapping); + std::vector address_spaces(total_shards); + for (AddressSpaceID space = 0; + space < runtime->total_address_spaces; space++) + { + for (unsigned idx = 0; idx < shards_per_address_space; idx++) + address_spaces[space * shards_per_address_space + idx] = space; + } + shard_manager->set_address_spaces(address_spaces); + // We also need to make the callback barrier here, but its easy here + // because we know that this has to contain all address spaces + shard_manager->create_callback_barrier(runtime->total_address_spaces); + if (runtime->legion_spy_enabled) + LegionSpy::log_replication(implicit_top->get_unique_id(), repl_context, + true/*control replication*/); + // Distribute the shard manager to all the remove nodes + std::vector empty_shards; + for (AddressSpaceID space = 1; + space < runtime->total_address_spaces; space++) + shard_manager->distribute_shards(space, empty_shards); + // Then send any pending responses + if (!remote_spaces.empty()) + { + for (std::vector >::const_iterator it = + remote_spaces.begin(); it != remote_spaces.end(); it++) + { + Serializer rez; + { + RezCheck z(rez); + rez.serialize(it->second); + rez.serialize(top_context->get_context_uid()); + rez.serialize(repl_context); + } + runtime->send_control_replicate_implicit_response(it->first, rez); + } + } + } + + //-------------------------------------------------------------------------- + void ImplicitShardManager::request_shard_manager(void) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(shard_manager == NULL); + assert(!manager_ready.exists()); +#endif + manager_ready = Runtime::create_rt_user_event(); + Serializer rez; + { + RezCheck z(rez); + rez.serialize(task_id); + rez.serialize(mapper_id); + rez.serialize(kind); + rez.serialize(shards_per_address_space); + rez.serialize(this); + } + runtime->send_control_replicate_implicit_request(0/*owner*/, rez); + } + + //-------------------------------------------------------------------------- + void ImplicitShardManager::process_implicit_request(void *remote, + AddressSpaceID space) + //-------------------------------------------------------------------------- + { + AutoLock m_lock(manager_lock); + if (shard_manager != NULL) + { + Serializer rez; + { + RezCheck z(rez); + rez.serialize(remote); + rez.serialize(top_context->get_context_uid()); + rez.serialize(shard_manager->repl_id); + } + runtime->send_control_replicate_implicit_response(space, rez); + } + else + remote_spaces.push_back(std::pair(space, remote)); + } + + //-------------------------------------------------------------------------- + RtUserEvent ImplicitShardManager::process_implicit_response(ShardManager *m, + InnerContext *c) + //-------------------------------------------------------------------------- + { + AutoLock m_lock(manager_lock); +#ifdef DEBUG_LEGION + assert(top_context == NULL); + assert(shard_manager == NULL); + assert(manager_ready.exists()); +#endif + top_context = c; + shard_manager = m; + RtUserEvent to_trigger = manager_ready; + manager_ready = RtUserEvent::NO_RT_USER_EVENT; + return to_trigger; + } + + //-------------------------------------------------------------------------- + /*static*/ void ImplicitShardManager::handle_remote_request( + Deserializer &derez, Runtime *runtime, AddressSpaceID remote_space) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + TaskID task_id; + derez.deserialize(task_id); + MapperID mapper_id; + derez.deserialize(mapper_id); + Processor::Kind kind; + derez.deserialize(kind); + unsigned shards_per_address_space; + derez.deserialize(shards_per_address_space); + void *remote; + derez.deserialize(remote); + ImplicitShardManager *manager = runtime->find_implicit_shard_manager( + task_id, mapper_id, kind, shards_per_address_space, false/*local*/); + manager->process_implicit_request(remote, remote_space); + if (manager->remove_reference()) + delete manager; + } + + //-------------------------------------------------------------------------- + /*static*/ void ImplicitShardManager::handle_remote_response( + Deserializer &derez, Runtime *runtime) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + ImplicitShardManager *manager; + derez.deserialize(manager); + UniqueID context_uid; + derez.deserialize(context_uid); + ReplicationID repl_id; + derez.deserialize(repl_id); + ShardManager *shard_manager = runtime->find_shard_manager(repl_id); + RtEvent context_ready; + InnerContext *context = + runtime->find_context(context_uid, false, &context_ready); + RtUserEvent to_trigger = + manager->process_implicit_response(shard_manager, context); + Runtime::trigger_event(to_trigger, context_ready); + } + + ///////////////////////////////////////////////////////////// + // Processor Manager + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ProcessorManager::ProcessorManager(Processor proc, Processor::Kind kind, + Runtime *rt, unsigned def_mappers, + bool no_steal, bool replay) + : runtime(rt), local_proc(proc), proc_kind(kind), + stealing_disabled(no_steal), replay_execution(replay), + next_local_index(0), task_scheduler_enabled(false), + outstanding_task_scheduler(false), + total_active_contexts(0), total_active_mappers(0) + //-------------------------------------------------------------------------- + { + context_states.resize(LEGION_DEFAULT_CONTEXTS); + // Find our set of visible memories + Machine::MemoryQuery vis_mems(runtime->machine); + vis_mems.has_affinity_to(proc); + for (Machine::MemoryQuery::iterator it = vis_mems.begin(); it != vis_mems.end(); it++) visible_memories.insert(*it); } @@ -8102,6 +9099,11 @@ namespace Legion { runtime->handle_send_phi_view(derez, remote_address_space); break; } + case SEND_SHARDED_VIEW: + { + runtime->handle_send_sharded_view(derez, remote_address_space); + break; + } case SEND_REDUCTION_VIEW: { runtime->handle_send_reduction_view(derez, remote_address_space); @@ -8229,6 +9231,75 @@ namespace Legion { runtime->handle_future_map_future_response(derez); break; } + case SEND_REPL_FUTURE_MAP_REQUEST: + { + runtime->handle_control_replicate_future_map_request(derez); + break; + } + case SEND_REPL_FUTURE_MAP_RESPONSE: + { + runtime->handle_control_replicate_future_map_response(derez); + break; + } + case SEND_REPL_TOP_VIEW_REQUEST: + { + runtime->handle_control_replicate_top_view_request(derez, + remote_address_space); + break; + } + case SEND_REPL_TOP_VIEW_RESPONSE: + { + runtime->handle_control_replicate_top_view_response(derez); + break; + } + case SEND_REPL_EQ_REQUEST: + { + runtime->handle_control_replicate_eq_request(derez); + break; + } + case SEND_REPL_EQ_RESPONSE: + { + runtime->handle_control_replicate_eq_response(derez); + break; + } + case SEND_REPL_INTRA_SPACE_DEP: + { + runtime->handle_control_replicate_intra_space_dependence(derez); + break; + } + case SEND_REPL_RESOURCE_UPDATE: + { + runtime->handle_control_replicate_resource_update(derez); + break; + } + case SEND_REPL_TRACE_EVENT_REQUEST: + { + runtime->handle_control_replicate_trace_event_request(derez, + remote_address_space); + break; + } + case SEND_REPL_TRACE_EVENT_RESPONSE: + { + runtime->handle_control_replicate_trace_event_response(derez); + break; + } + case SEND_REPL_TRACE_UPDATE: + { + runtime->handle_control_replicate_trace_update(derez, + remote_address_space); + break; + } + case SEND_REPL_IMPLICIT_REQUEST: + { + runtime->handle_control_replicate_implicit_request(derez, + remote_address_space); + break; + } + case SEND_REPL_IMPLICIT_RESPONSE: + { + runtime->handle_control_replicate_implicit_response(derez); + break; + } case SEND_MAPPER_MESSAGE: { runtime->handle_mapper_message(derez); @@ -8558,6 +9629,41 @@ namespace Legion { runtime->handle_mpi_rank_exchange(derez); break; } + case SEND_REPLICATE_LAUNCH: + { + runtime->handle_replicate_launch(derez, remote_address_space); + break; + } + case SEND_REPLICATE_DELETE: + { + runtime->handle_replicate_delete(derez); + break; + } + case SEND_REPLICATE_POST_MAPPED: + { + runtime->handle_replicate_post_mapped(derez); + break; + } + case SEND_REPLICATE_POST_EXECUTION: + { + runtime->handle_replicate_post_execution(derez); + break; + } + case SEND_REPLICATE_TRIGGER_COMPLETE: + { + runtime->handle_replicate_trigger_complete(derez); + break; + } + case SEND_REPLICATE_TRIGGER_COMMIT: + { + runtime->handle_replicate_trigger_commit(derez); + break; + } + case SEND_CONTROL_REPLICATE_COLLECTIVE_MESSAGE: + { + runtime->handle_control_replicate_collective_message(derez); + break; + } case SEND_LIBRARY_MAPPER_REQUEST: { runtime->handle_library_mapper_request(derez, @@ -8590,6 +9696,17 @@ namespace Legion { runtime->handle_library_projection_response(derez); break; } + case SEND_LIBRARY_SHARDING_REQUEST: + { + runtime->handle_library_sharding_request(derez, + remote_address_space); + break; + } + case SEND_LIBRARY_SHARDING_RESPONSE: + { + runtime->handle_library_sharding_response(derez); + break; + } case SEND_LIBRARY_TASK_REQUEST: { runtime->handle_library_task_request(derez, remote_address_space); @@ -8980,20 +10097,6 @@ namespace Legion { } } - //-------------------------------------------------------------------------- - void Runtime::handle_remote_op_report_uninitialized(Deserializer &derez) - //-------------------------------------------------------------------------- - { - RemoteOp::handle_report_uninitialized(derez); - } - - //-------------------------------------------------------------------------- - void Runtime::handle_remote_op_profiling_count_update(Deserializer &derez) - //-------------------------------------------------------------------------- - { - RemoteOp::handle_report_profiling_count_update(derez); - } - //-------------------------------------------------------------------------- void Runtime::handle_remote_tracing_update(Deserializer &derez, AddressSpaceID source) @@ -9707,7 +10810,7 @@ namespace Legion { Runtime *runtime) //-------------------------------------------------------------------------- { - return (task_id % runtime->runtime_stride); + return (task_id % runtime->total_address_spaces); } ///////////////////////////////////////////////////////////// @@ -9727,7 +10830,8 @@ namespace Legion { layout_constraints(registrar.layout_constraints), user_data_size(udata_size), leaf_variant(registrar.leaf_variant), inner_variant(registrar.inner_variant), - idempotent_variant(registrar.idempotent_variant) + idempotent_variant(registrar.idempotent_variant), + replicable_variant(registrar.replicable_variant) //-------------------------------------------------------------------------- { if (udata != NULL) @@ -10000,6 +11104,7 @@ namespace Legion { rez.serialize(leaf_variant); rez.serialize(inner_variant); rez.serialize(idempotent_variant); + rez.serialize(replicable_variant); size_t name_size = strlen(variant_name)+1; rez.serialize(variant_name, name_size); // Pack the constraints @@ -10067,6 +11172,7 @@ namespace Legion { derez.deserialize(registrar.leaf_variant); derez.deserialize(registrar.inner_variant); derez.deserialize(registrar.idempotent_variant); + derez.deserialize(registrar.replicable_variant); // The last thing will be the name registrar.task_variant_name = (const char*)derez.get_current_pointer(); size_t name_size = strlen(registrar.task_variant_name)+1; @@ -10529,7 +11635,7 @@ namespace Legion { LayoutConstraintID layout_id, Runtime *runtime) //-------------------------------------------------------------------------- { - return (layout_id % runtime->runtime_stride); + return (layout_id % runtime->total_address_spaces); } //-------------------------------------------------------------------------- @@ -10606,7 +11712,9 @@ namespace Legion { unsigned index, LogicalRegion upper_bound, const DomainPoint &point) //-------------------------------------------------------------------------- { - return upper_bound; + // We know we don't use the domain so we can fake it + Domain launch_domain; + return project(upper_bound, point, launch_domain); } //-------------------------------------------------------------------------- @@ -10614,7 +11722,32 @@ namespace Legion { unsigned index, LogicalPartition upper_bound, const DomainPoint &point) //-------------------------------------------------------------------------- { - return runtime->get_logical_subregion_by_color(upper_bound, point); + // We know we don't use the domain so we can fake it + Domain launch_domain; + return project(upper_bound, point, launch_domain); + } + + //-------------------------------------------------------------------------- + LogicalRegion IdentityProjectionFunctor::project(LogicalRegion upper_bound, + const DomainPoint &point, const Domain &launch_domain) + //-------------------------------------------------------------------------- + { + return upper_bound; + } + + //-------------------------------------------------------------------------- + LogicalRegion IdentityProjectionFunctor::project(LogicalPartition up_bound, + const DomainPoint &point, const Domain &launch_domain) + //-------------------------------------------------------------------------- + { + return runtime->get_logical_subregion_by_color(up_bound, point); + } + + //-------------------------------------------------------------------------- + bool IdentityProjectionFunctor::is_functional(void) const + //-------------------------------------------------------------------------- + { + return true; } //-------------------------------------------------------------------------- @@ -10639,6 +11772,7 @@ namespace Legion { ProjectionFunction::ProjectionFunction(ProjectionID pid, ProjectionFunctor *func) : depth(func->get_depth()), is_exclusive(func->is_exclusive()), + is_functional(func->is_functional()), is_invertible(func->is_invertible()), projection_id(pid), functor(func) //-------------------------------------------------------------------------- { @@ -10647,8 +11781,8 @@ namespace Legion { //-------------------------------------------------------------------------- ProjectionFunction::ProjectionFunction(const ProjectionFunction &rhs) : depth(rhs.depth), is_exclusive(rhs.is_exclusive), - is_invertible(rhs.is_invertible), projection_id(rhs.projection_id), - functor(rhs.functor) + is_functional(rhs.is_functional), is_invertible(rhs.is_invertible), + projection_id(rhs.projection_id), functor(rhs.functor) //-------------------------------------------------------------------------- { // should never be called @@ -10666,7 +11800,7 @@ namespace Legion { //-------------------------------------------------------------------------- LogicalRegion ProjectionFunction::project_point(Task *task, unsigned idx, - Runtime *runtime, const DomainPoint &point) + Runtime *runtime, const Domain &launch_domain, const DomainPoint &point) //-------------------------------------------------------------------------- { const RegionRequirement &req = task->regions[idx]; @@ -10678,14 +11812,17 @@ namespace Legion { AutoLock p_lock(projection_reservation); if (req.handle_type == LEGION_PARTITION_PROJECTION) { - LogicalRegion result = functor->project(task, idx, - req.partition, point); + LogicalRegion result = is_functional ? + functor->project(req.partition, point, launch_domain) : + functor->project(task, idx, req.partition, point); check_projection_partition_result(req, task, idx, result, runtime); return result; } else { - LogicalRegion result = functor->project(task, idx, req.region, point); + LogicalRegion result = is_functional ? + functor->project(req.region, point, launch_domain) : + functor->project(task, idx, req.region, point); check_projection_region_result(req, task, idx, result, runtime); return result; } @@ -10694,14 +11831,17 @@ namespace Legion { { if (req.handle_type == LEGION_PARTITION_PROJECTION) { - LogicalRegion result = functor->project(task, idx, - req.partition, point); + LogicalRegion result = is_functional ? + functor->project(req.partition, point, launch_domain) : + functor->project(task, idx, req.partition, point); check_projection_partition_result(req, task, idx, result, runtime); return result; } else { - LogicalRegion result = functor->project(task, idx, req.region, point); + LogicalRegion result = is_functional ? + functor->project(req.region, point, launch_domain) : + functor->project(task, idx, req.region, point); check_projection_region_result(req, task, idx, result, runtime); return result; } @@ -10710,9 +11850,8 @@ namespace Legion { //-------------------------------------------------------------------------- void ProjectionFunction::project_points(const RegionRequirement &req, - unsigned idx, Runtime *runtime, - const std::vector &point_tasks, - IndexSpaceNode *launch_space) + unsigned idx, Runtime *runtime, const Domain &launch_domain, + const std::vector &point_tasks) //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION @@ -10720,9 +11859,6 @@ namespace Legion { #endif std::map > dependences; const bool find_dependences = is_invertible && IS_WRITE(req); - Domain launch_domain; - if (find_dependences) - launch_space->get_launch_space_domain(launch_domain); if (!is_exclusive) { AutoLock p_lock(projection_reservation); @@ -10731,9 +11867,12 @@ namespace Legion { for (std::vector::const_iterator it = point_tasks.begin(); it != point_tasks.end(); it++) { - LogicalRegion result = functor->project(*it, idx, req.partition, - (*it)->get_domain_point()); - check_projection_partition_result(req, static_cast(*it), + LogicalRegion result = is_functional ? + functor->project(req.partition, + (*it)->get_domain_point(), launch_domain) : + functor->project(*it, idx, req.partition, + (*it)->get_domain_point()); + check_projection_partition_result(req, static_cast(*it), idx, result, runtime); (*it)->set_projection_result(idx, result); if (find_dependences) @@ -10755,8 +11894,10 @@ namespace Legion { for (std::vector::const_iterator it = point_tasks.begin(); it != point_tasks.end(); it++) { - LogicalRegion result = functor->project(*it, idx, req.region, - (*it)->get_domain_point()); + LogicalRegion result = is_functional ? + functor->project(req.region, + (*it)->get_domain_point(), launch_domain) : + functor->project(*it, idx, req.region,(*it)->get_domain_point()); check_projection_region_result(req, static_cast(*it), idx, result, runtime); (*it)->set_projection_result(idx, result); @@ -10782,9 +11923,12 @@ namespace Legion { for (std::vector::const_iterator it = point_tasks.begin(); it != point_tasks.end(); it++) { - LogicalRegion result = functor->project(*it, idx, req.partition, - (*it)->get_domain_point()); - check_projection_partition_result(req, static_cast(*it), + LogicalRegion result = is_functional ? + functor->project(req.partition, + (*it)->get_domain_point(), launch_domain) : + functor->project(*it, idx, req.partition, + (*it)->get_domain_point()); + check_projection_partition_result(req, static_cast(*it), idx, result, runtime); (*it)->set_projection_result(idx, result); if (find_dependences) @@ -10806,9 +11950,11 @@ namespace Legion { for (std::vector::const_iterator it = point_tasks.begin(); it != point_tasks.end(); it++) { - LogicalRegion result = functor->project(*it, idx, req.region, - (*it)->get_domain_point()); - check_projection_region_result(req, static_cast(*it), + LogicalRegion result = is_functional ? + functor->project(req.region, + (*it)->get_domain_point(), launch_domain) : + functor->project(*it, idx, req.region,(*it)->get_domain_point()); + check_projection_region_result(req, static_cast(*it), idx, result, runtime); (*it)->set_projection_result(idx, result); if (find_dependences) @@ -10830,7 +11976,8 @@ namespace Legion { //-------------------------------------------------------------------------- void ProjectionFunction::project_points(Operation *op, unsigned idx, - const RegionRequirement &req, Runtime *runtime, + const RegionRequirement &req, + Runtime *runtime, const Domain &launch_domain, const std::vector &points) //-------------------------------------------------------------------------- { @@ -10842,7 +11989,6 @@ namespace Legion { // TODO: support for invertible point operations if (is_invertible && (req.privilege == LEGION_READ_WRITE)) assert(false); - if (!is_exclusive) { AutoLock p_lock(projection_reservation); @@ -10851,8 +11997,11 @@ namespace Legion { for (std::vector::const_iterator it = points.begin(); it != points.end(); it++) { - LogicalRegion result = functor->project(mappable, idx, - req.partition, (*it)->get_domain_point()); + LogicalRegion result = is_functional ? + functor->project(req.partition, + (*it)->get_domain_point(), launch_domain) : + functor->project(mappable, idx, req.partition, + (*it)->get_domain_point()); check_projection_partition_result(req, op, idx, result, runtime); (*it)->set_projection_result(idx, result); } @@ -10862,8 +12011,11 @@ namespace Legion { for (std::vector::const_iterator it = points.begin(); it != points.end(); it++) { - LogicalRegion result = functor->project(mappable, idx, req.region, - (*it)->get_domain_point()); + LogicalRegion result = is_functional ? + functor->project(req.region, + (*it)->get_domain_point(), launch_domain) : + functor->project(mappable, idx, req.region, + (*it)->get_domain_point()); check_projection_region_result(req, op, idx, result, runtime); (*it)->set_projection_result(idx, result); } @@ -10876,8 +12028,11 @@ namespace Legion { for (std::vector::const_iterator it = points.begin(); it != points.end(); it++) { - LogicalRegion result = functor->project(mappable, idx, - req.partition, (*it)->get_domain_point()); + LogicalRegion result = is_functional ? + functor->project(req.partition, + (*it)->get_domain_point(), launch_domain) : + functor->project(mappable, idx, req.partition, + (*it)->get_domain_point()); check_projection_partition_result(req, op, idx, result, runtime); (*it)->set_projection_result(idx, result); } @@ -10887,8 +12042,11 @@ namespace Legion { for (std::vector::const_iterator it = points.begin(); it != points.end(); it++) { - LogicalRegion result = functor->project(mappable, idx, req.region, - (*it)->get_domain_point()); + LogicalRegion result = is_functional ? + functor->project(req.region, + (*it)->get_domain_point(), launch_domain) : + functor->project(mappable, idx, req.region, + (*it)->get_domain_point()); check_projection_region_result(req, op, idx, result, runtime); (*it)->set_projection_result(idx, result); } @@ -11098,6 +12256,469 @@ namespace Legion { #endif } + //-------------------------------------------------------------------------- + ProjectionFunction::ElideCloseResult::ElideCloseResult(void) + : node(NULL), result(false) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ProjectionFunction::ElideCloseResult::ElideCloseResult(IndexTreeNode *n, + const std::set &proj, bool res) + : node(n), projections(proj), result(res) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + bool ProjectionFunction::ElideCloseResult::matches(IndexTreeNode *other, + const std::set &other_projections) const + //-------------------------------------------------------------------------- + { + if (node != other) + return false; + if (projections.size() != other_projections.size()) + return false; + std::set::const_iterator it1 = projections.begin(); + std::set::const_iterator it2 = + other_projections.begin(); + while (it1 != projections.end()) + { + if (it1 != it2) + return false; + it1++; it2++; + } + return true; + } + + //-------------------------------------------------------------------------- + bool ProjectionFunction::find_elide_close_result(const ProjectionInfo &info, + const std::set &projections, + RegionTreeNode *node, bool &result) const + //-------------------------------------------------------------------------- + { + // No memoizing if we're not functional + if (!is_functional) + return false; + ProjectionSummary key(info); + IndexTreeNode *row_source = node->get_row_source(); + AutoLock p_lock(projection_reservation,1,false/*exclusive*/); + std::map >::const_iterator + finder = elide_close_results.find(key); + if (finder == elide_close_results.end()) + return false; + for (std::vector::const_iterator it = + finder->second.begin(); it != finder->second.end(); it++) + { + if (it->matches(row_source, projections)) + { + result = it->result; + return true; + } + } + return false; + } + + //-------------------------------------------------------------------------- + void ProjectionFunction::record_elide_close_result( + const ProjectionInfo &info, + const std::set &projections, + RegionTreeNode *node, bool result) + //-------------------------------------------------------------------------- + { + if (!is_functional) + return; + ProjectionSummary key(info); + IndexTreeNode *row_source = node->get_row_source(); + AutoLock p_lock(projection_reservation); + std::vector &close_results = elide_close_results[key]; + // See if someone else saved the result in between to avoid duplicates + for (std::vector::const_iterator it = + close_results.begin(); it != close_results.end(); it++) + if (it->matches(row_source, projections)) + return; + close_results.push_back(ElideCloseResult(row_source, projections,result)); + } + + //-------------------------------------------------------------------------- + ProjectionTree* ProjectionFunction::construct_projection_tree(Operation *op, + unsigned index, RegionTreeNode *root, + IndexSpaceNode *launch_space, + ShardingFunction *sharding_function, + IndexSpaceNode *sharding_space) const + //-------------------------------------------------------------------------- + { + Mappable *mappable = is_functional ? NULL : op->get_mappable(); + IndexTreeNode *row_source = root->get_row_source(); + RegionTreeForest *context = root->context; + ProjectionTree *result = new ProjectionTree(row_source); + std::map node_map; + node_map[row_source] = result; + // Iterate over the points, compute the projections, and build the tree + Domain launch_domain, sharding_domain; + launch_space->get_launch_space_domain(launch_domain); + if ((sharding_function != NULL) && (launch_space != sharding_space)) + sharding_space->get_launch_space_domain(sharding_domain); + else + sharding_domain = launch_domain; + if (root->is_region()) + { + RegionNode *region = root->as_region_node(); + for (Domain::DomainPointIterator itr(launch_domain); itr; itr++) + { + LogicalRegion result; + if (!is_exclusive) + { + AutoLock p_lock(projection_reservation); + if (is_functional) + result = functor->project(region->handle, itr.p, launch_domain); + else + result = functor->project(mappable, index, region->handle, itr.p); + } + else + { + if (is_functional) + result = functor->project(region->handle, itr.p, launch_domain); + else + result = functor->project(mappable, index, region->handle, itr.p); + } + if (!result.exists()) + continue; + if (sharding_function != NULL) + { + ShardID own = sharding_function->find_owner(itr.p, sharding_domain); + add_to_projection_tree(result, row_source, context, node_map, own); + } + else + add_to_projection_tree(result, row_source, context, node_map); + } + } + else + { + PartitionNode *partition = root->as_partition_node(); + for (Domain::DomainPointIterator itr(launch_domain); itr; itr++) + { + LogicalRegion result; + if (!is_exclusive) + { + AutoLock p_lock(projection_reservation); + if (is_functional) + result = functor->project(partition->handle, itr.p,launch_domain); + else + result = functor->project(mappable,index,partition->handle,itr.p); + } + else + { + if (is_functional) + result = functor->project(partition->handle, itr.p,launch_domain); + else + result = functor->project(mappable,index,partition->handle,itr.p); + } + if (!result.exists()) + continue; + if (sharding_function != NULL) + { + ShardID own = sharding_function->find_owner(itr.p, sharding_domain); + add_to_projection_tree(result, row_source, context, node_map, own); + } + else + add_to_projection_tree(result, row_source, context, node_map); + } + } + return result; + } + + //-------------------------------------------------------------------------- + void ProjectionFunction::construct_projection_tree(Operation *op, + unsigned index, RegionTreeNode *root, IndexSpaceNode *launch_space, + ShardingFunction *sharding_function, IndexSpaceNode *sharding_space, + std::map &node_map) const + //-------------------------------------------------------------------------- + { + Mappable *mappable = is_functional ? NULL : op->get_mappable(); + IndexTreeNode *row_source = root->get_row_source(); + RegionTreeForest *context = root->context; + // Iterate over the points, compute the projections, and build the tree + Domain launch_domain, sharding_domain; + launch_space->get_launch_space_domain(launch_domain); + if ((sharding_function != NULL) && (launch_space != sharding_space)) + sharding_space->get_launch_space_domain(sharding_domain); + else + sharding_domain = launch_domain; + if (root->is_region()) + { + RegionNode *region = root->as_region_node(); + for (Domain::DomainPointIterator itr(launch_domain); itr; itr++) + { + LogicalRegion result; + if (!is_exclusive) + { + AutoLock p_lock(projection_reservation); + if (is_functional) + result = functor->project(region->handle, itr.p, launch_domain); + else + result = functor->project(mappable, index, region->handle, itr.p); + } + else + { + if (is_functional) + result = functor->project(region->handle, itr.p, launch_domain); + else + result = functor->project(mappable, index, region->handle, itr.p); + } + if (!result.exists()) + continue; + if (sharding_function != NULL) + { + ShardID own = sharding_function->find_owner(itr.p, sharding_domain); + add_to_projection_tree(result, row_source, context, node_map, own); + } + else + add_to_projection_tree(result, row_source, context, node_map); + } + } + else + { + PartitionNode *partition = root->as_partition_node(); + for (Domain::DomainPointIterator itr(launch_domain); itr; itr++) + { + LogicalRegion result; + if (!is_exclusive) + { + AutoLock p_lock(projection_reservation); + if (is_functional) + result = functor->project(partition->handle, itr.p,launch_domain); + else + result = functor->project(mappable,index,partition->handle,itr.p); + } + else + { + if (is_functional) + result = functor->project(partition->handle, itr.p,launch_domain); + else + result = functor->project(mappable,index,partition->handle,itr.p); + } + if (!result.exists()) + continue; + if (sharding_function != NULL) + { + ShardID own = sharding_function->find_owner(itr.p, sharding_domain); + add_to_projection_tree(result, row_source, context, node_map, own); + } + else + add_to_projection_tree(result, row_source, context, node_map); + } + } + } + + //-------------------------------------------------------------------------- + /*static*/ void ProjectionFunction::add_to_projection_tree(LogicalRegion r, + IndexTreeNode *root, RegionTreeForest *context, + std::map &node_map, + ShardID owner_shard) + //-------------------------------------------------------------------------- + { + IndexTreeNode *child = context->get_node(r)->row_source; + std::map::const_iterator finder = + node_map.find(child); + ProjectionTree *current = NULL; + if (finder == node_map.end()) + { + current = new ProjectionTree(child, owner_shard); + node_map[child] = current; + } + else + current = finder->second; + while (child != root) + { + // Find the next one to add this to + IndexTreeNode *parent = child->get_parent(); + finder = node_map.find(parent); + ProjectionTree *next = NULL; + if (finder == node_map.end()) + { + next = new ProjectionTree(parent); + node_map[parent] = next; + } + else + next = finder->second; + next->add_child(current); + // Now we can walk up the tree + child = parent; + current = next; + } + } + + ///////////////////////////////////////////////////////////// + // Cyclic Sharding Functor + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + CyclicShardingFunctor::CyclicShardingFunctor(void) + : ShardingFunctor() + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + CyclicShardingFunctor::CyclicShardingFunctor( + const CyclicShardingFunctor &rhs) + : ShardingFunctor() + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + CyclicShardingFunctor::~CyclicShardingFunctor(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + CyclicShardingFunctor& CyclicShardingFunctor::operator=( + const CyclicShardingFunctor &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + template + size_t CyclicShardingFunctor::linearize_point( + const Realm::IndexSpace &is, + const Realm::Point &point) const + //-------------------------------------------------------------------------- + { + if (is.dense()) + { + Realm::AffineLinearizedIndexSpace linearizer(is); + return linearizer.linearize(point); + } + else + { + size_t offset = 0; + for (Realm::IndexSpaceIterator it(is); it.valid; it.step()) + { + if (it.rect.contains(point)) + { + Realm::AffineLinearizedIndexSpace + linearizer(Realm::IndexSpace(it.rect)); + return offset + linearizer.linearize(point); + } + else + offset += it.rect.volume(); + } + return offset; + } + } + + //-------------------------------------------------------------------------- + ShardID CyclicShardingFunctor::shard(const DomainPoint &point, + const Domain &full_space, + const size_t total_shards) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(point.get_dim() == full_space.get_dim()); +#endif + switch (point.get_dim()) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + const DomainT is = full_space; \ + const Point p1 = point; \ + return (linearize_point(is, p1) % total_shards); \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + return 0; + } + + ///////////////////////////////////////////////////////////// + // Sharding Function + ///////////////////////////////////////////////////////////// + + //-------------------------------------------------------------------------- + ShardingFunction::ShardingFunction(ShardingFunctor *func, + RegionTreeForest *f, ShardingID id, size_t total) + : functor(func), forest(f), sharding_id(id), total_shards(total) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardingFunction::ShardingFunction(const ShardingFunction &rhs) + : functor(NULL), forest(NULL), sharding_id(0), total_shards(0) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + } + + //-------------------------------------------------------------------------- + ShardingFunction::~ShardingFunction(void) + //-------------------------------------------------------------------------- + { + } + + //-------------------------------------------------------------------------- + ShardingFunction& ShardingFunction::operator=(const ShardingFunction &rhs) + //-------------------------------------------------------------------------- + { + // should never be called + assert(false); + return *this; + } + + //-------------------------------------------------------------------------- + ShardID ShardingFunction::find_owner(const DomainPoint &point, + const Domain &sharding_space) + //-------------------------------------------------------------------------- + { +#ifdef DEBUG_LEGION + assert(sharding_space.contains(point)); +#endif + ShardID result = functor->shard(point, sharding_space, total_shards); + if (total_shards <= result) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARDING_FUNCTOR_OUTPUT, + "Illegal output shard %d from sharding functor %d. " + "Shards for this index space launch must be " + "between 0 and %zd (exclusive).", result, + sharding_id, total_shards) + return result; + } + + //-------------------------------------------------------------------------- + IndexSpace ShardingFunction::find_shard_space(ShardID shard, + IndexSpaceNode *full_space, IndexSpace shard_space) + //-------------------------------------------------------------------------- + { + const ShardKey key(shard, full_space->handle, shard_space); + // Check to see if we already have it + { + AutoLock s_lock(sharding_lock,1,false/*exclusive*/); + std::map::const_iterator + finder = shard_index_spaces.find(key); + if (finder != shard_index_spaces.end()) + return finder->second; + } + // Otherwise we need to make it + IndexSpace result = + full_space->create_shard_space(this, shard, shard_space); + AutoLock s_lock(sharding_lock); + shard_index_spaces[key] = result; + return result; + } + ///////////////////////////////////////////////////////////// // Legion Runtime ///////////////////////////////////////////////////////////// @@ -11124,6 +12745,8 @@ namespace Legion { initial_meta_task_vector_width(config.initial_meta_task_vector_width), max_message_size(config.max_message_size), gc_epoch_size(config.gc_epoch_size), + max_control_replication_contexts( + config.max_control_replication_contexts), max_local_fields(config.max_local_fields), max_replay_parallelism(config.max_replay_parallelism), program_order_execution(config.program_order_execution), @@ -11147,6 +12770,7 @@ namespace Legion { #else unsafe_mapper(!config.safe_mapper), #endif + safe_control_replication(config.safe_control_replication), disable_independence_tests(config.disable_independence_tests), #ifdef LEGION_SPY legion_spy_enabled(true), @@ -11166,11 +12790,6 @@ namespace Legion { check_privileges(config.check_privileges), num_profiling_nodes(config.num_profiling_nodes), legion_collective_radix(config.legion_collective_radix), - legion_collective_log_radix(config.legion_collective_log_radix), - legion_collective_stages(config.legion_collective_stages), - legion_collective_last_radix(config.legion_collective_last_radix), - legion_collective_participating_spaces( - config.legion_collective_participating_spaces), mpi_rank_table((mpi_rank >= 0) ? new MPIRankTable(this) : NULL), prepared_for_shutdown(false), total_outstanding_tasks(0), // In the case where the runtime is backgrounded, have node 0 keep @@ -11195,6 +12814,7 @@ namespace Legion { ((unique == 0) ? runtime_stride : unique)), unique_constraint_id((unique == 0) ? runtime_stride : unique), unique_is_expr_id((unique == 0) ? runtime_stride : unique), + unique_control_replication_id((unique == 0) ? runtime_stride : unique), #ifdef LEGION_SPY unique_indirections_id((unique == 0) ? runtime_stride : unique), #endif @@ -11202,11 +12822,13 @@ namespace Legion { unique_mapper_id(get_current_static_mapper_id()+unique), unique_trace_id(get_current_static_trace_id()+unique), unique_projection_id(get_current_static_projection_id()+unique), + unique_sharding_id(get_current_static_sharding_id()+unique), unique_redop_id(get_current_static_reduction_id()+unique), unique_serdez_id(get_current_static_serdez_id()+unique), unique_library_mapper_id(LEGION_INITIAL_LIBRARY_ID_OFFSET), unique_library_trace_id(LEGION_INITIAL_LIBRARY_ID_OFFSET), unique_library_projection_id(LEGION_INITIAL_LIBRARY_ID_OFFSET), + unique_library_sharding_id(LEGION_INITIAL_LIBRARY_ID_OFFSET), unique_library_task_id(LEGION_INITIAL_LIBRARY_ID_OFFSET), unique_library_redop_id(LEGION_INITIAL_LIBRARY_ID_OFFSET), unique_library_serdez_id(LEGION_INITIAL_LIBRARY_ID_OFFSET), @@ -11332,6 +12954,7 @@ namespace Legion { initial_meta_task_vector_width(rhs.initial_meta_task_vector_width), max_message_size(rhs.max_message_size), gc_epoch_size(rhs.gc_epoch_size), + max_control_replication_contexts(rhs.max_control_replication_contexts), max_local_fields(rhs.max_local_fields), max_replay_parallelism(rhs.max_replay_parallelism), program_order_execution(rhs.program_order_execution), @@ -11351,6 +12974,7 @@ namespace Legion { resilient_mode(rhs.resilient_mode), unsafe_launch(rhs.unsafe_launch), unsafe_mapper(rhs.unsafe_mapper), + safe_control_replication(rhs.safe_control_replication), disable_independence_tests(rhs.disable_independence_tests), legion_spy_enabled(rhs.legion_spy_enabled), supply_default_mapper(rhs.supply_default_mapper), @@ -11366,11 +12990,6 @@ namespace Legion { check_privileges(rhs.check_privileges), num_profiling_nodes(rhs.num_profiling_nodes), legion_collective_radix(rhs.legion_collective_radix), - legion_collective_log_radix(rhs.legion_collective_log_radix), - legion_collective_stages(rhs.legion_collective_stages), - legion_collective_last_radix(rhs.legion_collective_last_radix), - legion_collective_participating_spaces( - rhs.legion_collective_participating_spaces), mpi_rank_table(NULL), local_procs(rhs.local_procs), local_utils(rhs.local_utils), proc_spaces(rhs.proc_spaces) //-------------------------------------------------------------------------- @@ -11417,6 +13036,13 @@ namespace Legion { delete it->second; } projection_functions.clear(); + for (std::map::iterator it = + sharding_functors.begin(); it != + sharding_functors.end(); it++) + { + delete it->second; + } + sharding_functors.clear(); } for (std::deque::const_iterator it = available_individual_tasks.begin(); @@ -11649,25 +13275,188 @@ namespace Legion { delete (*it); } available_all_reduce_ops.clear(); - for (std::map::const_iterator it = - task_table.begin(); it != task_table.end(); it++) + for (std::deque::const_iterator it = + available_repl_individual_tasks.begin(); it != + available_repl_individual_tasks.end(); it++) { - delete (it->second); + delete (*it); } - task_table.clear(); - // Skip this if we are in separate runtime mode - if (!separate_runtime_instances) + available_repl_individual_tasks.clear(); + for (std::deque::const_iterator it = + available_repl_index_tasks.begin(); it != + available_repl_index_tasks.end(); it++) { - for (std::deque::const_iterator it = - variant_table.begin(); it != variant_table.end(); it++) - { - delete (*it); - } + delete (*it); } - variant_table.clear(); - // Skip this if we are in separate runtime mode - if (!separate_runtime_instances) + available_repl_index_tasks.clear(); + for (std::deque::const_iterator it = + available_repl_merge_close_ops.begin(); it != + available_repl_merge_close_ops.end(); it++) { + delete (*it); + } + available_repl_merge_close_ops.clear(); + for (std::deque::const_iterator it = + available_repl_fill_ops.begin(); it != + available_repl_fill_ops.end(); it++) + { + delete (*it); + } + available_repl_fill_ops.clear(); + for (std::deque::const_iterator it = + available_repl_index_fill_ops.begin(); it != + available_repl_index_fill_ops.end(); it++) + { + delete (*it); + } + available_repl_index_fill_ops.clear(); + for (std::deque::const_iterator it = + available_repl_copy_ops.begin(); it != + available_repl_copy_ops.end(); it++) + { + delete (*it); + } + available_repl_copy_ops.clear(); + for (std::deque::const_iterator it = + available_repl_index_copy_ops.begin(); it != + available_repl_index_copy_ops.end(); it++) + { + delete (*it); + } + available_repl_index_copy_ops.clear(); + for (std::deque::const_iterator it = + available_repl_deletion_ops.begin(); it != + available_repl_deletion_ops.end(); it++) + { + delete (*it); + } + available_repl_deletion_ops.clear(); + for (std::deque::const_iterator it = + available_repl_pending_partition_ops.begin(); it != + available_repl_pending_partition_ops.end(); it++) + { + delete (*it); + } + available_repl_pending_partition_ops.clear(); + for (std::deque::const_iterator it = + available_repl_dependent_partition_ops.begin(); it != + available_repl_dependent_partition_ops.end(); it++) + { + delete (*it); + } + available_repl_dependent_partition_ops.clear(); + for (std::deque::const_iterator it = + available_repl_must_epoch_ops.begin(); it != + available_repl_must_epoch_ops.end(); it++) + { + delete (*it); + } + available_repl_must_epoch_ops.clear(); + for (std::deque::const_iterator it = + available_repl_timing_ops.begin(); it != + available_repl_timing_ops.end(); it++) + { + delete (*it); + } + available_repl_timing_ops.clear(); + for (std::deque::const_iterator it = + available_repl_all_reduce_ops.begin(); it != + available_repl_all_reduce_ops.end(); it++) + { + delete (*it); + } + available_repl_all_reduce_ops.clear(); + for (std::deque::const_iterator it = + available_repl_fence_ops.begin(); it != + available_repl_fence_ops.end(); it++) + { + delete (*it); + } + available_repl_fence_ops.clear(); + for (std::deque::const_iterator it = + available_repl_map_ops.begin(); it != + available_repl_map_ops.end(); it++) + { + delete (*it); + } + available_repl_map_ops.clear(); + for (std::deque::const_iterator it = + available_repl_attach_ops.begin(); it != + available_repl_attach_ops.end(); it++) + { + delete (*it); + } + available_repl_attach_ops.clear(); + for (std::deque::const_iterator it = + available_repl_detach_ops.begin(); it != + available_repl_detach_ops.end(); it++) + { + delete (*it); + } + available_repl_detach_ops.clear(); + for (std::deque::const_iterator it = + available_repl_capture_ops.begin(); it != + available_repl_capture_ops.end(); it++) + { + delete (*it); + } + available_repl_capture_ops.clear(); + for (std::deque::const_iterator it = + available_repl_trace_ops.begin(); it != + available_repl_trace_ops.end(); it++) + { + delete (*it); + } + available_repl_trace_ops.clear(); + for (std::deque::const_iterator it = + available_repl_replay_ops.begin(); it != + available_repl_replay_ops.end(); it++) + { + delete (*it); + } + available_repl_replay_ops.clear(); + for (std::deque::const_iterator it = + available_repl_begin_ops.begin(); it != + available_repl_begin_ops.end(); it++) + { + delete (*it); + } + available_repl_begin_ops.clear(); + for (std::deque::const_iterator it = + available_repl_summary_ops.begin(); it != + available_repl_summary_ops.end(); it++) + { + delete (*it); + } + available_repl_summary_ops.clear(); + for (std::map::const_iterator it = + task_table.begin(); it != task_table.end(); it++) + { + delete (it->second); + } + task_table.clear(); + // Skip this if we are in separate runtime mode + if (!separate_runtime_instances) + { + for (std::deque::const_iterator it = + variant_table.begin(); it != variant_table.end(); it++) + { + delete (*it); + } + } + variant_table.clear(); + // Skip this if we are in separate runtime mode + if (!separate_runtime_instances) + { + while (!layout_constraints_table.empty()) + { + std::map::iterator next_it = + layout_constraints_table.begin(); + LayoutConstraints *next = next_it->second; + layout_constraints_table.erase(next_it); + if (next->remove_base_resource_ref(RUNTIME_REF)) + delete (next); + } while (!layout_constraints_table.empty()) { std::map::iterator next_it = @@ -11816,6 +13605,22 @@ namespace Legion { true/*was preregistered*/, NULL, true/*preregistered*/); } + //-------------------------------------------------------------------------- + void Runtime::register_static_sharding_functors(void) + //-------------------------------------------------------------------------- + { + std::map &pending_sharding_functors = + get_pending_sharding_table(); + for (std::map::const_iterator it = + pending_sharding_functors.begin(); it != + pending_sharding_functors.end(); it++) + register_sharding_functor(it->first, it->second, true/*zero check*/, + true/*was preregistered*/, NULL, true/*preregistered*/); + register_sharding_functor(0, + new CyclicShardingFunctor(), false/*need check*/, + true/*was preregistered*/, NULL, true/*preregistered*/); + } + //-------------------------------------------------------------------------- void Runtime::initialize_legion_prof(const LegionConfiguration &config) //-------------------------------------------------------------------------- @@ -12136,6 +13941,7 @@ namespace Legion { register_static_variants(); register_static_constraints(); register_static_projections(); + register_static_sharding_functors(); // Initialize our virtual manager and our mappers initialize_virtual_manager(); // Finally perform the registration callback methods @@ -12439,12 +14245,21 @@ namespace Legion { } //-------------------------------------------------------------------------- - void Runtime::create_shared_ownership(IndexSpace handle) + void Runtime::create_shared_ownership(IndexSpace handle, + const bool total_sharding_collective) //-------------------------------------------------------------------------- { const AddressSpaceID owner_space = IndexSpaceNode::get_owner_space(handle, this); - if (owner_space != address_space) + if (owner_space == address_space) + { + IndexSpaceNode *node = forest->get_node(handle); + if (!node->check_valid_and_increment(APPLICATION_REF)) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARED_OWNERSHIP, + "Illegal call to add shared ownership to index space %x " + "which has already been deleted", handle.get_id()) + } + else if (!total_sharding_collective) { Serializer rez; { @@ -12454,23 +14269,24 @@ namespace Legion { } send_shared_ownership(owner_space, rez); } - else - { - IndexSpaceNode *node = forest->get_node(handle); - if (!node->check_valid_and_increment(APPLICATION_REF)) - REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARED_OWNERSHIP, - "Illegal call to add shared ownership to index space %x " - "which has already been deleted", handle.get_id()) - } } //-------------------------------------------------------------------------- - void Runtime::create_shared_ownership(IndexPartition handle) + void Runtime::create_shared_ownership(IndexPartition handle, + const bool total_sharding_collective) //-------------------------------------------------------------------------- { const AddressSpaceID owner_space = IndexPartNode::get_owner_space(handle, this); - if (owner_space != address_space) + if (owner_space == address_space) + { + IndexPartNode *node = forest->get_node(handle); + if (!node->check_valid_and_increment(APPLICATION_REF)) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARED_OWNERSHIP, + "Illegal call to add shared ownership to index partition %x " + "which has already been deleted", handle.get_id()) + } + else if (!total_sharding_collective) { Serializer rez; { @@ -12480,23 +14296,24 @@ namespace Legion { } send_shared_ownership(owner_space, rez); } - else - { - IndexPartNode *node = forest->get_node(handle); - if (!node->check_valid_and_increment(APPLICATION_REF)) - REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARED_OWNERSHIP, - "Illegal call to add shared ownership to index partition %x " - "which has already been deleted", handle.get_id()) - } } //-------------------------------------------------------------------------- - void Runtime::create_shared_ownership(FieldSpace handle) + void Runtime::create_shared_ownership(FieldSpace handle, + const bool total_sharding_collective) //-------------------------------------------------------------------------- { const AddressSpaceID owner_space = FieldSpaceNode::get_owner_space(handle, this); - if (owner_space != address_space) + if (owner_space == address_space) + { + FieldSpaceNode *node = forest->get_node(handle); + if (!node->check_valid_and_increment(APPLICATION_REF)) + REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARED_OWNERSHIP, + "Illegal call to add shared ownership to field space %x " + "which has already been deleted", handle.get_id()) + } + else if (!total_sharding_collective) { Serializer rez; { @@ -12506,33 +14323,16 @@ namespace Legion { } send_shared_ownership(owner_space, rez); } - else - { - FieldSpaceNode *node = forest->get_node(handle); - if (!node->check_valid_and_increment(APPLICATION_REF)) - REPORT_LEGION_ERROR(ERROR_ILLEGAL_SHARED_OWNERSHIP, - "Illegal call to add shared ownership to field space %x " - "which has already been deleted", handle.get_id()) - } } //-------------------------------------------------------------------------- - void Runtime::create_shared_ownership(LogicalRegion handle) + void Runtime::create_shared_ownership(LogicalRegion handle, + const bool total_sharding_collective) //-------------------------------------------------------------------------- { const AddressSpaceID owner_space = RegionNode::get_owner_space(handle, this); - if (owner_space != address_space) - { - Serializer rez; - { - RezCheck z(rez); - rez.serialize(3); - rez.serialize(handle); - } - send_shared_ownership(owner_space, rez); - } - else + if (owner_space == address_space) { RegionNode *node = forest->get_node(handle); if (!node->check_valid_and_increment(APPLICATION_REF)) @@ -12542,6 +14342,16 @@ namespace Legion { handle.index_space.get_id(), handle.field_space.get_id(), handle.tree_id) } + else if (!total_sharding_collective) + { + Serializer rez; + { + RezCheck z(rez); + rez.serialize(3); + rez.serialize(handle); + } + send_shared_ownership(owner_space, rez); + } } //-------------------------------------------------------------------------- @@ -13567,14 +15377,7 @@ namespace Legion { if (ctx == DUMMY_CONTEXT) REPORT_DUMMY_CONTEXT( "Illegal dummy context create phase barrier!"); -#ifdef DEBUG_LEGION - log_run.debug("Creating phase barrier in task %s (ID %lld)", - ctx->get_task_name(), ctx->get_unique_id()); -#endif - ctx->begin_runtime_call(); - ApBarrier result(Realm::Barrier::create_barrier(arrivals)); - ctx->end_runtime_call(); - return PhaseBarrier(result); + return PhaseBarrier(ctx->create_phase_barrier(arrivals)); } //-------------------------------------------------------------------------- @@ -13584,13 +15387,7 @@ namespace Legion { if (ctx == DUMMY_CONTEXT) REPORT_DUMMY_CONTEXT( "Illegal dummy context destroy phase barrier!"); -#ifdef DEBUG_LEGION - log_run.debug("Destroying phase barrier in task %s (ID %lld)", - ctx->get_task_name(), ctx->get_unique_id()); -#endif - ctx->begin_runtime_call(); - ctx->destroy_user_barrier(pb.phase_barrier); - ctx->end_runtime_call(); + ctx->destroy_phase_barrier(pb.phase_barrier); } //-------------------------------------------------------------------------- @@ -13600,18 +15397,7 @@ namespace Legion { if (ctx == DUMMY_CONTEXT) REPORT_DUMMY_CONTEXT( "Illegal dummy context advance phase barrier!"); -#ifdef DEBUG_LEGION - log_run.debug("Advancing phase barrier in task %s (ID %lld)", - ctx->get_task_name(), ctx->get_unique_id()); -#endif - ctx->begin_runtime_call(); - PhaseBarrier result = pb; - Runtime::advance_barrier(result); -#ifdef LEGION_SPY - LegionSpy::log_event_dependence(pb.phase_barrier, result.phase_barrier); -#endif - ctx->end_runtime_call(); - return result; + return ctx->advance_phase_barrier(pb); } //-------------------------------------------------------------------------- @@ -13625,15 +15411,8 @@ namespace Legion { if (ctx == DUMMY_CONTEXT) REPORT_DUMMY_CONTEXT( "Illegal dummy context create dynamic collective!"); -#ifdef DEBUG_LEGION - log_run.debug("Creating dynamic collective in task %s (ID %lld)", - ctx->get_task_name(), ctx->get_unique_id()); -#endif - ctx->begin_runtime_call(); - ApBarrier result(Realm::Barrier::create_barrier(arrivals, redop, - init_value, init_size)); - ctx->end_runtime_call(); - return DynamicCollective(result, redop); + return DynamicCollective(ctx->create_phase_barrier(arrivals, redop, + init_value, init_size), redop); } //-------------------------------------------------------------------------- @@ -13643,13 +15422,7 @@ namespace Legion { if (ctx == DUMMY_CONTEXT) REPORT_DUMMY_CONTEXT( "Illegal dummy context destroy dynamic collective!"); -#ifdef DEBUG_LEGION - log_run.debug("Destroying dynamic collective in task %s (ID %lld)", - ctx->get_task_name(), ctx->get_unique_id()); -#endif - ctx->begin_runtime_call(); - ctx->destroy_user_barrier(dc.phase_barrier); - ctx->end_runtime_call(); + ctx->destroy_phase_barrier(dc.phase_barrier); } //-------------------------------------------------------------------------- @@ -13661,14 +15434,7 @@ namespace Legion { if (ctx == DUMMY_CONTEXT) REPORT_DUMMY_CONTEXT( "Illegal dummy context arrive dynamic collective!"); -#ifdef DEBUG_LEGION - log_run.debug("Arrive dynamic collective in task %s (ID %lld)", - ctx->get_task_name(), ctx->get_unique_id()); -#endif - ctx->begin_runtime_call(); - Runtime::phase_barrier_arrive(dc, count, ApEvent::NO_AP_EVENT, - buffer, size); - ctx->end_runtime_call(); + ctx->arrive_dynamic_collective(dc, buffer, size, count); } //-------------------------------------------------------------------------- @@ -13681,17 +15447,7 @@ namespace Legion { if (ctx == DUMMY_CONTEXT) REPORT_DUMMY_CONTEXT( "Illegal dummy context defer dynamic collective arrival!"); -#ifdef DEBUG_LEGION - log_run.debug("Defer dynamic collective arrival in " - "task %s (ID %lld)", - ctx->get_task_name(), ctx->get_unique_id()); -#endif - ctx->begin_runtime_call(); - // Record this future as a contribution to the collective - // for future dependence analysis - ctx->record_dynamic_collective_contribution(dc, f); - f.impl->contribute_to_collective(dc, count); - ctx->end_runtime_call(); + ctx->defer_dynamic_collective_arrival(dc, f, count); } //-------------------------------------------------------------------------- @@ -13713,18 +15469,7 @@ namespace Legion { if (ctx == DUMMY_CONTEXT) REPORT_DUMMY_CONTEXT( "Illegal dummy context advance dynamic collective!"); -#ifdef DEBUG_LEGION - log_run.debug("Advancing dynamic collective in task %s (ID %lld)", - ctx->get_task_name(), ctx->get_unique_id()); -#endif - ctx->begin_runtime_call(); - DynamicCollective result = dc; - Runtime::advance_barrier(result); -#ifdef LEGION_SPY - LegionSpy::log_event_dependence(dc.phase_barrier, result.phase_barrier); -#endif - ctx->end_runtime_call(); - return result; + return ctx->advance_dynamic_collective(dc); } //-------------------------------------------------------------------------- @@ -14094,6 +15839,24 @@ namespace Legion { ctx->end_runtime_call(); } + //-------------------------------------------------------------------------- + void Runtime::print_once(Context ctx, FILE *f, const char *message) + //-------------------------------------------------------------------------- + { + if (ctx == DUMMY_CONTEXT) + REPORT_DUMMY_CONTEXT("Illegal dummy context print once!"); + ctx->print_once(f, message); + } + + //-------------------------------------------------------------------------- + void Runtime::log_once(Context ctx, Realm::LoggerMessage &message) + //-------------------------------------------------------------------------- + { + if (ctx == DUMMY_CONTEXT) + REPORT_DUMMY_CONTEXT("Illegal dummy context log once!"); + ctx->log_once(message); + } + //-------------------------------------------------------------------------- void Runtime::yield(Context ctx) //-------------------------------------------------------------------------- @@ -14609,8 +16372,6 @@ namespace Legion { (warning_string == NULL) ? "" : warning_string) ProjectionFunction *function = new ProjectionFunction(pid, functor); AutoLock p_lock(projection_lock); - // No need for a lock because these all need to be reserved at - // registration time before the runtime starts up std::map:: const_iterator finder = projection_functions.find(pid); if (finder != projection_functions.end()) @@ -14690,57 +16451,309 @@ namespace Legion { } //-------------------------------------------------------------------------- - void Runtime::attach_semantic_information(TaskID task_id, SemanticTag tag, - const void *buffer, size_t size, bool is_mutable, bool send_to_owner) - //-------------------------------------------------------------------------- - { - if ((implicit_context != NULL) && - !implicit_context->perform_semantic_attach(send_to_owner)) - return; - if ((tag == LEGION_NAME_SEMANTIC_TAG) && legion_spy_enabled) - LegionSpy::log_task_name(task_id, static_cast(buffer)); - TaskImpl *impl = find_or_create_task_impl(task_id); - impl->attach_semantic_information(tag, address_space, buffer, size, - is_mutable, send_to_owner); - if (implicit_context != NULL) - implicit_context->post_semantic_attach(); - } - - //-------------------------------------------------------------------------- - void Runtime::attach_semantic_information(IndexSpace handle, - SemanticTag tag, - const void *buffer, size_t size, - bool is_mutable) + ShardingID Runtime::generate_dynamic_sharding_id(bool check_context/*true*/) //-------------------------------------------------------------------------- { - bool global = true; - if ((implicit_context != NULL) && - !implicit_context->perform_semantic_attach(global)) - return; - forest->attach_semantic_information(handle, tag, address_space, - buffer, size, is_mutable, !global); - if (implicit_context != NULL) - implicit_context->post_semantic_attach(); + if (check_context && (implicit_context != NULL)) + return implicit_context->generate_dynamic_sharding_id(); + ShardingID result = + __sync_fetch_and_add(&unique_sharding_id, runtime_stride); + // Check for hitting the library limit + if (result >= LEGION_INITIAL_LIBRARY_ID_OFFSET) + REPORT_LEGION_FATAL(LEGION_FATAL_EXCEEDED_LIBRARY_ID_OFFSET, + "Dynamic Shardinging IDs exceeded library ID offset %d", + LEGION_INITIAL_LIBRARY_ID_OFFSET) + return result; } //-------------------------------------------------------------------------- - void Runtime::attach_semantic_information(IndexPartition handle, - SemanticTag tag, - const void *buffer, size_t size, - bool is_mutable) + ShardingID Runtime::generate_library_sharding_ids(const char *name, + size_t cnt) //-------------------------------------------------------------------------- { - bool global = true; - if ((implicit_context != NULL) && - !implicit_context->perform_semantic_attach(global)) - return; - forest->attach_semantic_information(handle, tag, address_space, - buffer, size, is_mutable, !global); - if (implicit_context != NULL) - implicit_context->post_semantic_attach(); - } - - //-------------------------------------------------------------------------- + // Easy case if the user asks for no IDs + if (cnt == 0) + return LEGION_AUTO_GENERATE_ID; + const std::string library_name(name); + // Take the lock in read only mode and see if we can find the result + RtEvent wait_on; + { + AutoLock l_lock(library_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + library_sharding_ids.find(library_name); + if (finder != library_sharding_ids.end()) + { + // First do a check to see if the counts match + if (finder->second.count != cnt) + REPORT_LEGION_ERROR(ERROR_LIBRARY_COUNT_MISMATCH, + "ShardingID generation counts %zd and %zd differ for library %s", + finder->second.count, cnt, name) + if (finder->second.result_set) + return finder->second.result; + // This should never happen unless we are on a node other than 0 +#ifdef DEBUG_LEGION + assert(address_space > 0); +#endif + wait_on = finder->second.ready; + } + } + RtUserEvent request_event; + if (!wait_on.exists()) + { + AutoLock l_lock(library_lock); + // Check to make sure we didn't lose the race + std::map::const_iterator finder = + library_sharding_ids.find(library_name); + if (finder != library_sharding_ids.end()) + { + // First do a check to see if the counts match + if (finder->second.count != cnt) + REPORT_LEGION_ERROR(ERROR_LIBRARY_COUNT_MISMATCH, + "ShardingID generation counts %zd and %zd differ for library %s", + finder->second.count, cnt, name) + if (finder->second.result_set) + return finder->second.result; + // This should never happen unless we are on a node other than 0 +#ifdef DEBUG_LEGION + assert(address_space > 0); +#endif + wait_on = finder->second.ready; + } + if (!wait_on.exists()) + { + LibraryShardingIDs &record = library_sharding_ids[library_name]; + record.count = cnt; + if (address_space == 0) + { + // We're going to make the result + record.result = unique_library_sharding_id; + unique_library_sharding_id += cnt; +#ifdef DEBUG_LEGION + assert(unique_library_sharding_id > record.result); +#endif + record.result_set = true; + return record.result; + } + else + { + // We're going to request the result + request_event = Runtime::create_rt_user_event(); + record.ready = request_event; + record.result_set = false; + wait_on = request_event; + } + } + } + // Should only get here on nodes other than 0 +#ifdef DEBUG_LEGION + assert(address_space > 0); + assert(wait_on.exists()); +#endif + if (request_event.exists()) + { + // Include the null terminator in length + const size_t string_length = strlen(name) + 1; + // Send the request to node 0 for the result + Serializer rez; + { + RezCheck z(rez); + rez.serialize(string_length); + rez.serialize(name, string_length); + rez.serialize(cnt); + rez.serialize(request_event); + } + send_library_sharding_request(0/*target*/, rez); + } + wait_on.wait(); + // When we wake up we should be able to find the result + AutoLock l_lock(library_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + library_sharding_ids.find(library_name); +#ifdef DEBUG_LEGION + assert(finder != library_sharding_ids.end()); + assert(finder->second.result_set); +#endif + return finder->second.result; + } + + //-------------------------------------------------------------------------- + /*static*/ ShardingID& Runtime::get_current_static_sharding_id(void) + //-------------------------------------------------------------------------- + { + static ShardingID current_sharding_id = + LEGION_MAX_APPLICATION_SHARDING_ID; + return current_sharding_id; + } + + //-------------------------------------------------------------------------- + /*static*/ ShardingID Runtime::generate_static_sharding_id(void) + //-------------------------------------------------------------------------- + { + ShardingID &next_sharding = get_current_static_sharding_id(); + if (runtime_started) + REPORT_LEGION_ERROR(ERROR_STATIC_CALL_POST_RUNTIME_START, + "Illegal call to 'generate_static_sharding_id' after " + "the runtime has been started!") + return next_sharding++; + } + + //-------------------------------------------------------------------------- + void Runtime::register_sharding_functor(ShardingID sid, + ShardingFunctor *functor, + bool need_zero_check, + bool silence_warnings, + const char *warning_string, + bool preregistered) + //-------------------------------------------------------------------------- + { + if (sid == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_RESERVED_SHARDING_ID, + "ERROR: %d (UINT_MAX) is a reserved sharding ID.", UINT_MAX) + else if (need_zero_check && (sid == 0)) + REPORT_LEGION_ERROR(ERROR_RESERVED_SHARDING_ID, + "ERROR: ShardingID zero is reserved.") + if (!preregistered && !inside_registration_callback && !silence_warnings) + REPORT_LEGION_WARNING(LEGION_WARNING_NON_CALLBACK_REGISTRATION, + "Sharding functor %d was dynamically registered outside of a " + "registration callback invocation. In the near future this will " + "become an error in order to support task subprocesses. Please " + "use 'perform_registration_callback' to generate a callback where " + "it will be safe to perform dynamic registrations.", sid) + if (!silence_warnings && (total_address_spaces > 1) && + (inside_registration_callback != GLOBAL_REGISTRATION_CALLBACK)) + REPORT_LEGION_WARNING(LEGION_WARNING_DYNAMIC_SHARDING_REG, + "WARNING: Sharding functor %d is being dynamically " + "registered for a multi-node run with %d nodes. It is " + "currently the responsibility of the application to " + "ensure that this sharding functor is registered on " + "all nodes where it will be required. " + "Warning string: %s", sid, total_address_spaces, + (warning_string == NULL) ? "" : warning_string) + AutoLock s_lock(sharding_lock); + std::map::const_iterator finder = + sharding_functors.find(sid); + if (finder != sharding_functors.end()) + REPORT_LEGION_ERROR(ERROR_DUPLICATE_SHARDING_ID, + "ERROR: ShardingID %d has already been used by another " + "sharding functor.", sid) + sharding_functors[sid] = functor; + } + + //-------------------------------------------------------------------------- + /*static*/ void Runtime::preregister_sharding_functor(ShardingID sid, + ShardingFunctor *functor) + //-------------------------------------------------------------------------- + { + if (runtime_started) + REPORT_LEGION_ERROR(ERROR_STATIC_CALL_POST_RUNTIME_START, + "Illegal call to 'preregister_sharding_functor' after " + "the runtime has started!"); + if (sid == UINT_MAX) + REPORT_LEGION_ERROR(ERROR_RESERVED_SHARDING_ID, + "ERROR: %d (UINT_MAX) is a reserved sharding ID.", UINT_MAX) + else if (sid == 0) + REPORT_LEGION_ERROR(ERROR_RESERVED_SHARDING_ID, + "ERROR: ShardingID zero is reserved.") + std::map &pending_sharding_functors = + get_pending_sharding_table(); + std::map::const_iterator finder = + pending_sharding_functors.find(sid); + if (finder != pending_sharding_functors.end()) + REPORT_LEGION_ERROR(ERROR_DUPLICATE_SHARDING_ID, + "ERROR: ShardingID %d has already been used by another " + "sharding functor.", sid) + pending_sharding_functors[sid] = functor; + } + + //-------------------------------------------------------------------------- + ShardingFunctor* Runtime::find_sharding_functor(ShardingID sid, + bool can_fail) + //-------------------------------------------------------------------------- + { + AutoLock s_lock(sharding_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + sharding_functors.find(sid); + if (finder == sharding_functors.end()) + { + if (can_fail) + return NULL; + REPORT_LEGION_ERROR(ERROR_INVALID_SHARDING_ID, + "Unable to find registered sharding functor ID %d.", sid) + } + return finder->second; + } + + //-------------------------------------------------------------------------- + /*static*/ ShardingFunctor* Runtime::get_sharding_functor(ShardingID sid) + //-------------------------------------------------------------------------- + { + if (!runtime_started) + { + std::map &pending_sharding_functors = + get_pending_sharding_table(); + std::map::const_iterator finder = + pending_sharding_functors.find(sid); + if (finder == pending_sharding_functors.end()) + return NULL; + else + return finder->second; + } + else + return the_runtime->find_sharding_functor(sid, true/*can fail*/); + } + + //-------------------------------------------------------------------------- + void Runtime::attach_semantic_information(TaskID task_id, SemanticTag tag, + const void *buffer, size_t size, bool is_mutable, bool send_to_owner) + //-------------------------------------------------------------------------- + { + if ((implicit_context != NULL) && + !implicit_context->perform_semantic_attach(send_to_owner)) + return; + if ((tag == LEGION_NAME_SEMANTIC_TAG) && legion_spy_enabled) + LegionSpy::log_task_name(task_id, static_cast(buffer)); + TaskImpl *impl = find_or_create_task_impl(task_id); + impl->attach_semantic_information(tag, address_space, buffer, size, + is_mutable, send_to_owner); + if (implicit_context != NULL) + implicit_context->post_semantic_attach(); + } + + //-------------------------------------------------------------------------- + void Runtime::attach_semantic_information(IndexSpace handle, + SemanticTag tag, + const void *buffer, size_t size, + bool is_mutable) + //-------------------------------------------------------------------------- + { + bool global = true; + if ((implicit_context != NULL) && + !implicit_context->perform_semantic_attach(global)) + return; + forest->attach_semantic_information(handle, tag, address_space, + buffer, size, is_mutable, !global); + if (implicit_context != NULL) + implicit_context->post_semantic_attach(); + } + + //-------------------------------------------------------------------------- + void Runtime::attach_semantic_information(IndexPartition handle, + SemanticTag tag, + const void *buffer, size_t size, + bool is_mutable) + //-------------------------------------------------------------------------- + { + bool global = true; + if ((implicit_context != NULL) && + !implicit_context->perform_semantic_attach(global)) + return; + forest->attach_semantic_information(handle, tag, address_space, + buffer, size, is_mutable, !global); + if (implicit_context != NULL) + implicit_context->post_semantic_attach(); + } + + //-------------------------------------------------------------------------- void Runtime::attach_semantic_information(FieldSpace handle, SemanticTag tag, const void *buffer, size_t size, @@ -15624,7 +17637,7 @@ namespace Legion { else init = source.address_space(); // The runtime stride is the same as the number of nodes - const int total_nodes = runtime_stride; + const int total_nodes = total_address_spaces; for (int r = 1; r <= radix; r++) { int offset = base + r; @@ -16500,7 +18513,7 @@ namespace Legion { //-------------------------------------------------------------------------- { find_messenger(target)->send_message(rez, SEND_MATERIALIZED_VIEW, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- @@ -16508,7 +18521,7 @@ namespace Legion { //-------------------------------------------------------------------------- { find_messenger(target)->send_message(rez, SEND_FILL_VIEW, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- @@ -16516,7 +18529,15 @@ namespace Legion { //-------------------------------------------------------------------------- { find_messenger(target)->send_message(rez, SEND_PHI_VIEW, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_sharded_view(AddressSpaceID target, Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_SHARDED_VIEW, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- @@ -16524,7 +18545,7 @@ namespace Legion { //-------------------------------------------------------------------------- { find_messenger(target)->send_message(rez, SEND_REDUCTION_VIEW, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- @@ -16532,7 +18553,7 @@ namespace Legion { //-------------------------------------------------------------------------- { find_messenger(target)->send_message(rez, SEND_INSTANCE_MANAGER, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- @@ -16541,7 +18562,7 @@ namespace Legion { //-------------------------------------------------------------------------- { find_messenger(target)->send_message(rez, SEND_COLLECTIVE_MANAGER, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- @@ -16717,85 +18738,206 @@ namespace Legion { } //-------------------------------------------------------------------------- - void Runtime::send_mapper_message(AddressSpaceID target, Serializer &rez) + void Runtime::send_control_replicate_future_map_request( + AddressSpaceID target, Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, SEND_MAPPER_MESSAGE, - MAPPER_VIRTUAL_CHANNEL, true/*flush*/); + find_messenger(target)->send_message(rez, SEND_REPL_FUTURE_MAP_REQUEST, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); } //-------------------------------------------------------------------------- - void Runtime::send_mapper_broadcast(AddressSpaceID target, Serializer &rez) + void Runtime::send_control_replicate_future_map_response( + AddressSpaceID target, Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, SEND_MAPPER_BROADCAST, - MAPPER_VIRTUAL_CHANNEL, true/*flush*/); + find_messenger(target)->send_message(rez, SEND_REPL_FUTURE_MAP_RESPONSE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- - void Runtime::send_task_impl_semantic_request(AddressSpaceID target, - Serializer &rez) + void Runtime::send_control_replicate_top_view_request(AddressSpaceID target, + Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, SEND_TASK_IMPL_SEMANTIC_REQ, + find_messenger(target)->send_message(rez, SEND_REPL_TOP_VIEW_REQUEST, DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } //-------------------------------------------------------------------------- - void Runtime::send_index_space_semantic_request(AddressSpaceID target, - Serializer &rez) + void Runtime::send_control_replicate_top_view_response( + AddressSpaceID target, Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, SEND_INDEX_SPACE_SEMANTIC_REQ, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + find_messenger(target)->send_message(rez, SEND_REPL_TOP_VIEW_RESPONSE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- - void Runtime::send_index_partition_semantic_request(AddressSpaceID target, - Serializer &rez) + void Runtime::send_control_replicate_equivalence_set_request( + AddressSpaceID target, Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, - SEND_INDEX_PARTITION_SEMANTIC_REQ, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + find_messenger(target)->send_message(rez, SEND_REPL_EQ_REQUEST, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); } //-------------------------------------------------------------------------- - void Runtime::send_field_space_semantic_request(AddressSpaceID target, - Serializer &rez) + void Runtime::send_control_replicate_equivalence_set_response( + AddressSpaceID target, Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, SEND_FIELD_SPACE_SEMANTIC_REQ, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + find_messenger(target)->send_message(rez, SEND_REPL_EQ_RESPONSE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } //-------------------------------------------------------------------------- - void Runtime::send_field_semantic_request(AddressSpaceID target, - Serializer &rez) + void Runtime::send_control_replicate_intra_space_dependence( + AddressSpaceID target, Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, SEND_FIELD_SEMANTIC_REQ, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + find_messenger(target)->send_message(rez, SEND_REPL_INTRA_SPACE_DEP, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); } //-------------------------------------------------------------------------- - void Runtime::send_logical_region_semantic_request(AddressSpaceID target, - Serializer &rez) + void Runtime::send_control_replicate_resource_update(AddressSpaceID target, + Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, - SEND_LOGICAL_REGION_SEMANTIC_REQ, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + find_messenger(target)->send_message(rez, SEND_REPL_RESOURCE_UPDATE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); } //-------------------------------------------------------------------------- - void Runtime::send_logical_partition_semantic_request( - AddressSpaceID target, Serializer &rez) + void Runtime::send_control_replicate_trace_event_request( + AddressSpaceID target, Serializer &rez) //-------------------------------------------------------------------------- { - find_messenger(target)->send_message(rez, - SEND_LOGICAL_PARTITION_SEMANTIC_REQ, - DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + find_messenger(target)->send_message(rez, SEND_REPL_TRACE_EVENT_REQUEST, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_control_replicate_trace_event_response( + AddressSpaceID target, Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_REPL_TRACE_EVENT_RESPONSE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_control_replicate_trace_update(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_REPL_TRACE_UPDATE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_control_replicate_implicit_request(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_REPL_IMPLICIT_REQUEST, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_control_replicate_implicit_response( + AddressSpaceID target, Serializer &rez) + //-------------------------------------------------------------------------- + { + // This has to go on the task virtual channel so that it is ordered + // with respect to any distributions + // See Runtime::send_replicate_launch + find_messenger(target)->send_message(rez, SEND_REPL_IMPLICIT_RESPONSE, + TASK_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_mapper_message(AddressSpaceID target, Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_MAPPER_MESSAGE, + MAPPER_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_mapper_broadcast(AddressSpaceID target, Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_MAPPER_BROADCAST, + MAPPER_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_task_impl_semantic_request(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_TASK_IMPL_SEMANTIC_REQ, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_index_space_semantic_request(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_INDEX_SPACE_SEMANTIC_REQ, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_index_partition_semantic_request(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, + SEND_INDEX_PARTITION_SEMANTIC_REQ, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_field_space_semantic_request(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_FIELD_SPACE_SEMANTIC_REQ, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_field_semantic_request(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_FIELD_SEMANTIC_REQ, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_logical_region_semantic_request(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, + SEND_LOGICAL_REGION_SEMANTIC_REQ, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_logical_partition_semantic_request( + AddressSpaceID target, Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, + SEND_LOGICAL_PARTITION_SEMANTIC_REQ, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); } //-------------------------------------------------------------------------- @@ -17215,6 +19357,7 @@ namespace Legion { Serializer &rez) //-------------------------------------------------------------------------- { + // This is paging in constraints so it needs its own virtual channel find_messenger(target)->send_message(rez, SEND_CONSTRAINT_REQUEST, LAYOUT_CONSTRAINT_VIRTUAL_CHANNEL, true/*flush*/); } @@ -17235,7 +19378,7 @@ namespace Legion { //-------------------------------------------------------------------------- { find_messenger(target)->send_message(rez, SEND_CONSTRAINT_RELEASE, - LAYOUT_CONSTRAINT_VIRTUAL_CHANNEL, true/*flush*/); + LAYOUT_CONSTRAINT_VIRTUAL_CHANNEL, true/*flush*/); } //-------------------------------------------------------------------------- @@ -17246,6 +19389,72 @@ namespace Legion { DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); } + //-------------------------------------------------------------------------- + void Runtime::send_replicate_launch(AddressSpaceID target,Serializer &rez) + //-------------------------------------------------------------------------- + { + // Put this on the task virtual channel so it can be ordered with + // respect to requests for shard managers in implicit cases. + // See ImplicitShardManager::create_shard_manager + // See Runtime::send_control_replicate_implicit_response + find_messenger(target)->send_message(rez, SEND_REPLICATE_LAUNCH, + TASK_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_replicate_delete(AddressSpaceID target,Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_REPLICATE_DELETE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_replicate_post_mapped(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_REPLICATE_POST_MAPPED, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_replicate_post_execution(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_REPLICATE_POST_EXECUTION, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_replicate_trigger_complete(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_REPLICATE_TRIGGER_COMPLETE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_replicate_trigger_commit(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_REPLICATE_TRIGGER_COMMIT, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_control_replicate_collective_message( + AddressSpaceID target, Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, + SEND_CONTROL_REPLICATE_COLLECTIVE_MESSAGE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + //-------------------------------------------------------------------------- void Runtime::send_library_mapper_request(AddressSpaceID target, Serializer &rez) @@ -17300,6 +19509,24 @@ namespace Legion { DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); } + //-------------------------------------------------------------------------- + void Runtime::send_library_sharding_request(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_LIBRARY_SHARDING_REQUEST, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/); + } + + //-------------------------------------------------------------------------- + void Runtime::send_library_sharding_response(AddressSpaceID target, + Serializer &rez) + //-------------------------------------------------------------------------- + { + find_messenger(target)->send_message(rez, SEND_LIBRARY_SHARDING_RESPONSE, + DEFAULT_VIRTUAL_CHANNEL, true/*flush*/, true/*response*/); + } + //-------------------------------------------------------------------------- void Runtime::send_library_task_request(AddressSpaceID target, Serializer &rez) @@ -18199,6 +20426,14 @@ namespace Legion { PhiView::handle_send_phi_view(this, derez, source); } + //-------------------------------------------------------------------------- + void Runtime::handle_send_sharded_view(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + ShardedView::handle_send_sharded_view(this, derez, source); + } + //-------------------------------------------------------------------------- void Runtime::handle_send_reduction_view(Deserializer &derez, AddressSpaceID source) @@ -18271,83 +20506,82 @@ namespace Legion { } //-------------------------------------------------------------------------- - void Runtime::handle_view_register_user(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_manager_request(Deserializer &derez, + AddressSpaceID source) //-------------------------------------------------------------------------- { - InstanceView::handle_view_register_user(derez, this, source); + PhysicalManager::handle_manager_request(derez, this, source); } +#ifdef ENABLE_VIEW_REPLICATION //-------------------------------------------------------------------------- - void Runtime::handle_view_copy_pre_request(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_view_replication_request(Deserializer &derez, + AddressSpaceID source) //-------------------------------------------------------------------------- { - InstanceView::handle_view_find_copy_pre_request(derez, this, source); + InstanceView::handle_view_replication_request(derez, this, source); } - + //-------------------------------------------------------------------------- - void Runtime::handle_view_copy_pre_response(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_view_replication_response(Deserializer &derez) //-------------------------------------------------------------------------- { - InstanceView::handle_view_find_copy_pre_response(derez, this, source); + InstanceView::handle_view_replication_response(derez, this); } //-------------------------------------------------------------------------- - void Runtime::handle_view_add_copy_user(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_view_replication_removal(Deserializer &derez, + AddressSpaceID source) //-------------------------------------------------------------------------- { - InstanceView::handle_view_add_copy_user(derez, this, source); + InstanceView::handle_view_replication_removal(derez, this, source); } +#endif // ENABLE_VIEW_REPLICATION -#ifdef ENABLE_VIEW_REPLICATION //-------------------------------------------------------------------------- - void Runtime::handle_view_replication_request(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_future_result(Deserializer &derez) //-------------------------------------------------------------------------- { - InstanceView::handle_view_replication_request(derez, this, source); + FutureImpl::handle_future_result(derez, this); } - + //-------------------------------------------------------------------------- - void Runtime::handle_view_replication_response(Deserializer &derez) + void Runtime::handle_future_subscription(Deserializer &derez, + AddressSpaceID source) //-------------------------------------------------------------------------- { - InstanceView::handle_view_replication_response(derez, this); + FutureImpl::handle_future_subscription(derez, this, source); } //-------------------------------------------------------------------------- - void Runtime::handle_view_replication_removal(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_future_map_future_request(Deserializer &derez, + AddressSpaceID source) //-------------------------------------------------------------------------- { - InstanceView::handle_view_replication_removal(derez, this, source); + FutureMapImpl::handle_future_map_future_request(derez, this, source); } -#endif // ENABLE_VIEW_REPLICATION //-------------------------------------------------------------------------- - void Runtime::handle_manager_request(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_future_map_future_response(Deserializer &derez) //-------------------------------------------------------------------------- { - PhysicalManager::handle_manager_request(derez, this, source); + FutureMapImpl::handle_future_map_future_response(derez, this); } //-------------------------------------------------------------------------- - void Runtime::handle_future_result(Deserializer &derez) + void Runtime::handle_control_replicate_future_map_request( + Deserializer &derez) //-------------------------------------------------------------------------- { - FutureImpl::handle_future_result(derez, this); + ShardManager::handle_future_map_request(derez, this); } //-------------------------------------------------------------------------- - void Runtime::handle_future_subscription(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_control_replicate_future_map_response( + Deserializer &derez) //-------------------------------------------------------------------------- { - FutureImpl::handle_future_subscription(derez, this, source); + ReplFutureMapImpl::handle_future_map_response(derez, this); } //-------------------------------------------------------------------------- @@ -18366,38 +20600,140 @@ namespace Legion { } //-------------------------------------------------------------------------- - void Runtime::handle_future_map_future_request(Deserializer &derez, - AddressSpaceID source) + void Runtime::handle_control_replicate_top_view_request(Deserializer &derez, + AddressSpaceID source) //-------------------------------------------------------------------------- { - FutureMapImpl::handle_future_map_future_request(derez, this, source); + ShardManager::handle_top_view_request(derez, this, source); } //-------------------------------------------------------------------------- - void Runtime::handle_future_map_future_response(Deserializer &derez) + void Runtime::handle_control_replicate_top_view_response( + Deserializer &derez) //-------------------------------------------------------------------------- { - FutureMapImpl::handle_future_map_future_response(derez, this); + ShardManager::handle_top_view_response(derez, this); } //-------------------------------------------------------------------------- - void Runtime::handle_mapper_message(Deserializer &derez) + void Runtime::handle_control_replicate_eq_request(Deserializer &derez) //-------------------------------------------------------------------------- { - DerezCheck z(derez); - Processor target; - derez.deserialize(target); - MapperID map_id; - derez.deserialize(map_id); - Processor source; - derez.deserialize(source); - unsigned message_kind; - derez.deserialize(message_kind); - size_t message_size; - derez.deserialize(message_size); - const void *message = derez.get_current_pointer(); - derez.advance_pointer(message_size); - process_mapper_message(target, map_id, source, message, + ShardManager::handle_eq_request(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_eq_response(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ReplicateContext::handle_eq_response(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_intra_space_dependence( + Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_intra_space_dependence(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_resource_update(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_resource_update(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_trace_event_request( + Deserializer &derez, AddressSpaceID source) + //-------------------------------------------------------------------------- + { + ShardManager::handle_trace_event_request(derez, this, source); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_trace_event_response( + Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_trace_event_response(derez); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_trace_update(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + ShardManager::handle_trace_update(derez, this, source); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_implicit_request(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + ImplicitShardManager::handle_remote_request(derez, this, source); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_implicit_response( + Deserializer &derez) + //-------------------------------------------------------------------------- + { + ImplicitShardManager::handle_remote_response(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_view_register_user(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + InstanceView::handle_view_register_user(derez, this, source); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_view_copy_pre_request(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + InstanceView::handle_view_find_copy_pre_request(derez, this, source); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_view_copy_pre_response(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + InstanceView::handle_view_find_copy_pre_response(derez, this, source); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_view_add_copy_user(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + InstanceView::handle_view_add_copy_user(derez, this, source); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_mapper_message(Deserializer &derez) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + Processor target; + derez.deserialize(target); + MapperID map_id; + derez.deserialize(map_id); + Processor source; + derez.deserialize(source); + unsigned message_kind; + derez.deserialize(message_kind); + size_t message_size; + derez.deserialize(message_size); + const void *message = derez.get_current_pointer(); + derez.advance_pointer(message_size); + process_mapper_message(target, map_id, source, message, message_size, message_kind); } @@ -18950,9 +21286,60 @@ namespace Legion { //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert(mpi_rank_table != NULL); + assert(Runtime::mpi_rank_table != NULL); #endif - mpi_rank_table->handle_mpi_rank_exchange(derez); + Runtime::mpi_rank_table->handle_mpi_rank_exchange(derez); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_replicate_launch(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + ShardManager::handle_launch(derez, this, source); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_replicate_delete(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_delete(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_replicate_post_mapped(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_post_mapped(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_replicate_post_execution(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_post_execution(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_replicate_trigger_complete(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_trigger_complete(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_replicate_trigger_commit(Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_trigger_commit(derez, this); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_control_replicate_collective_message( + Deserializer &derez) + //-------------------------------------------------------------------------- + { + ShardManager::handle_collective_message(derez, this); } //-------------------------------------------------------------------------- @@ -19126,6 +21513,63 @@ namespace Legion { Runtime::trigger_event(done); } + //-------------------------------------------------------------------------- + void Runtime::handle_library_sharding_request(Deserializer &derez, + AddressSpaceID source) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + size_t string_length; + derez.deserialize(string_length); + const char *name = (const char*)derez.get_current_pointer(); + derez.advance_pointer(string_length); + size_t count; + derez.deserialize(count); + RtUserEvent done; + derez.deserialize(done); + + ShardingID result = generate_library_sharding_ids(name, count); + Serializer rez; + { + RezCheck z2(rez); + rez.serialize(string_length); + rez.serialize(name, string_length); + rez.serialize(result); + rez.serialize(done); + } + send_library_sharding_response(source, rez); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_library_sharding_response(Deserializer &derez) + //-------------------------------------------------------------------------- + { + DerezCheck z(derez); + size_t string_length; + derez.deserialize(string_length); + const char *name = (const char*)derez.get_current_pointer(); + derez.advance_pointer(string_length); + ShardingID result; + derez.deserialize(result); + RtUserEvent done; + derez.deserialize(done); + + const std::string library_name(name); + { + AutoLock l_lock(library_lock); + std::map::iterator finder = + library_sharding_ids.find(library_name); +#ifdef DEBUG_LEGION + assert(finder != library_sharding_ids.end()); + assert(!finder->second.result_set); + assert(finder->second.ready == done); +#endif + finder->second.result = result; + finder->second.result_set = true; + } + Runtime::trigger_event(done); + } + //-------------------------------------------------------------------------- void Runtime::handle_library_task_request(Deserializer &derez, AddressSpaceID source) @@ -19297,6 +21741,20 @@ namespace Legion { Runtime::trigger_event(done); } + //-------------------------------------------------------------------------- + void Runtime::handle_remote_op_report_uninitialized(Deserializer &derez) + //-------------------------------------------------------------------------- + { + RemoteOp::handle_report_uninitialized(derez); + } + + //-------------------------------------------------------------------------- + void Runtime::handle_remote_op_profiling_count_update(Deserializer &derez) + //-------------------------------------------------------------------------- + { + RemoteOp::handle_report_profiling_count_update(derez); + } + //-------------------------------------------------------------------------- void Runtime::handle_shutdown_notification(Deserializer &derez, AddressSpaceID source) @@ -19684,7 +22142,7 @@ namespace Legion { return result; } DistributedID result = unique_distributed_id; - unique_distributed_id += runtime_stride; + unique_distributed_id += total_address_spaces; #ifdef DEBUG_LEGION assert(result < LEGION_DISTRIBUTED_ID_MASK); #endif @@ -19738,7 +22196,7 @@ namespace Legion { AddressSpaceID Runtime::determine_owner(DistributedID did) const //-------------------------------------------------------------------------- { - return ((did & LEGION_DISTRIBUTED_ID_MASK) % runtime_stride); + return ((did & LEGION_DISTRIBUTED_ID_MASK) % total_address_spaces); } //-------------------------------------------------------------------------- @@ -19762,7 +22220,8 @@ namespace Legion { if (finder != pending_collectables.end()) { #ifdef DEBUG_LEGION - assert(finder->second.first == dc); + assert((finder->second.first == dc) || + (finder->second.first == NULL)); #endif to_trigger = finder->second.second; pending_collectables.erase(finder); @@ -19798,17 +22257,11 @@ namespace Legion { DistributedID did) //-------------------------------------------------------------------------- { - const DistributedID to_find = LEGION_DISTRIBUTED_ID_FILTER(did); - AutoLock d_lock(distributed_collectable_lock,1,false/*exclusive*/); - std::map::const_iterator finder = - dist_collectables.find(to_find); -#ifdef DEBUG_LEGION - if (finder == dist_collectables.end()) - log_run.error("Unable to find distributed collectable %llx " - "with type %lld", did, LEGION_DISTRIBUTED_HELP_DECODE(did)); - assert(finder != dist_collectables.end()); -#endif - return finder->second; + RtEvent wait_on; + DistributedCollectable *dc = find_distributed_collectable(did, wait_on); + if (wait_on.exists() && !wait_on.has_triggered()) + wait_on.wait(); + return dc; } //-------------------------------------------------------------------------- @@ -19816,26 +22269,38 @@ namespace Legion { DistributedID did, RtEvent &ready) //-------------------------------------------------------------------------- { + bool found = false; const DistributedID to_find = LEGION_DISTRIBUTED_ID_FILTER(did); - AutoLock d_lock(distributed_collectable_lock,1,false/*exclusive*/); - std::map::const_iterator finder = - dist_collectables.find(to_find); - if (finder == dist_collectables.end()) { - // Check to see if it is in the pending set too - std::map >::const_iterator - pending_finder = pending_collectables.find(to_find); - if (pending_finder != pending_collectables.end()) + AutoLock d_lock(distributed_collectable_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + dist_collectables.find(to_find); + if (finder == dist_collectables.end()) { - ready = pending_finder->second.second; - return pending_finder->second.first; + // Check to see if it is in the pending set too + std::map >::const_iterator + pending_finder = pending_collectables.find(to_find); + if (pending_finder != pending_collectables.end()) + { + found = true; + ready = pending_finder->second.second; + if (pending_finder->second.first != NULL) + return pending_finder->second.first; + } } + else + return finder->second; } -#ifdef DEBUG_LEGION - if (finder == dist_collectables.end()) + if (!found) log_run.error("Unable to find distributed collectable %llx " "with type %lld", did, LEGION_DISTRIBUTED_HELP_DECODE(did)); + // Wait for it to be ready + ready.wait(); + AutoLock d_lock(distributed_collectable_lock,1,false/*exclusive*/); + std::map::const_iterator finder = + dist_collectables.find(to_find); +#ifdef DEBUG_LEGION assert(finder != dist_collectables.end()); #endif return finder->second; @@ -19875,6 +22340,40 @@ namespace Legion { return false; } + //-------------------------------------------------------------------------- + void Runtime::record_pending_distributed_collectable(DistributedID did) + //-------------------------------------------------------------------------- + { + const RtUserEvent registered = Runtime::create_rt_user_event(); + AutoLock d_lock(distributed_collectable_lock); +#ifdef DEBUG_LEGION + assert(dist_collectables.find(did) == dist_collectables.end()); + assert(pending_collectables.find(did) == pending_collectables.end()); +#endif + pending_collectables[did] = + std::pair(NULL, registered); + } + + //-------------------------------------------------------------------------- + void Runtime::revoke_pending_distributed_collectable(DistributedID did) + //-------------------------------------------------------------------------- + { + RtUserEvent to_trigger; + { + AutoLock d_lock(distributed_collectable_lock); + std::map >::iterator finder = + pending_collectables.find(did); + if (finder != pending_collectables.end()) + { + to_trigger = finder->second.second; + pending_collectables.erase(finder); + } + } + if (to_trigger.exists()) + Runtime::trigger_event(to_trigger); + } + //-------------------------------------------------------------------------- LogicalView* Runtime::find_or_request_logical_view(DistributedID did, RtEvent &ready) @@ -19890,6 +22389,12 @@ namespace Legion { else if (LogicalView::is_fill_did(did)) dc = find_or_request_distributed_collectable< FillView, SEND_VIEW_REQUEST, DEFAULT_VIRTUAL_CHANNEL>(did, ready); + else if (LogicalView::is_phi_did(did)) + dc = find_or_request_distributed_collectable< + PhiView, SEND_VIEW_REQUEST, DEFAULT_VIRTUAL_CHANNEL>(did, ready); + else if (LogicalView::is_sharded_did(did)) + dc = find_or_request_distributed_collectable< + ShardedView, SEND_VIEW_REQUEST, DEFAULT_VIRTUAL_CHANNEL>(did, ready); else assert(false); // Have to static cast since the memory might not have been initialized @@ -19981,7 +22486,12 @@ namespace Legion { //-------------------------------------------------------------------------- FutureImpl* Runtime::find_or_create_future(DistributedID did, - ReferenceMutator *mutator) + ReferenceMutator *mutator, + Operation *op, GenerationID gen, +#ifdef LEGION_SPY + UniqueID op_uid, +#endif + int op_depth) //-------------------------------------------------------------------------- { did &= LEGION_DISTRIBUTED_ID_MASK; @@ -20004,8 +22514,15 @@ namespace Legion { #ifdef DEBUG_LEGION assert(owner_space != address_space); #endif - FutureImpl *result = new FutureImpl(this, false/*register*/, did, - owner_space, ApEvent::NO_AP_EVENT); + FutureImpl *result = (op == NULL) ? + new FutureImpl(this, false/*register*/, did, owner_space, + ApEvent::NO_AP_EVENT) : + new FutureImpl(this, false/*register*/, did, owner_space, + ApEvent::NO_AP_EVENT, op, gen, +#ifdef LEGION_SPY + op_uid, +#endif + op_depth); // Retake the lock and see if we lost the race { AutoLock d_lock(distributed_collectable_lock); @@ -20033,7 +22550,8 @@ namespace Legion { //-------------------------------------------------------------------------- FutureMapImpl* Runtime::find_or_create_future_map(DistributedID did, - TaskContext *ctx, RtEvent complete, ReferenceMutator *mutator) + TaskContext *ctx, const Domain &domain, + RtEvent complete, ReferenceMutator *mutator) //-------------------------------------------------------------------------- { did &= LEGION_DISTRIBUTED_ID_MASK; @@ -20052,12 +22570,32 @@ namespace Legion { return result; } } + // Check to see if we need to prefetch sparsity map data to this node + if (!domain.dense()) + { + switch (domain.get_dim()) + { +#define DIMFUNC(DIM) \ + case DIM: \ + { \ + const DomainT domaint = domain; \ + const RtEvent wait_on(domaint.make_valid()); \ + if (wait_on.exists() && !wait_on.has_triggered()) \ + wait_on.wait(); \ + break; \ + } + LEGION_FOREACH_N(DIMFUNC) +#undef DIMFUNC + default: + assert(false); + } + } const AddressSpaceID owner_space = determine_owner(did); #ifdef DEBUG_LEGION assert(owner_space != address_space); #endif - FutureMapImpl *result = new FutureMapImpl(ctx, this, did, owner_space, - complete, false/*register now */); + FutureMapImpl *result = new FutureMapImpl(ctx, this, domain, did, + owner_space, complete, false/*register now */); // Retake the lock and see if we lost the race { AutoLock d_lock(distributed_collectable_lock); @@ -20699,6 +23237,165 @@ namespace Legion { return get_available(all_reduce_op_lock, available_all_reduce_ops); } + //-------------------------------------------------------------------------- + ReplIndividualTask* Runtime::get_available_repl_individual_task(void) + //-------------------------------------------------------------------------- + { + return get_available(individual_task_lock, + available_repl_individual_tasks); + } + + //-------------------------------------------------------------------------- + ReplIndexTask* Runtime::get_available_repl_index_task(void) + //-------------------------------------------------------------------------- + { + return get_available(index_task_lock, available_repl_index_tasks); + } + + //-------------------------------------------------------------------------- + ReplMergeCloseOp* Runtime::get_available_repl_merge_close_op(void) + //-------------------------------------------------------------------------- + { + return get_available(merge_close_op_lock, available_repl_merge_close_ops); + } + + //-------------------------------------------------------------------------- + ReplFillOp* Runtime::get_available_repl_fill_op(void) + //-------------------------------------------------------------------------- + { + return get_available(fill_op_lock, available_repl_fill_ops); + } + + //-------------------------------------------------------------------------- + ReplIndexFillOp* Runtime::get_available_repl_index_fill_op(void) + //-------------------------------------------------------------------------- + { + return get_available(fill_op_lock, available_repl_index_fill_ops); + } + + //-------------------------------------------------------------------------- + ReplCopyOp* Runtime::get_available_repl_copy_op(void) + //-------------------------------------------------------------------------- + { + return get_available(copy_op_lock, available_repl_copy_ops); + } + + //-------------------------------------------------------------------------- + ReplIndexCopyOp* Runtime::get_available_repl_index_copy_op(void) + //-------------------------------------------------------------------------- + { + return get_available(copy_op_lock, available_repl_index_copy_ops); + } + + //-------------------------------------------------------------------------- + ReplDeletionOp* Runtime::get_available_repl_deletion_op(void) + //-------------------------------------------------------------------------- + { + return get_available(deletion_op_lock, available_repl_deletion_ops); + } + + //-------------------------------------------------------------------------- + ReplPendingPartitionOp* + Runtime::get_available_repl_pending_partition_op(void) + //-------------------------------------------------------------------------- + { + return get_available(pending_partition_op_lock, + available_repl_pending_partition_ops); + } + + //-------------------------------------------------------------------------- + ReplDependentPartitionOp* + Runtime::get_available_repl_dependent_partition_op(void) + //-------------------------------------------------------------------------- + { + return get_available(dependent_partition_op_lock, + available_repl_dependent_partition_ops); + } + + //-------------------------------------------------------------------------- + ReplMustEpochOp* Runtime::get_available_repl_epoch_op(void) + //-------------------------------------------------------------------------- + { + return get_available(epoch_op_lock, available_repl_must_epoch_ops); + } + + //-------------------------------------------------------------------------- + ReplTimingOp* Runtime::get_available_repl_timing_op(void) + //-------------------------------------------------------------------------- + { + return get_available(timing_op_lock, available_repl_timing_ops); + } + + //-------------------------------------------------------------------------- + ReplAllReduceOp* Runtime::get_available_repl_all_reduce_op(void) + //-------------------------------------------------------------------------- + { + return get_available(all_reduce_op_lock, available_repl_all_reduce_ops); + } + + //-------------------------------------------------------------------------- + ReplFenceOp* Runtime::get_available_repl_fence_op(void) + //-------------------------------------------------------------------------- + { + return get_available(fence_op_lock, available_repl_fence_ops); + } + + //-------------------------------------------------------------------------- + ReplMapOp* Runtime::get_available_repl_map_op(void) + //-------------------------------------------------------------------------- + { + return get_available(map_op_lock, available_repl_map_ops); + } + + //-------------------------------------------------------------------------- + ReplAttachOp* Runtime::get_available_repl_attach_op(void) + //-------------------------------------------------------------------------- + { + return get_available(attach_op_lock, available_repl_attach_ops); + } + + //-------------------------------------------------------------------------- + ReplDetachOp* Runtime::get_available_repl_detach_op(void) + //-------------------------------------------------------------------------- + { + return get_available(detach_op_lock, available_repl_detach_ops); + } + + //-------------------------------------------------------------------------- + ReplTraceCaptureOp* Runtime::get_available_repl_capture_op(void) + //-------------------------------------------------------------------------- + { + return get_available(capture_op_lock, available_repl_capture_ops); + } + + //-------------------------------------------------------------------------- + ReplTraceCompleteOp* Runtime::get_available_repl_trace_op(void) + //-------------------------------------------------------------------------- + { + return get_available(trace_op_lock, available_repl_trace_ops); + } + + //-------------------------------------------------------------------------- + ReplTraceReplayOp* Runtime::get_available_repl_replay_op(void) + //-------------------------------------------------------------------------- + { + return get_available(replay_op_lock, available_repl_replay_ops); + } + + //-------------------------------------------------------------------------- + ReplTraceBeginOp* Runtime::get_available_repl_begin_op(void) + //-------------------------------------------------------------------------- + { + return get_available(begin_op_lock, available_repl_begin_ops); + } + + //-------------------------------------------------------------------------- + ReplTraceSummaryOp* Runtime::get_available_repl_summary_op(void) + //-------------------------------------------------------------------------- + { + return get_available(summary_op_lock, available_repl_summary_ops); + } + //-------------------------------------------------------------------------- void Runtime::free_individual_task(IndividualTask *task) //-------------------------------------------------------------------------- @@ -20880,139 +23577,315 @@ namespace Legion { } //-------------------------------------------------------------------------- - void Runtime::free_acquire_op(AcquireOp *op) + void Runtime::free_acquire_op(AcquireOp *op) + //-------------------------------------------------------------------------- + { + AutoLock a_lock(acquire_op_lock); + release_operation(available_acquire_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_release_op(ReleaseOp *op) + //-------------------------------------------------------------------------- + { + AutoLock r_lock(release_op_lock); + release_operation(available_release_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_capture_op(TraceCaptureOp *op) + //-------------------------------------------------------------------------- + { + AutoLock c_lock(capture_op_lock); + release_operation(available_capture_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_trace_op(TraceCompleteOp *op) + //-------------------------------------------------------------------------- + { + AutoLock t_lock(trace_op_lock); + release_operation(available_trace_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_replay_op(TraceReplayOp *op) + //-------------------------------------------------------------------------- + { + AutoLock t_lock(replay_op_lock); + release_operation(available_replay_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_begin_op(TraceBeginOp *op) + //-------------------------------------------------------------------------- + { + AutoLock t_lock(begin_op_lock); + release_operation(available_begin_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_summary_op(TraceSummaryOp *op) + //-------------------------------------------------------------------------- + { + AutoLock t_lock(summary_op_lock); + release_operation(available_summary_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_epoch_op(MustEpochOp *op) + //-------------------------------------------------------------------------- + { + AutoLock e_lock(epoch_op_lock); + release_operation(available_epoch_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_pending_partition_op(PendingPartitionOp *op) + //-------------------------------------------------------------------------- + { + AutoLock p_lock(pending_partition_op_lock); + release_operation(available_pending_partition_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_dependent_partition_op(DependentPartitionOp *op) + //-------------------------------------------------------------------------- + { + AutoLock p_lock(dependent_partition_op_lock); + release_operation(available_dependent_partition_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_point_dep_part_op(PointDepPartOp *op) + //-------------------------------------------------------------------------- + { + AutoLock p_lock(dependent_partition_op_lock); + release_operation(available_point_dep_part_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_fill_op(FillOp *op) + //-------------------------------------------------------------------------- + { + AutoLock f_lock(fill_op_lock); + release_operation(available_fill_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_index_fill_op(IndexFillOp *op) + //-------------------------------------------------------------------------- + { + AutoLock f_lock(fill_op_lock); + release_operation(available_index_fill_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_point_fill_op(PointFillOp *op) + //-------------------------------------------------------------------------- + { + AutoLock f_lock(fill_op_lock); + release_operation(available_point_fill_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_attach_op(AttachOp *op) + //-------------------------------------------------------------------------- + { + AutoLock a_lock(attach_op_lock); + release_operation(available_attach_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_detach_op(DetachOp *op) + //-------------------------------------------------------------------------- + { + AutoLock d_lock(detach_op_lock); + release_operation(available_detach_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_timing_op(TimingOp *op) + //-------------------------------------------------------------------------- + { + AutoLock t_lock(timing_op_lock); + release_operation(available_timing_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_repl_individual_task(ReplIndividualTask *task) + //-------------------------------------------------------------------------- + { + AutoLock i_lock(individual_task_lock); + release_operation(available_repl_individual_tasks, task); + } + + //-------------------------------------------------------------------------- + void Runtime::free_repl_index_task(ReplIndexTask *task) + //-------------------------------------------------------------------------- + { + AutoLock i_lock(index_task_lock); + release_operation(available_repl_index_tasks, task); + } + + //-------------------------------------------------------------------------- + void Runtime::free_repl_merge_close_op(ReplMergeCloseOp *op) + //-------------------------------------------------------------------------- + { + AutoLock m_lock(merge_close_op_lock); + release_operation(available_repl_merge_close_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_repl_fill_op(ReplFillOp *op) + //-------------------------------------------------------------------------- + { + AutoLock f_lock(fill_op_lock); + release_operation(available_repl_fill_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_repl_index_fill_op(ReplIndexFillOp *op) + //-------------------------------------------------------------------------- + { + AutoLock f_lock(fill_op_lock); + release_operation(available_repl_index_fill_ops, op); + } + + //-------------------------------------------------------------------------- + void Runtime::free_repl_copy_op(ReplCopyOp *op) //-------------------------------------------------------------------------- { - AutoLock a_lock(acquire_op_lock); - release_operation(available_acquire_ops, op); + AutoLock c_lock(copy_op_lock); + release_operation(available_repl_copy_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_release_op(ReleaseOp *op) + void Runtime::free_repl_index_copy_op(ReplIndexCopyOp *op) //-------------------------------------------------------------------------- { - AutoLock r_lock(release_op_lock); - release_operation(available_release_ops, op); + AutoLock c_lock(copy_op_lock); + release_operation(available_repl_index_copy_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_capture_op(TraceCaptureOp *op) + void Runtime::free_repl_deletion_op(ReplDeletionOp *op) //-------------------------------------------------------------------------- { - AutoLock c_lock(capture_op_lock); - release_operation(available_capture_ops, op); + AutoLock d_lock(deletion_op_lock); + release_operation(available_repl_deletion_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_trace_op(TraceCompleteOp *op) + void Runtime::free_repl_pending_partition_op(ReplPendingPartitionOp *op) //-------------------------------------------------------------------------- { - AutoLock t_lock(trace_op_lock); - release_operation(available_trace_ops, op); + AutoLock p_lock(pending_partition_op_lock); + release_operation(available_repl_pending_partition_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_replay_op(TraceReplayOp *op) + void Runtime::free_repl_dependent_partition_op(ReplDependentPartitionOp *op) //-------------------------------------------------------------------------- { - AutoLock t_lock(replay_op_lock); - release_operation(available_replay_ops, op); + AutoLock d_lock(dependent_partition_op_lock); + release_operation(available_repl_dependent_partition_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_begin_op(TraceBeginOp *op) + void Runtime::free_repl_epoch_op(ReplMustEpochOp *op) //-------------------------------------------------------------------------- { - AutoLock t_lock(begin_op_lock); - release_operation(available_begin_ops, op); + AutoLock m_lock(epoch_op_lock); + release_operation(available_repl_must_epoch_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_summary_op(TraceSummaryOp *op) + void Runtime::free_repl_timing_op(ReplTimingOp *op) //-------------------------------------------------------------------------- { - AutoLock t_lock(summary_op_lock); - release_operation(available_summary_ops, op); + AutoLock t_lock(timing_op_lock); + release_operation(available_repl_timing_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_epoch_op(MustEpochOp *op) + void Runtime::free_repl_all_reduce_op(ReplAllReduceOp *op) //-------------------------------------------------------------------------- { - AutoLock e_lock(epoch_op_lock); - release_operation(available_epoch_ops, op); + AutoLock a_lock(all_reduce_op_lock); + release_operation(available_repl_all_reduce_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_pending_partition_op(PendingPartitionOp *op) + void Runtime::free_repl_fence_op(ReplFenceOp *op) //-------------------------------------------------------------------------- { - AutoLock p_lock(pending_partition_op_lock); - release_operation(available_pending_partition_ops, op); + AutoLock t_lock(fence_op_lock); + release_operation(available_repl_fence_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_dependent_partition_op(DependentPartitionOp *op) + void Runtime::free_repl_map_op(ReplMapOp *op) //-------------------------------------------------------------------------- { - AutoLock p_lock(dependent_partition_op_lock); - release_operation(available_dependent_partition_ops, op); + AutoLock m_lock(map_op_lock); + release_operation(available_repl_map_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_point_dep_part_op(PointDepPartOp *op) + void Runtime::free_repl_attach_op(ReplAttachOp *op) //-------------------------------------------------------------------------- { - AutoLock p_lock(dependent_partition_op_lock); - release_operation(available_point_dep_part_ops, op); + AutoLock a_lock(attach_op_lock); + release_operation(available_repl_attach_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_fill_op(FillOp *op) + void Runtime::free_repl_detach_op(ReplDetachOp *op) //-------------------------------------------------------------------------- { - AutoLock f_lock(fill_op_lock); - release_operation(available_fill_ops, op); + AutoLock d_lock(detach_op_lock); + release_operation(available_repl_detach_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_index_fill_op(IndexFillOp *op) + void Runtime::free_repl_capture_op(ReplTraceCaptureOp *op) //-------------------------------------------------------------------------- { - AutoLock f_lock(fill_op_lock); - release_operation(available_index_fill_ops, op); + AutoLock c_lock(capture_op_lock); + release_operation(available_repl_capture_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_point_fill_op(PointFillOp *op) + void Runtime::free_repl_trace_op(ReplTraceCompleteOp *op) //-------------------------------------------------------------------------- { - AutoLock f_lock(fill_op_lock); - release_operation(available_point_fill_ops, op); + AutoLock t_lock(trace_op_lock); + release_operation(available_repl_trace_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_attach_op(AttachOp *op) + void Runtime::free_repl_replay_op(ReplTraceReplayOp *op) //-------------------------------------------------------------------------- { - AutoLock a_lock(attach_op_lock); - release_operation(available_attach_ops, op); + AutoLock t_lock(replay_op_lock); + release_operation(available_repl_replay_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_detach_op(DetachOp *op) + void Runtime::free_repl_begin_op(ReplTraceBeginOp *op) //-------------------------------------------------------------------------- { - AutoLock d_lock(detach_op_lock); - release_operation(available_detach_ops, op); + AutoLock t_lock(summary_op_lock); + release_operation(available_repl_begin_ops, op); } //-------------------------------------------------------------------------- - void Runtime::free_timing_op(TimingOp *op) + void Runtime::free_repl_summary_op(ReplTraceSummaryOp *op) //-------------------------------------------------------------------------- { - AutoLock t_lock(timing_op_lock); - release_operation(available_timing_ops, op); + AutoLock t_lock(summary_op_lock); + release_operation(available_repl_summary_ops, op); } //-------------------------------------------------------------------------- @@ -21074,7 +23947,8 @@ namespace Legion { //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert((context_uid % runtime_stride) == address_space); // sanity check + // sanity check + assert((context_uid % total_address_spaces) == address_space); #endif AutoLock ctx_lock(context_lock); #ifdef DEBUG_LEGION @@ -21088,7 +23962,8 @@ namespace Legion { //-------------------------------------------------------------------------- { #ifdef DEBUG_LEGION - assert((context_uid % runtime_stride) == address_space); // sanity check + // sanity check + assert((context_uid % total_address_spaces) == address_space); #endif AutoLock ctx_lock(context_lock); std::map::iterator finder = @@ -21305,6 +24180,49 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + void Runtime::register_shard_manager(ReplicationID repl_id, + ShardManager *manager) + //-------------------------------------------------------------------------- + { + AutoLock s_lock(shard_lock); +#ifdef DEBUG_LEGION + assert(shard_managers.find(repl_id) == shard_managers.end()); +#endif + shard_managers[repl_id] = manager; + } + + //-------------------------------------------------------------------------- + void Runtime::unregister_shard_manager( + ReplicationID repl_id, bool reclaim_id) + //-------------------------------------------------------------------------- + { + AutoLock s_lock(shard_lock); + std::map::iterator + finder = shard_managers.find(repl_id); +#ifdef DEBUG_LEGION + assert(finder != shard_managers.end()); +#endif + shard_managers.erase(finder); + } + + //-------------------------------------------------------------------------- + ShardManager* Runtime::find_shard_manager(ReplicationID repl_id, + bool can_fail) + //-------------------------------------------------------------------------- + { + AutoLock s_lock(shard_lock,1,false/*exclusive*/); + std::map::const_iterator + finder = shard_managers.find(repl_id); + if (finder == shard_managers.end()) + { + if (can_fail) + return NULL; + assert(false); // Should never get here + } + return finder->second; + } + //-------------------------------------------------------------------------- bool Runtime::is_local(Processor proc) const //-------------------------------------------------------------------------- @@ -21499,6 +24417,18 @@ namespace Legion { } #endif + //-------------------------------------------------------------------------- + ReplicationID Runtime::get_unique_replication_id(void) + //-------------------------------------------------------------------------- + { + ReplicationID result = + __sync_fetch_and_add(&unique_control_replication_id, runtime_stride); +#ifdef DEBUG_LEGION + assert(result <= unique_control_replication_id); +#endif + return result; + } + //-------------------------------------------------------------------------- LegionErrorType Runtime::verify_requirement( const RegionRequirement &req, FieldID &bad_field) @@ -22462,6 +25392,8 @@ namespace Legion { .add_option_bool("-lg:unsafe_launch",config.unsafe_launch,!filter) .add_option_bool("-lg:unsafe_mapper",config.unsafe_mapper,!filter) .add_option_bool("-lg:safe_mapper",config.safe_mapper,!filter) + .add_option_bool("-lg:safe_ctrlrepl", + config.safe_control_replication, !filter) .add_option_bool("-lg:inorder",config.program_order_execution,!filter) .add_option_bool("-lg:dump_physical_traces", config.dump_physical_traces, !filter) @@ -22645,6 +25577,72 @@ namespace Legion { return result; } + //-------------------------------------------------------------------------- + IndividualTask* Runtime::create_implicit_top_level(TaskID top_task_id, + MapperID top_mapper_id, Processor proxy, const char *task_name) + //-------------------------------------------------------------------------- + { + // Save the top-level task name if necessary + if (task_name != NULL) + attach_semantic_information(top_task_id, + LEGION_NAME_SEMANTIC_TAG, task_name, + strlen(task_name) + 1, true/*mutable*/); + // Get an individual task to be the top-level task + IndividualTask *top_task = get_available_individual_task(); + // Get a remote task to serve as the top of the top-level task + TopLevelContext *top_context = + new TopLevelContext(this, get_unique_operation_id()); + // Save the context in the implicit context + implicit_context = top_context; + // Add a reference to the top level context + top_context->add_reference(); + // Set the executing processor + top_context->set_executing_processor(proxy); + TaskLauncher launcher(top_task_id, TaskArgument(), + Predicate::TRUE_PRED, top_mapper_id); + // Mark that this task is the top-level task + top_task->initialize_task(top_context, launcher, false/*track parent*/, + true/*top level task*/, true/*implicit top level task*/); + increment_outstanding_top_level_tasks(); + // Launch a task to deactivate the top-level context + // when the top-level task is done + TopFinishArgs args(top_context); + ApEvent pre = top_task->get_task_completion(); + issue_runtime_meta_task(args, LG_LATENCY_WORK_PRIORITY, + Runtime::protect_event(pre)); + return top_task; + } + + //-------------------------------------------------------------------------- + ImplicitShardManager* Runtime::find_implicit_shard_manager( + TaskID top_task_id, MapperID mapper_id, Processor::Kind kind, + unsigned shards_per_address_space, bool local) + //-------------------------------------------------------------------------- + { + AutoLock s_lock(shard_lock); + std::map::iterator finder = + implicit_shard_managers.find(top_task_id); + ImplicitShardManager *result = NULL; + if (finder == implicit_shard_managers.end()) + { + result = new ImplicitShardManager(this, top_task_id, mapper_id, + kind, shards_per_address_space); + result->add_reference(); + implicit_shard_managers[top_task_id] = result; + finder = implicit_shard_managers.find(top_task_id); + } + else + result = finder->second; + result->add_reference(); + if (result->record_arrival(local)) + { + if (finder->second->remove_reference()) + assert(false); // should never hit this assertion + implicit_shard_managers.erase(finder); + } + return result; + } + //-------------------------------------------------------------------------- Context Runtime::begin_implicit_task(TaskID top_task_id, MapperID top_mapper_id, @@ -22665,35 +25663,6 @@ namespace Legion { "Implicit top-level tasks are not allowed to be started on " "processors managed by Legion. They can only be started on " "external threads that Legion does not control.") - // Wait for the runtime to have started if necessary - if (!runtime_started_event.has_triggered()) - runtime_started_event.external_wait(); - - // Record that this is an external implicit task - external_implicit_task = true; - - InnerContext *execution_context = NULL; - // Now that the runtime is started we can make our context - if (control_replicable && (total_address_spaces > 1)) - REPORT_LEGION_ERROR(ERROR_ILLEGAL_IMPLICIT_TOP_LEVEL_TASK, - "Implicit top-level tasks are only supported on multiple " - "nodes in the control_replication and later branches.") - // Save the top-level task name if necessary - if (task_name != NULL) - attach_semantic_information(top_task_id, - LEGION_NAME_SEMANTIC_TAG, task_name, - strlen(task_name) + 1, true/*mutable*/); - // Get an individual task to be the top-level task - IndividualTask *top_task = get_available_individual_task(); - // Get a remote task to serve as the top of the top-level task - TopLevelContext *top_context = - new TopLevelContext(this, get_unique_operation_id()); - // Save the context in the implicit context - implicit_context = top_context; - implicit_runtime = this; - // Add a reference to the top level context - top_context->add_reference(); - // Set the executing processor #ifdef DEBUG_LEGION assert(!local_procs.empty()); #endif @@ -22714,26 +25683,39 @@ namespace Legion { // as a new kind of processor to use assert(proxy.exists()); #endif - top_context->set_executing_processor(proxy); - TaskLauncher launcher(top_task_id, TaskArgument(), - Predicate::TRUE_PRED, top_mapper_id); - // Mark that this task is the top-level task - top_task->initialize_task(top_context, launcher, false/*track parent*/, - true/*top level task*/, true/*implicit top level task*/); - increment_outstanding_top_level_tasks(); - top_context->increment_pending(); + // Wait for the runtime to have started if necessary + if (!runtime_started_event.has_triggered()) + runtime_started_event.external_wait(); + // Record that this is an external implicit task + external_implicit_task = true; + SingleTask *local_task = NULL; + // Now that the runtime is started we can make our context + if (control_replicable && (total_address_spaces > 1)) + { + // Either find or make an implicit shard manager for hooking up + ImplicitShardManager *implicit_shard_manager = + find_implicit_shard_manager(top_task_id, top_mapper_id, proc_kind, + shards_per_address_space, true/*local*/); + local_task = + implicit_shard_manager->create_shard(shard_id, proxy, task_name); + if (implicit_shard_manager->remove_reference()) + delete implicit_shard_manager; + } + else + { + local_task = create_implicit_top_level(top_task_id, top_mapper_id, + proxy, task_name); + // Increment the pending count here + local_task->get_context()->increment_pending(); + } #ifdef DEBUG_LEGION increment_total_outstanding_tasks(top_task_id, false); #else increment_total_outstanding_tasks(); #endif - // Launch a task to deactivate the top-level context - // when the top-level task is done - TopFinishArgs args(top_context); - ApEvent pre = top_task->get_task_completion(); - issue_runtime_meta_task(args, LG_LATENCY_WORK_PRIORITY, - Runtime::protect_event(pre)); - execution_context = top_task->create_implicit_context(); + InnerContext *execution_context = local_task->create_implicit_context(); + implicit_context = execution_context; + implicit_runtime = this; Legion::Runtime *dummy_rt; execution_context->begin_task(dummy_rt); execution_context->set_executing_processor(proxy); @@ -22760,7 +25742,7 @@ namespace Legion { #ifdef DEBUG_LEGION if (config.num_profiling_nodes > 0) { - // Give a massive warning about profiling with Legion Spy enabled + // Give a massive warning about profiling with debug enabled for (int i = 0; i < 2; i++) fprintf(stderr,"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); for (int i = 0; i < 4; i++) @@ -22834,7 +25816,7 @@ namespace Legion { #ifdef BOUNDS_CHECKS if (config.num_profiling_nodes > 0) { - // Give a massive warning about profiling with Legion Spy enabled + // Give a massive warning about profiling with bounds checks enabled for (int i = 0; i < 2; i++) fprintf(stderr,"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); for (int i = 0; i < 4; i++) @@ -22859,7 +25841,7 @@ namespace Legion { #ifdef PRIVILEGE_CHECKS if (config.num_profiling_nodes > 0) { - // Give a massive warning about profiling with Legion Spy enabled + // Give a massive warning about profiling with privilege checks enabled for (int i = 0; i < 2; i++) fprintf(stderr,"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); for (int i = 0; i < 4; i++) @@ -23029,8 +26011,6 @@ namespace Legion { address_spaces.insert(sid); proc_spaces[*it] = sid; } - if (address_spaces.size() > 1) - config.configure_collective_settings(address_spaces.size()); InputArgs input_args; input_args.argc = argc; input_args.argv = argv; @@ -23067,8 +26047,6 @@ namespace Legion { address_spaces.insert(sid); proc_spaces[*it] = sid; } - if (address_spaces.size() > 1) - config.configure_collective_settings(address_spaces.size()); // Make one runtime instance and record it with all the processors const AddressSpace local_space = local_procs.begin()->address_space(); InputArgs input_args; @@ -23175,7 +26153,17 @@ namespace Legion { #endif #endif // Lastly do any other registrations we might have +#ifdef DEBUG_LEGION_COLLECTIVES + ReductionOpTable& red_table = get_reduction_table(true/*safe*/); + red_table[CollectiveCheckReduction::REDOP] = + Realm::ReductionOpUntyped::create_reduction_op< + CollectiveCheckReduction>(); + red_table[CloseCheckReduction::REDOP]= + Realm::ReductionOpUntyped::create_reduction_op< + CloseCheckReduction>(); +#else const ReductionOpTable& red_table = get_reduction_table(true/*safe*/); +#endif for(ReductionOpTable::const_iterator it = red_table.begin(); it != red_table.end(); it++) @@ -23450,16 +26438,19 @@ namespace Legion { "the runtime has been started with multiple runtime instances.") const RtEvent done_event = the_runtime->perform_registration_callback(callback, global); - if (done_event.exists() && !done_event.has_triggered()) + if (done_event.exists()) { // If we have a context then record that no operations are // allowed to be executed until after this registration is done if (implicit_context != NULL) implicit_context->handle_registration_callback_effects(done_event); - else if (Processor::get_executing_processor().exists()) - done_event.wait(); - else - done_event.external_wait(); + else if (!done_event.has_triggered()) + { + if (Processor::get_executing_processor().exists()) + done_event.wait(); + else + done_event.external_wait(); + } } } else // can safely ignore global as this call must be done everywhere @@ -23539,13 +26530,22 @@ namespace Legion { if (redop_id == 0) REPORT_LEGION_ERROR(ERROR_RESERVED_REDOP_ID, "ERROR: ReductionOpID zero is reserved.") + // TODO: figure out a way to make this safe with dynamic registration +#if 0 + if (redop_id >= LEGION_MAX_APPLICATION_REDOP_ID) + REPORT_LEGION_ERROR(ERROR_RESERVED_REDOP_ID, + "ERROR: ReductionOpID %d is greater than or equal " + "to the LEGION_MAX_APPLICATION_REDOP_ID of %d " + "set in legion_config.h.", redop_id, + LEGION_MAX_APPLICATION_REDOP_ID) +#endif ReductionOpTable &red_table = Runtime::get_reduction_table(true/*safe*/); // Check to make sure we're not overwriting a prior reduction op if (!permit_duplicates && (red_table.find(redop_id) != red_table.end())) REPORT_LEGION_ERROR(ERROR_DUPLICATE_REDOP_ID, "ERROR: ReductionOpID " - "%d has already been used in the reduction table\n",redop_id) + "%d has already been used in the reduction table",redop_id) red_table[redop_id] = redop; if ((init_fnptr != NULL) || (fold_fnptr != NULL)) { @@ -23618,25 +26618,24 @@ namespace Legion { if (!runtime_started || has_lock) { if (serdez_id == 0) - { - fprintf(stderr,"ERROR: Custom Serdez ID zero is reserved.\n"); -#ifdef DEBUG_LEGION - assert(false); + REPORT_LEGION_ERROR(ERROR_RESERVED_SERDEZ_ID, + "ERROR: Custom Serdez ID zero is reserved.\n") + // TODO: figure out a way to make this safe with dynamic registration +#if 0 + if (serdez_id >= LEGION_MAX_APPLICATION_SERDEZ_ID) + REPORT_LEGION_ERROR(ERROR_RESERVED_SERDEZ_ID, + "ERROR: ReductionOpID %d is greater than or equal " + "to the LEGION_MAX_APPLICATION_SERDEZ_ID of %d set " + "in legion_config.h.", serdez_id, + LEGION_MAX_APPLICATION_SERDEZ_ID) #endif - exit(ERROR_RESERVED_SERDEZ_ID); - } SerdezOpTable &serdez_table = Runtime::get_serdez_table(true/*safe*/); // Check to make sure we're not overwriting a prior serdez op if (!permit_duplicates && (serdez_table.find(serdez_id) != serdez_table.end())) - { - fprintf(stderr,"ERROR: CustomSerdezID %d has already been used " - "in the serdez operation table\n", serdez_id); -#ifdef DEBUG_LEGION - assert(false); -#endif - exit(ERROR_DUPLICATE_SERDEZ_ID); - } + REPORT_LEGION_ERROR(ERROR_DUPLICATE_SERDEZ_ID, + "ERROR: CustomSerdezID %d has already been used " + "in the serdez operation table", serdez_id) serdez_table[serdez_id] = serdez_op; } else @@ -23672,6 +26671,15 @@ namespace Legion { return pending_projection_table; } + //-------------------------------------------------------------------------- + /*static*/ std::map& + Runtime::get_pending_sharding_table(void) + //-------------------------------------------------------------------------- + { + static std::map pending_sharding_table; + return pending_sharding_table; + } + //-------------------------------------------------------------------------- /*static*/ std::vector& Runtime::get_pending_handshake_table(void) @@ -24007,6 +27015,9 @@ namespace Legion { data += sizeof(implicit_provenance); arglen -= sizeof(implicit_provenance); LgTaskID tid = *((const LgTaskID*)data); +#ifdef DEBUG_LEGION_WAITS + meta_task_id = tid; +#endif data += sizeof(tid); arglen -= sizeof(tid); switch (tid) @@ -24135,27 +27146,27 @@ namespace Legion { } case LG_MUST_INDIV_ID: { - MustEpochTriggerer::handle_individual(args); + MustEpochOp::handle_trigger_individual(args); break; } case LG_MUST_INDEX_ID: { - MustEpochTriggerer::handle_index(args); + MustEpochOp::handle_trigger_index(args); break; } case LG_MUST_MAP_ID: { - MustEpochMapper::handle_map_task(args); + MustEpochOp::handle_map_task(args); break; } case LG_MUST_DIST_ID: { - MustEpochDistributor::handle_distribute_task(args); + MustEpochOp::handle_distribute_task(args); break; } case LG_MUST_LAUNCH_ID: { - MustEpochDistributor::handle_launch_task(args); + MustEpochOp::handle_launch_task(args); break; } case LG_DEFERRED_FUTURE_SET_ID: @@ -24184,7 +27195,8 @@ namespace Legion { for (Domain::DomainPointIterator itr(future_args->domain); itr; itr++) { - Future f = future_args->future_map->get_future(itr.p); + Future f = + future_args->future_map->get_future(itr.p, true/*internal*/); if (result_size > 0) f.impl->set_result(result, result_size, false/*own*/); } @@ -24238,10 +27250,8 @@ namespace Legion { } case LG_DISJOINTNESS_TASK_ID: { - RegionTreeForest::DisjointnessArgs *dargs = - (RegionTreeForest::DisjointnessArgs*)args; - runtime->forest->compute_partition_disjointness(dargs->handle, - dargs->ready); + RegionTreeForest *forest = runtime->forest; + IndexPartNode::handle_disjointness_computation(args, forest); break; } case LG_DEFER_PHYSICAL_REGISTRATION_TASK_ID: @@ -24265,11 +27275,6 @@ namespace Legion { dargs->parent, dargs->left, dargs->right); break; } - case LG_PENDING_CHILD_TASK_ID: - { - IndexPartNode::handle_pending_child_task(args); - break; - } case LG_POST_DECREMENT_TASK_ID: { InnerContext::PostDecrementArgs *dargs = @@ -24395,6 +27400,11 @@ namespace Legion { enqueue_args->manager->add_to_ready_queue(enqueue_args->task); break; } + case LG_DEFERRED_TASK_COMPLETE_TASK_ID: + { + TaskOp::handle_deferred_task_complete(args); + break; + } case LG_DEFER_MAPPER_MESSAGE_TASK_ID: { MapperManager::handle_deferred_message(args); @@ -24464,6 +27474,21 @@ namespace Legion { PhiView::handle_deferred_view_registration(args); break; } + case LG_CONTROL_REP_LAUNCH_TASK_ID: + { + ShardManager::handle_launch(args); + break; + } + case LG_CONTROL_REP_DELETE_TASK_ID: + { + ShardManager::handle_delete(args); + break; + } + case LG_RECLAIM_FUTURE_MAP_TASK_ID: + { + ReplFutureMapImpl::handle_future_map_reclaim(args); + break; + } case LG_TIGHTEN_INDEX_SPACE_TASK_ID: { IndexSpaceExpression::handle_tighten_index_space(args); @@ -24617,8 +27642,18 @@ namespace Legion { break; } #endif + case LG_DEFER_CONSENSUS_MATCH_TASK_ID: + { + ConsensusMatchBase::handle_consensus_match(args); + break; + } case LG_YIELD_TASK_ID: break; // nothing to do here + case LG_DEFER_TRACE_UPDATE_TASK_ID: + { + ShardedPhysicalTemplate::handle_deferred_trace_update(args,runtime); + break; + } case LG_RETRY_SHUTDOWN_TASK_ID: { const ShutdownManager::RetryShutdownArgs *shutdown_args = @@ -24697,59 +27732,6 @@ namespace Legion { runtime->handle_endpoint_creation(derez); } - //-------------------------------------------------------------------------- - void Runtime::LegionConfiguration::configure_collective_settings( - int total_spaces) const - //-------------------------------------------------------------------------- - { -#ifdef DEBUG_LEGION - assert(legion_collective_radix > 0); -#endif - const int MultiplyDeBruijnBitPosition[32] = - { - 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 - }; - // First adjust the radix based on the number of nodes if necessary - if (legion_collective_radix > total_spaces) - legion_collective_radix = total_spaces; - // Adjust the radix to the next smallest power of 2 - uint32_t radix_copy = legion_collective_radix; - for (int i = 0; i < 5; i++) - radix_copy |= radix_copy >> (1 << i); - legion_collective_log_radix = - MultiplyDeBruijnBitPosition[(uint32_t)(radix_copy * 0x07C4ACDDU) >> 27]; - if (legion_collective_radix != (1 << legion_collective_log_radix)) - legion_collective_radix = (1 << legion_collective_log_radix); - - // Compute the number of stages - uint32_t node_copy = total_spaces; - for (int i = 0; i < 5; i++) - node_copy |= node_copy >> (1 << i); - // Now we have it log 2 - int log_nodes = - MultiplyDeBruijnBitPosition[(uint32_t)(node_copy * 0x07C4ACDDU) >> 27]; - // Stages round up in case of incomplete stages - legion_collective_stages = (log_nodes + - legion_collective_log_radix - 1) / legion_collective_log_radix; - int log_remainder = log_nodes % legion_collective_log_radix; - if (log_remainder > 0) - { - // We have an incomplete last stage - legion_collective_last_radix = 1 << log_remainder; - // Now we can compute the number of participating stages - legion_collective_participating_spaces = - 1 << ((legion_collective_stages - 1) * legion_collective_log_radix + - log_remainder); - } - else - { - legion_collective_last_radix = legion_collective_radix; - legion_collective_participating_spaces = - 1 << (legion_collective_stages * legion_collective_log_radix); - } - } - #ifdef TRACE_ALLOCATION //-------------------------------------------------------------------------- /*static*/ void LegionAllocation::trace_allocation( diff --git a/runtime/legion/runtime.h b/runtime/legion/runtime.h index bb5e643358..c6daa43cf6 100644 --- a/runtime/legion/runtime.h +++ b/runtime/legion/runtime.h @@ -120,12 +120,19 @@ namespace Legion { public: FutureMap freeze(TaskContext *ctx); void unfreeze(void); + protected: + void free_point_set(void); public: Runtime *const runtime; private: FutureMap future_map; std::map arguments; + std::set point_set_deletion_preconditions; + Domain point_set; + unsigned dimensionality; unsigned dependent_futures; // number of futures with producer ops + bool update_point_set; + bool own_point_set; bool equivalent; // argument and future_map the same }; @@ -197,8 +204,15 @@ namespace Legion { }; public: FutureImpl(Runtime *rt, bool register_future, DistributedID did, - AddressSpaceID owner_space, ApEvent complete_event, + AddressSpaceID owner_space, ApEvent complete, Operation *op = NULL); + FutureImpl(Runtime *rt, bool register_future, DistributedID did, + AddressSpaceID owner_space, ApEvent complete, + Operation *op, GenerationID gen, +#ifdef LEGION_SPY + UniqueID op_uid, +#endif + int op_depth); FutureImpl(const FutureImpl &rhs); virtual ~FutureImpl(void); public: @@ -229,6 +243,10 @@ namespace Legion { bool get_boolean_value(bool &valid); // Request that the value be made ready on this node ApEvent subscribe(void); + // Request the value be made ready on this node for + // internal use which means we can see the value before + // the future actually completes + RtEvent subscribe_internal(void); public: virtual void notify_active(ReferenceMutator *mutator); virtual void notify_valid(ReferenceMutator *mutator); @@ -268,6 +286,7 @@ namespace Legion { mutable LocalLock future_lock; ApEvent future_complete; ApUserEvent subscription_event; + RtUserEvent subscription_internal; // On the owner node, keep track of the registered waiters std::set subscribers; void *result; @@ -289,43 +308,53 @@ namespace Legion { public: static const AllocationType alloc_type = FUTURE_MAP_ALLOC; public: - FutureMapImpl(TaskContext *ctx, Operation *op, RtEvent ready_event, - Runtime *rt, DistributedID did, AddressSpaceID owner_space); - FutureMapImpl(TaskContext *ctx, Runtime *rt, + FutureMapImpl(TaskContext *ctx, Operation *op, + RtEvent ready, const Domain &domain, + Runtime *rt, DistributedID did, AddressSpaceID owner_space, + RtUserEvent deletion_trigger=RtUserEvent::NO_RT_USER_EVENT); + FutureMapImpl(TaskContext *ctx, Runtime *rt, const Domain &domain, DistributedID did, AddressSpaceID owner_space, - RtEvent ready_event, bool register_now = true); // remote + RtEvent ready_event, bool register_now = true, // remote + RtUserEvent deletion_trigger=RtUserEvent::NO_RT_USER_EVENT); FutureMapImpl(const FutureMapImpl &rhs); virtual ~FutureMapImpl(void); public: FutureMapImpl& operator=(const FutureMapImpl &rhs); public: inline RtEvent get_ready_event(void) const { return ready_event; } + inline const Domain& get_domain(void) const { return future_map_domain; } + virtual bool is_replicate_future_map(void) const { return false; } public: virtual void notify_active(ReferenceMutator *mutator); virtual void notify_valid(ReferenceMutator *mutator); virtual void notify_invalid(ReferenceMutator *mutator); virtual void notify_inactive(ReferenceMutator *mutator); public: - Future get_future(const DomainPoint &point, RtEvent *wait_on = NULL); - // Will return NULL if it does not exist - FutureImpl* find_future(const DomainPoint &point); - void set_all_futures(const std::map &others); + virtual Future get_future(const DomainPoint &point, + bool internal_only, + RtEvent *wait_on = NULL); void set_future(const DomainPoint &point, FutureImpl *impl, ReferenceMutator *mutator); void get_void_result(const DomainPoint &point, bool silence_warnings = true, const char *warning_string = NULL); - void wait_all_results(bool silence_warnings = true, - const char *warning_string = NULL); - // This marks that all the futures are ready somewhere + virtual void wait_all_results(bool silence_warnings = true, + const char *warning_string = NULL); bool reset_all_futures(RtEvent new_ready_event); + // Use this method to detect when we're wrapped by an argument + // map which is mainly needed in control replication + virtual void argument_map_wrap(void) { } public: - void get_all_futures(std::map &futures) const; -#ifdef DEBUG_LEGION + virtual void get_all_futures(std::map &futures); + void set_all_futures(const std::map &futures); + // Dump helper method for template classes + static inline FutureImpl* unpack_future(const Future &future) + { return future.impl; } public: - void add_valid_domain(const Domain &d); - void add_valid_point(const DomainPoint &dp); -#endif + // Will return NULL if it does not exist + virtual FutureImpl* find_shard_local_future(const DomainPoint &point); + virtual void get_shard_local_futures( + std::map &futures); public: void register_dependence(Operation *consumer_op); public: @@ -343,15 +372,111 @@ namespace Legion { #ifdef LEGION_SPY const UniqueID op_uid; #endif - private: + const Domain future_map_domain; + protected: mutable LocalLock future_map_lock; RtEvent ready_event; + RtUserEvent delete_event; std::map futures; -#ifdef DEBUG_LEGION - private: - std::vector valid_domains; - std::set valid_points; -#endif + }; + + /** + * \class ReplFutureMapImpl + * This a special kind of future map that is created + * in control replication contexts + */ + class ReplFutureMapImpl : public FutureMapImpl { + public: + struct PendingRequest { + public: + PendingRequest(void) { } + PendingRequest(const DomainPoint &p, DistributedID src, + RtUserEvent done, bool intern) + : point(p), src_did(src), done_event(done), internal(intern) { } + public: + DomainPoint point; + DistributedID src_did; + RtUserEvent done_event; + bool internal; + }; + struct ReclaimFutureMapArgs : + public LgTaskArgs { + public: + static const LgTaskID TASK_ID = LG_RECLAIM_FUTURE_MAP_TASK_ID; + public: + ReclaimFutureMapArgs(ReplicateContext *c, + ReplFutureMapImpl *map, UniqueID uid) + : LgTaskArgs(uid), + ctx(c), impl(map) { } + public: + ReplicateContext *const ctx; + ReplFutureMapImpl *const impl; + }; + public: + ReplFutureMapImpl(ReplicateContext *ctx, Operation *op, RtEvent ready, + const Domain &domain, const Domain &shard_domain, + Runtime *rt, DistributedID did, AddressSpaceID owner, + RtUserEvent deletion_trigger= + RtUserEvent::NO_RT_USER_EVENT); + ReplFutureMapImpl(const ReplFutureMapImpl &rhs); + virtual ~ReplFutureMapImpl(void); + public: + ReplFutureMapImpl& operator=(const ReplFutureMapImpl &rhs); + public: + virtual bool is_replicate_future_map(void) const { return true; } + public: + // Override this so we can trigger our deletion barrier + virtual void notify_inactive(ReferenceMutator *mutator); + public: + virtual Future get_future(const DomainPoint &point, + bool internal, RtEvent *wait_on = NULL); + virtual void get_all_futures(std::map &futures); + virtual void wait_all_results(bool silence_warnings = true, + const char *warning_string = NULL); + virtual void argument_map_wrap(void) { has_non_trivial_call = true; } + public: + // Will return NULL if it does not exist + virtual FutureImpl* find_shard_local_future(const DomainPoint &point); + virtual void get_shard_local_futures( + std::map &futures); + public: + void set_sharding_function(ShardingFunction *function); + void handle_future_map_request(Deserializer &derez); + protected: + void process_future_map_request(const DomainPoint &point, + DistributedID src_did, + const bool internal, + RtUserEvent done_event); + public: + static void handle_future_map_response(Deserializer &derez, + Runtime *runtime); + static void handle_future_map_reclaim(const void *args); + public: + ReplicateContext *const repl_ctx; + const Domain shard_domain; + const unsigned future_map_barrier_index; + const RtBarrier future_map_barrier; + const CollectiveID collective_index; // in case we have to do all-to-all + // Unlike normal future maps, we know these only ever exist on the + // node where they are made so we store their producer op information + // in case they have to make futures from remote shards + const int op_depth; + const UniqueID op_uid; + // Use this for checking safety of control replication + const size_t op_ctx_index; + protected: + std::vector pending_future_map_requests; + std::set exchange_events; + RtUserEvent sharding_function_ready; + ShardingFunction *sharding_function; + bool collective_performed; + // For replicated future maps we track whether there have been any + // non-triival calls to this shard of the future map. If there are + // then we know there could be non-trivial calls in other shards. + // Conversely, if there are no non-trivial calls here then there + // shouldn't be in other shards as well because of the rules of + // control replication. + bool has_non_trivial_call; }; /** @@ -381,6 +506,9 @@ namespace Legion { PhysicalRegionImpl& operator=(const PhysicalRegionImpl &rhs); public: inline bool created_accessor(void) const { return made_accessor; } + public: + void set_sharded_view(ShardedView *view); + inline ShardedView* get_sharded_view(void) const { return sharded_view; } public: void wait_until_valid(bool silence_warnings, const char *warning_string, bool warn = false, const char *src = NULL); @@ -453,6 +581,8 @@ namespace Legion { // Instance ref InstanceSet references; RegionRequirement req; + // Only used for control replication + ShardedView *sharded_view; bool mapped; // whether it is currently mapped bool valid; // whether it is currently valid // whether to trigger the termination event @@ -569,7 +699,7 @@ namespace Legion { void complete_exchange(void); public: Runtime *const runtime; - const bool participating; + bool participating; public: std::map forward_mapping; std::map reverse_mapping; @@ -578,9 +708,65 @@ namespace Legion { RtUserEvent done_event; std::vector stage_notifications; std::vector sent_stages; + protected: + int collective_radix; + int collective_log_radix; + int collective_stages; + int collective_participating_spaces; + int collective_last_radix; + // Handle a small race on deciding who gets to + // trigger the done event bool done_triggered; }; + /** + * \class ImplicitShardManager + * This is a class for helping to construct implicitly + * control replicated top-level tasks from external threads. + * It helps to setup tasks just as though they had been + * control replicated, except everything was already control + * replicated remotely. + */ + class ImplicitShardManager : public Collectable { + public: + ImplicitShardManager(Runtime *rt, TaskID tid, MapperID mid, + Processor::Kind k, unsigned shards_per_address_space); + ImplicitShardManager(const ImplicitShardManager &rhs); + ~ImplicitShardManager(void); + public: + ImplicitShardManager& operator=(const ImplicitShardManager &rhs); + public: + bool record_arrival(bool local); + ShardTask* create_shard(int shard_id, Processor proxy, + const char *task_name); + protected: + void create_shard_manager(Processor proxy, const char *task_name); + void request_shard_manager(void); + public: + void process_implicit_request(void *remote, AddressSpaceID space); + RtUserEvent process_implicit_response(ShardManager *manager, + InnerContext *context); + public: + static void handle_remote_request(Deserializer &derez, Runtime *runtime, + AddressSpaceID remote_space); + static void handle_remote_response(Deserializer &derez, Runtime *runtime); + public: + Runtime *const runtime; + const TaskID task_id; + const MapperID mapper_id; + const Processor::Kind kind; + const unsigned shards_per_address_space; + protected: + mutable LocalLock manager_lock; + unsigned expected_local_arrivals; + unsigned expected_remote_arrivals; + unsigned local_shard_id; + InnerContext *top_context; + ShardManager *volatile shard_manager; + RtUserEvent manager_ready; + std::vector > remote_spaces; + }; + /** * \class ProcessorManager * This class manages all the state for a single processor @@ -1339,6 +1525,7 @@ namespace Legion { inline bool is_leaf(void) const { return leaf_variant; } inline bool is_inner(void) const { return inner_variant; } inline bool is_idempotent(void) const { return idempotent_variant; } + inline bool is_replicable(void) const { return replicable_variant; } inline bool returns_value(void) const { return has_return_value; } inline const char* get_name(void) const { return variant_name; } inline const ExecutionConstraintSet& @@ -1380,6 +1567,7 @@ namespace Legion { bool leaf_variant; bool inner_variant; bool idempotent_variant; + bool replicable_variant; private: char *variant_name; }; @@ -1476,6 +1664,13 @@ namespace Legion { virtual LogicalRegion project(const Mappable *mappable, unsigned index, LogicalPartition upper_bound, const DomainPoint &point); + virtual LogicalRegion project(LogicalRegion upper_bound, + const DomainPoint &point, + const Domain &launch_domain); + virtual LogicalRegion project(LogicalPartition upper_bound, + const DomainPoint &point, + const Domain &launch_domain); + virtual bool is_functional(void) const; virtual bool is_exclusive(void) const; virtual unsigned get_depth(void) const; }; @@ -1496,6 +1691,20 @@ namespace Legion { * A class for wrapping projection functors */ class ProjectionFunction { + public: + class ElideCloseResult { + public: + ElideCloseResult(void); + ElideCloseResult(IndexTreeNode *node, + const std::set &projections, bool result); + public: + bool matches(IndexTreeNode *node, + const std::set &projections) const; + public: + IndexTreeNode *node; + std::set projections; + bool result; + }; public: ProjectionFunction(ProjectionID pid, ProjectionFunctor *functor); ProjectionFunction(const ProjectionFunction &rhs); @@ -1505,13 +1714,14 @@ namespace Legion { public: // The old path explicitly for tasks LogicalRegion project_point(Task *task, unsigned idx, Runtime *runtime, - const DomainPoint &point); + const Domain &launch_domain, const DomainPoint &point); void project_points(const RegionRequirement &req, unsigned idx, - Runtime *runtime, const std::vector &point_tasks, - IndexSpaceNode *launch_space_node); + Runtime *runtime, const Domain &launch_domain, + const std::vector &point_tasks); // Generalized and annonymized void project_points(Operation *op, unsigned idx, - const RegionRequirement &req, Runtime *runtime, + const RegionRequirement &req, + Runtime *runtime, const Domain &launch_domain, const std::vector &points); protected: // Old checking code explicitly for tasks @@ -1533,16 +1743,123 @@ namespace Legion { const std::vector &ordered_points); void check_containment(const Task *task, unsigned idx, const std::vector &ordered_points); + public: + bool find_elide_close_result(const ProjectionInfo &info, + const std::set &projections, + RegionTreeNode *node, bool &result) const; + void record_elide_close_result(const ProjectionInfo &info, + const std::set &projections, + RegionTreeNode *node, bool result); + // From scratch + ProjectionTree* construct_projection_tree(Operation *op, unsigned index, + RegionTreeNode *root, IndexSpaceNode *launch_domain, + ShardingFunction *sharding, + IndexSpaceNode *shard_domain) const; + // Contribute to an existing tree + void construct_projection_tree(Operation *op, unsigned index, + RegionTreeNode *root, IndexSpaceNode *launch_domain, + ShardingFunction *sharding, IndexSpaceNode *sharding_domain, + std::map &node_map) const; + static void add_to_projection_tree(LogicalRegion region, + IndexTreeNode *root, RegionTreeForest *context, + std::map &node_map, + ShardID owner_shard = 0); public: const int depth; const bool is_exclusive; + const bool is_functional; const bool is_invertible; const ProjectionID projection_id; ProjectionFunctor *const functor; - private: + protected: mutable LocalLock projection_reservation; + std::map > elide_close_results; }; + /** + * \class CyclicShardingFunctor + * The cyclic sharding functor just round-robins the points + * onto the available set of shards + */ + class CyclicShardingFunctor : public ShardingFunctor { + public: + CyclicShardingFunctor(void); + CyclicShardingFunctor(const CyclicShardingFunctor &rhs); + virtual ~CyclicShardingFunctor(void); + public: + CyclicShardingFunctor& operator=(const CyclicShardingFunctor &rhs); + public: + template + size_t linearize_point(const Realm::IndexSpace &is, + const Realm::Point &point) const; + public: + virtual ShardID shard(const DomainPoint &point, + const Domain &full_space, + const size_t total_shards); + }; + + /** + * \class ShardingFunction + * The sharding function class wraps a sharding functor and will + * cache results for queries so that we don't need to constantly + * be inverting the results of the sharding functor. + */ + class ShardingFunction { + public: + struct ShardKey { + public: + ShardKey(void) + : sid(0), full_space(IndexSpace::NO_SPACE), + shard_space(IndexSpace::NO_SPACE) { } + ShardKey(ShardID s, IndexSpace f, IndexSpace sh) + : sid(s), full_space(f), shard_space(sh) { } + public: + inline bool operator<(const ShardKey &rhs) const + { + if (sid < rhs.sid) + return true; + if (sid > rhs.sid) + return false; + if (full_space < rhs.full_space) + return true; + if (full_space > rhs.full_space) + return false; + return shard_space < rhs.shard_space; + } + inline bool operator==(const ShardKey &rhs) const + { + if (sid != rhs.sid) + return false; + if (full_space != rhs.full_space) + return false; + return shard_space == rhs.shard_space; + } + public: + ShardID sid; + IndexSpace full_space, shard_space; + }; + public: + ShardingFunction(ShardingFunctor *functor, RegionTreeForest *forest, + ShardingID sharding_id, size_t total_shards); + ShardingFunction(const ShardingFunction &rhs); + virtual ~ShardingFunction(void); + public: + ShardingFunction& operator=(const ShardingFunction &rhs); + public: + ShardID find_owner(const DomainPoint &point,const Domain &sharding_space); + IndexSpace find_shard_space(ShardID shard, IndexSpaceNode *full_space, + IndexSpace sharding_space); + public: + ShardingFunctor *const functor; + RegionTreeForest *const forest; + const ShardingID sharding_id; + const size_t total_shards; + protected: + mutable LocalLock sharding_lock; + std::map shard_index_spaces; + }; + /** * \class Runtime * This is the actual implementation of the Legion runtime functionality @@ -1567,6 +1884,8 @@ namespace Legion { LEGION_DEFAULT_META_TASK_VECTOR_WIDTH), max_message_size(LEGION_DEFAULT_MAX_MESSAGE_SIZE), gc_epoch_size(LEGION_DEFAULT_GC_EPOCH_SIZE), + max_control_replication_contexts( + LEGION_DEFAULT_MAX_CONTROL_REPLICATION_CONTEXTS), max_local_fields(LEGION_DEFAULT_LOCAL_FIELDS), max_replay_parallelism(LEGION_DEFAULT_MAX_REPLAY_PARALLELISM), program_order_execution(false), @@ -1587,6 +1906,7 @@ namespace Legion { unsafe_launch(false), unsafe_mapper(false), safe_mapper(false), + safe_control_replication(false), disable_independence_tests(false), legion_spy_enabled(false), enable_test_mapper(false), @@ -1606,17 +1926,14 @@ namespace Legion { prof_target_latency(100) { } public: int delay_start; - mutable int legion_collective_radix; - mutable int legion_collective_log_radix; - mutable int legion_collective_stages; - mutable int legion_collective_last_radix; - mutable int legion_collective_participating_spaces; + int legion_collective_radix; int initial_task_window_size; unsigned initial_task_window_hysteresis; unsigned initial_tasks_to_schedule; unsigned initial_meta_task_vector_width; unsigned max_message_size; unsigned gc_epoch_size; + unsigned max_control_replication_contexts; unsigned max_local_fields; unsigned max_replay_parallelism; public: @@ -1638,6 +1955,7 @@ namespace Legion { bool unsafe_launch; bool unsafe_mapper; bool safe_mapper; + bool safe_control_replication; bool disable_independence_tests; bool legion_spy_enabled; bool enable_test_mapper; @@ -1657,8 +1975,6 @@ namespace Legion { std::string prof_logfile; size_t prof_footprint_threshold; size_t prof_target_latency; - public: - void configure_collective_settings(int total_spaces) const; }; public: struct DeferredRecycleArgs : public LgTaskArgs { @@ -1748,6 +2064,8 @@ namespace Legion { const Machine machine; const AddressSpaceID address_space; const unsigned total_address_spaces; + // stride for uniqueness, may or may not be the same depending + // on the number of available control replication contexts const unsigned runtime_stride; // stride for uniqueness LegionProfiler *profiler; RegionTreeForest *const forest; @@ -1762,6 +2080,7 @@ namespace Legion { const unsigned initial_meta_task_vector_width; const unsigned max_message_size; const unsigned gc_epoch_size; + const unsigned max_control_replication_contexts; const unsigned max_local_fields; const unsigned max_replay_parallelism; public: @@ -1782,6 +2101,7 @@ namespace Legion { const bool resilient_mode; const bool unsafe_launch; const bool unsafe_mapper; + const bool safe_control_replication; const bool disable_independence_tests; const bool legion_spy_enabled; const bool supply_default_mapper; @@ -1799,15 +2119,12 @@ namespace Legion { const unsigned num_profiling_nodes; public: const int legion_collective_radix; - const int legion_collective_log_radix; - const int legion_collective_stages; - const int legion_collective_last_radix; - const int legion_collective_participating_spaces; MPIRankTable *const mpi_rank_table; public: void register_static_variants(void); void register_static_constraints(void); void register_static_projections(void); + void register_static_sharding_functors(void); void initialize_legion_prof(const LegionConfiguration &config); void log_machine(Machine machine) const; void initialize_mappers(void); @@ -1825,10 +2142,14 @@ namespace Legion { const TaskArgument &arg, MapperID map_id); void process_mapper_task_result(const MapperTaskArgs *args); public: - void create_shared_ownership(IndexSpace handle); - void create_shared_ownership(IndexPartition handle); - void create_shared_ownership(FieldSpace handle); - void create_shared_ownership(LogicalRegion handle); + void create_shared_ownership(IndexSpace handle, + const bool total_sharding_collective = false); + void create_shared_ownership(IndexPartition handle, + const bool total_sharding_collective = false); + void create_shared_ownership(FieldSpace handle, + const bool total_sharding_collective = false); + void create_shared_ownership(LogicalRegion handle, + const bool total_sharding_collective = false); public: IndexPartition get_index_partition(Context ctx, IndexSpace parent, Color color); @@ -2043,10 +2364,13 @@ namespace Legion { bool nuclear); void yield(Context ctx); public: + void print_once(Context ctx, FILE *f, const char *message); + void log_once(Context ctx, Realm::LoggerMessage &message); + public: + bool is_MPI_interop_configured(void); const std::map& find_forward_MPI_mapping(void); const std::map& find_reverse_MPI_mapping(void); int find_local_MPI_rank(void); - bool is_MPI_interop_configured(void); public: Mapping::MapperRuntime* get_mapper_runtime(void); MapperID generate_dynamic_mapper_id(bool check_context = true); @@ -2075,6 +2399,22 @@ namespace Legion { ProjectionFunction* find_projection_function(ProjectionID pid, bool can_fail = false); static ProjectionFunctor* get_projection_functor(ProjectionID pid); + public: + ShardingID generate_dynamic_sharding_id(bool check_context = true); + ShardingID generate_library_sharding_ids(const char *name, size_t count); + static ShardingID& get_current_static_sharding_id(void); + static ShardingID generate_static_sharding_id(void); + void register_sharding_functor(ShardingID sid, + ShardingFunctor *func, + bool need_zero_check = true, + bool silence_warnings= false, + const char *warning_string = NULL, + bool preregistered = false); + static void preregister_sharding_functor(ShardingID sid, + ShardingFunctor *func); + ShardingFunctor* find_sharding_functor(ShardingID sid, + bool can_fail = false); + static ShardingFunctor* get_sharding_functor(ShardingID sid); public: void register_reduction(ReductionOpID redop_id, ReductionOp *redop, @@ -2288,6 +2628,7 @@ namespace Legion { void send_materialized_view(AddressSpaceID target, Serializer &rez); void send_fill_view(AddressSpaceID target, Serializer &rez); void send_phi_view(AddressSpaceID target, Serializer &rez); + void send_sharded_view(AddressSpaceID target, Serializer &rez); void send_reduction_view(AddressSpaceID target, Serializer &rez); void send_instance_manager(AddressSpaceID target, Serializer &rez); void send_collective_instance_manager(AddressSpaceID target, @@ -2322,6 +2663,32 @@ namespace Legion { Serializer &rez); void send_future_map_response_future(AddressSpaceID target, Serializer &rez); + void send_control_replicate_future_map_request(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_future_map_response(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_top_view_request(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_top_view_response(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_equivalence_set_request(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_equivalence_set_response( + AddressSpaceID target, Serializer &rez); + void send_control_replicate_intra_space_dependence(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_resource_update(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_trace_event_request(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_trace_event_response(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_trace_update(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_implicit_request(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_implicit_response(AddressSpaceID target, + Serializer &rez); void send_mapper_message(AddressSpaceID target, Serializer &rez); void send_mapper_broadcast(AddressSpaceID target, Serializer &rez); void send_task_impl_semantic_request(AddressSpaceID target, @@ -2412,6 +2779,17 @@ namespace Legion { void send_constraint_response(AddressSpaceID target, Serializer &rez); void send_constraint_release(AddressSpaceID target, Serializer &rez); void send_mpi_rank_exchange(AddressSpaceID target, Serializer &rez); + void send_replicate_launch(AddressSpaceID target, Serializer &rez); + void send_replicate_delete(AddressSpaceID target, Serializer &rez); + void send_replicate_post_mapped(AddressSpaceID target, Serializer &rez); + void send_replicate_post_execution(AddressSpaceID target, + Serializer &rez); + void send_replicate_trigger_complete(AddressSpaceID target, + Serializer &rez); + void send_replicate_trigger_commit(AddressSpaceID target, + Serializer &rez); + void send_control_replicate_collective_message(AddressSpaceID target, + Serializer &rez); void send_library_mapper_request(AddressSpaceID target, Serializer &rez); void send_library_mapper_response(AddressSpaceID target, Serializer &rez); void send_library_trace_request(AddressSpaceID target, Serializer &rez); @@ -2420,6 +2798,9 @@ namespace Legion { Serializer &rez); void send_library_projection_response(AddressSpaceID target, Serializer &rez); + void send_library_sharding_request(AddressSpaceID target,Serializer &rez); + void send_library_sharding_response(AddressSpaceID target, + Serializer &rez); void send_library_task_request(AddressSpaceID target, Serializer &rez); void send_library_task_response(AddressSpaceID target, Serializer &rez); void send_library_redop_request(AddressSpaceID target, Serializer &rez); @@ -2537,6 +2918,7 @@ namespace Legion { AddressSpaceID source); void handle_send_fill_view(Deserializer &derez, AddressSpaceID source); void handle_send_phi_view(Deserializer &derez, AddressSpaceID source); + void handle_send_sharded_view(Deserializer &derez, AddressSpaceID source); void handle_send_reduction_view(Deserializer &derez, AddressSpaceID source); void handle_send_instance_manager(Deserializer &derez, @@ -2668,6 +3050,30 @@ namespace Legion { void handle_top_level_task_request(Deserializer &derez); void handle_top_level_task_complete(Deserializer &derez); void handle_mpi_rank_exchange(Deserializer &derez); + void handle_replicate_launch(Deserializer &derez,AddressSpaceID source); + void handle_replicate_delete(Deserializer &derez); + void handle_replicate_post_mapped(Deserializer &derez); + void handle_replicate_post_execution(Deserializer &derez); + void handle_replicate_trigger_complete(Deserializer &derez); + void handle_replicate_trigger_commit(Deserializer &derez); + void handle_control_replicate_collective_message(Deserializer &derez); + void handle_control_replicate_future_map_request(Deserializer &derez); + void handle_control_replicate_future_map_response(Deserializer &derez); + void handle_control_replicate_top_view_request(Deserializer &derez, + AddressSpaceID source); + void handle_control_replicate_top_view_response(Deserializer &derez); + void handle_control_replicate_eq_request(Deserializer &derez); + void handle_control_replicate_eq_response(Deserializer &derez); + void handle_control_replicate_intra_space_dependence(Deserializer &derez); + void handle_control_replicate_resource_update(Deserializer &derez); + void handle_control_replicate_trace_event_request(Deserializer &derez, + AddressSpaceID source); + void handle_control_replicate_trace_event_response(Deserializer &derez); + void handle_control_replicate_trace_update(Deserializer &derez, + AddressSpaceID source); + void handle_control_replicate_implicit_request(Deserializer &derez, + AddressSpaceID source); + void handle_control_replicate_implicit_response(Deserializer &derez); void handle_library_mapper_request(Deserializer &derez, AddressSpaceID source); void handle_library_mapper_response(Deserializer &derez); @@ -2677,6 +3083,9 @@ namespace Legion { void handle_library_projection_request(Deserializer &derez, AddressSpaceID source); void handle_library_projection_response(Deserializer &derez); + void handle_library_sharding_request(Deserializer &derez, + AddressSpaceID source); + void handle_library_sharding_response(Deserializer &derez); void handle_library_task_request(Deserializer &derez, AddressSpaceID source); void handle_library_task_response(Deserializer &derez); @@ -2789,6 +3198,8 @@ namespace Legion { DistributedCollectable* weak_find_distributed_collectable( DistributedID did); bool find_pending_collectable_location(DistributedID did,void *&location); + void record_pending_distributed_collectable(DistributedID did); + void revoke_pending_distributed_collectable(DistributedID did); public: LogicalView* find_or_request_logical_view(DistributedID did, RtEvent &ready); @@ -2802,9 +3213,16 @@ namespace Legion { DistributedID did, RtEvent &ready); public: FutureImpl* find_or_create_future(DistributedID did, - ReferenceMutator *mutator); + ReferenceMutator *mutator, + Operation *op = NULL, + GenerationID op_gen = 0, +#ifdef LEGION_SPY + UniqueID op_uid = 0, +#endif + int op_depth = 0); FutureMapImpl* find_or_create_future_map(DistributedID did, - TaskContext *ctx, RtEvent complete, ReferenceMutator *mutator); + TaskContext *ctx, const Domain &domain, + RtEvent complete, ReferenceMutator *mutator); IndexSpace find_or_create_index_slice_space(const Domain &launch_domain, TypeTag type_tag); public: @@ -2875,6 +3293,29 @@ namespace Legion { DetachOp* get_available_detach_op(void); TimingOp* get_available_timing_op(void); AllReduceOp* get_available_all_reduce_op(void); + public: // Control replication operations + ReplIndividualTask* get_available_repl_individual_task(void); + ReplIndexTask* get_available_repl_index_task(void); + ReplMergeCloseOp* get_available_repl_merge_close_op(void); + ReplFillOp* get_available_repl_fill_op(void); + ReplIndexFillOp* get_available_repl_index_fill_op(void); + ReplCopyOp* get_available_repl_copy_op(void); + ReplIndexCopyOp* get_available_repl_index_copy_op(void); + ReplDeletionOp* get_available_repl_deletion_op(void); + ReplPendingPartitionOp* get_available_repl_pending_partition_op(void); + ReplDependentPartitionOp* get_available_repl_dependent_partition_op(void); + ReplMustEpochOp* get_available_repl_epoch_op(void); + ReplTimingOp* get_available_repl_timing_op(void); + ReplAllReduceOp* get_available_repl_all_reduce_op(void); + ReplFenceOp* get_available_repl_fence_op(void); + ReplMapOp* get_available_repl_map_op(void); + ReplAttachOp* get_available_repl_attach_op(void); + ReplDetachOp* get_available_repl_detach_op(void); + ReplTraceCaptureOp* get_available_repl_capture_op(void); + ReplTraceCompleteOp* get_available_repl_trace_op(void); + ReplTraceReplayOp* get_available_repl_replay_op(void); + ReplTraceBeginOp* get_available_repl_begin_op(void); + ReplTraceSummaryOp* get_available_repl_summary_op(void); public: void free_individual_task(IndividualTask *task); void free_point_task(PointTask *task); @@ -2914,6 +3355,29 @@ namespace Legion { void free_detach_op(DetachOp *op); void free_timing_op(TimingOp *op); void free_all_reduce_op(AllReduceOp *op); + public: // Control replication operations + void free_repl_individual_task(ReplIndividualTask *task); + void free_repl_index_task(ReplIndexTask *task); + void free_repl_merge_close_op(ReplMergeCloseOp *op); + void free_repl_fill_op(ReplFillOp *op); + void free_repl_index_fill_op(ReplIndexFillOp *op); + void free_repl_copy_op(ReplCopyOp *op); + void free_repl_index_copy_op(ReplIndexCopyOp *op); + void free_repl_deletion_op(ReplDeletionOp *op); + void free_repl_pending_partition_op(ReplPendingPartitionOp *op); + void free_repl_dependent_partition_op(ReplDependentPartitionOp *op); + void free_repl_epoch_op(ReplMustEpochOp *op); + void free_repl_timing_op(ReplTimingOp *op); + void free_repl_all_reduce_op(ReplAllReduceOp *op); + void free_repl_fence_op(ReplFenceOp *op); + void free_repl_map_op(ReplMapOp *op); + void free_repl_attach_op(ReplAttachOp *op); + void free_repl_detach_op(ReplDetachOp *op); + void free_repl_capture_op(ReplTraceCaptureOp *op); + void free_repl_trace_op(ReplTraceCompleteOp *op); + void free_repl_replay_op(ReplTraceReplayOp *op); + void free_repl_begin_op(ReplTraceBeginOp *op); + void free_repl_summary_op(ReplTraceSummaryOp *op); public: RegionTreeContext allocate_region_tree_context(void); void free_region_tree_context(RegionTreeContext tree_ctx); @@ -2926,7 +3390,14 @@ namespace Legion { bool return_null_if_not_found = false, RtEvent *wait_for = NULL); inline AddressSpaceID get_runtime_owner(UniqueID uid) const - { return (uid % runtime_stride); } + { return (uid % total_address_spaces); } + public: + void register_shard_manager(ReplicationID repl_id, + ShardManager *manager); + void unregister_shard_manager(ReplicationID repl_id, + bool reclaim_id); + ShardManager* find_shard_manager(ReplicationID repl_id, + bool can_fail = false); public: bool is_local(Processor proc) const; void find_visible_memories(Processor proc, std::set &visible); @@ -2941,6 +3412,7 @@ namespace Legion { CodeDescriptorID get_unique_code_descriptor_id(void); LayoutConstraintID get_unique_constraint_id(void); IndexSpaceExprID get_unique_index_space_expr_id(void); + ReplicationID get_unique_replication_id(void); #ifdef LEGION_SPY unsigned get_unique_indirections_id(void); #endif @@ -2989,8 +3461,6 @@ namespace Legion { const void *args, size_t arglen, const void *userdata, size_t userlen, Processor p); - protected: - static void configure_collective_settings(int total_spaces); protected: // Internal runtime methods invoked by the above static methods // after the find the right runtime instance to call @@ -3009,10 +3479,11 @@ namespace Legion { public: std::vector outstanding_counts; #endif - protected: + public: // Internal runtime state // The local processor managed by this runtime const std::set local_procs; + protected: // The local utility processors owned by this runtime const std::set local_utils; // Processor managers for each of the local processors @@ -3070,6 +3541,7 @@ namespace Legion { unsigned unique_code_descriptor_id; unsigned unique_constraint_id; unsigned unique_is_expr_id; + unsigned unique_control_replication_id; #ifdef LEGION_SPY unsigned unique_indirections_id; #endif @@ -3077,6 +3549,7 @@ namespace Legion { unsigned unique_mapper_id; unsigned unique_trace_id; unsigned unique_projection_id; + unsigned unique_sharding_id; unsigned unique_redop_id; unsigned unique_serdez_id; protected: @@ -3112,6 +3585,17 @@ namespace Legion { std::map library_projection_ids; // This is only valid on node 0 unsigned unique_library_projection_id; + protected: + struct LibraryShardingIDs { + public: + ShardingID result; + size_t count; + RtEvent ready; + bool result_set; + }; + std::map library_sharding_ids; + // This is only valid on node 0 + unsigned unique_library_sharding_id; protected: struct LibraryTaskIDs { public: @@ -3164,6 +3648,9 @@ namespace Legion { protected: mutable LocalLock projection_lock; std::map projection_functions; + protected: + mutable LocalLock sharding_lock; + std::map sharding_functors; protected: mutable LocalLock group_lock; LegionMap::aligned, @@ -3196,6 +3683,11 @@ namespace Legion { std::pair > pending_remote_contexts; unsigned total_contexts; std::deque available_contexts; + protected: + // Keep track of managers for control replication execution + mutable LocalLock shard_lock; + std::map shard_managers; + std::map implicit_shard_managers; protected: // For generating random numbers mutable LocalLock random_lock; @@ -3290,6 +3782,31 @@ namespace Legion { std::deque available_detach_ops; std::deque available_timing_ops; std::deque available_all_reduce_ops; + protected: // Control replication operations + std::deque available_repl_individual_tasks; + std::deque available_repl_index_tasks; + std::deque available_repl_merge_close_ops; + std::deque available_repl_fill_ops; + std::deque available_repl_index_fill_ops; + std::deque available_repl_copy_ops; + std::deque available_repl_index_copy_ops; + std::deque available_repl_deletion_ops; + std::deque + available_repl_pending_partition_ops; + std::deque + available_repl_dependent_partition_ops; + std::deque available_repl_must_epoch_ops; + std::deque available_repl_timing_ops; + std::deque available_repl_all_reduce_ops; + std::deque available_repl_fence_ops; + std::deque available_repl_map_ops; + std::deque available_repl_attach_ops; + std::deque available_repl_detach_ops; + std::deque available_repl_capture_ops; + std::deque available_repl_trace_ops; + std::deque available_repl_replay_ops; + std::deque available_repl_begin_ops; + std::deque available_repl_summary_ops; #ifdef DEBUG_LEGION TreeStateLogger *tree_state_logger; // For debugging purposes keep track of @@ -3344,6 +3861,15 @@ namespace Legion { static int wait_for_shutdown(void); static void set_return_code(int return_code); Future launch_top_level_task(const TaskLauncher &launcher); + IndividualTask* create_implicit_top_level(TaskID top_task_id, + MapperID top_mapper_id, + Processor proxy, + const char *task_name); + ImplicitShardManager* find_implicit_shard_manager(TaskID top_task_id, + MapperID top_mapper_id, + Processor::Kind kind, + unsigned shards_per_space, + bool local); Context begin_implicit_task(TaskID top_task_id, MapperID top_mapper_id, Processor::Kind proc_kind, @@ -3391,6 +3917,8 @@ namespace Legion { get_pending_constraint_table(void); static std::map& get_pending_projection_table(void); + static std::map& + get_pending_sharding_table(void); static std::vector& get_pending_handshake_table(void); static std::vector& @@ -3476,14 +4004,29 @@ namespace Legion { static inline RtEvent protect_merge_events( const std::set &events); public: + static inline ApBarrier get_previous_phase(const PhaseBarrier &bar); static inline void phase_barrier_arrive(const PhaseBarrier &bar, unsigned cnt, ApEvent precondition = ApEvent::NO_AP_EVENT, const void *reduce_value = NULL, size_t reduce_value_size = 0); - static inline ApBarrier get_previous_phase(const PhaseBarrier &bar); - static inline void alter_arrival_count(PhaseBarrier &bar, int delta); static inline void advance_barrier(PhaseBarrier &bar); + static inline void alter_arrival_count(PhaseBarrier &bar, int delta); + public: + static inline ApBarrier get_previous_phase(const ApBarrier &bar); + static inline void phase_barrier_arrive(const ApBarrier &bar, + unsigned cnt, ApEvent precondition = ApEvent::NO_AP_EVENT, + const void *reduce_value = NULL, size_t reduce_value_size = 0); + static inline void advance_barrier(ApBarrier &bar); static inline bool get_barrier_result(ApBarrier bar, void *result, size_t result_size); + public: + static inline RtBarrier get_previous_phase(const RtBarrier &bar); + static inline void phase_barrier_arrive(const RtBarrier &bar, + unsigned cnt, RtEvent precondition = RtEvent::NO_RT_EVENT, + const void *reduce_value = NULL, size_t reduce_value_size = 0); + static inline void advance_barrier(RtBarrier &bar); + static inline bool get_barrier_result(RtBarrier bar, void *result, + size_t result_size); + static inline void alter_arrival_count(RtBarrier &bar, int delta); public: static inline ApEvent acquire_ap_reservation(Reservation r,bool exclusive, ApEvent precondition = ApEvent::NO_AP_EVENT); @@ -3923,6 +4466,37 @@ namespace Legion { bar.phase_barrier = ApBarrier(copy.advance_barrier()); } + //-------------------------------------------------------------------------- + /*static*/ inline ApBarrier Runtime::get_previous_phase( + const ApBarrier &bar) + //-------------------------------------------------------------------------- + { + Realm::Barrier copy = bar; + return ApBarrier(copy.get_previous_phase()); + } + + //-------------------------------------------------------------------------- + /*static*/ inline void Runtime::phase_barrier_arrive( + const ApBarrier &bar, unsigned count, ApEvent precondition, + const void *reduce_value, size_t reduce_value_size) + //-------------------------------------------------------------------------- + { + Realm::Barrier copy = bar; + copy.arrive(count, precondition, reduce_value, reduce_value_size); +#ifdef LEGION_SPY + if (precondition.exists()) + LegionSpy::log_event_dependence(precondition, bar); +#endif + } + + //-------------------------------------------------------------------------- + /*static*/ inline void Runtime::advance_barrier(ApBarrier &bar) + //-------------------------------------------------------------------------- + { + Realm::Barrier copy = bar; + bar = ApBarrier(copy.advance_barrier()); + } + //-------------------------------------------------------------------------- /*static*/ inline bool Runtime::get_barrier_result(ApBarrier bar, void *result, size_t result_size) @@ -3932,6 +4506,48 @@ namespace Legion { return copy.get_result(result, result_size); } + //-------------------------------------------------------------------------- + /*static*/ inline RtBarrier Runtime::get_previous_phase(const RtBarrier &b) + //-------------------------------------------------------------------------- + { + Realm::Barrier copy = b; + return RtBarrier(copy.get_previous_phase()); + } + + //-------------------------------------------------------------------------- + /*static*/ inline void Runtime::phase_barrier_arrive(const RtBarrier &bar, + unsigned count, RtEvent precondition, const void *value, size_t size) + //-------------------------------------------------------------------------- + { + Realm::Barrier copy = bar; + copy.arrive(count, precondition, value, size); + } + + //-------------------------------------------------------------------------- + /*static*/ inline void Runtime::advance_barrier(RtBarrier &bar) + //-------------------------------------------------------------------------- + { + Realm::Barrier copy = bar; + bar = RtBarrier(copy.advance_barrier()); + } + + //-------------------------------------------------------------------------- + /*static*/ inline bool Runtime::get_barrier_result(RtBarrier bar, + void *result, size_t result_size) + //-------------------------------------------------------------------------- + { + Realm::Barrier copy = bar; + return copy.get_result(result, result_size); + } + + //-------------------------------------------------------------------------- + /*static*/ inline void Runtime::alter_arrival_count(RtBarrier &b, int delta) + //-------------------------------------------------------------------------- + { + Realm::Barrier copy = b; + b = RtBarrier(copy.alter_arrival_count(delta)); + } + //-------------------------------------------------------------------------- /*static*/ inline ApEvent Runtime::acquire_ap_reservation(Reservation r, bool exclusive, ApEvent precondition) diff --git a/runtime/mappers/default_mapper.cc b/runtime/mappers/default_mapper.cc index 3183e5dc7d..d0882669ea 100644 --- a/runtime/mappers/default_mapper.cc +++ b/runtime/mappers/default_mapper.cc @@ -28,6 +28,7 @@ #define STATIC_MAX_SCHEDULE_COUNT 8 #define STATIC_MEMOIZE false #define STATIC_MAP_LOCALLY false +#define STATIC_REPLICATION_ENABLED true // This is the default implementation of the mapper interface for // the general low level runtime @@ -70,7 +71,8 @@ namespace Legion { stealing_enabled(STATIC_STEALING_ENABLED), max_schedule_count(STATIC_MAX_SCHEDULE_COUNT), memoize(STATIC_MEMOIZE), - map_locally(STATIC_MAP_LOCALLY) + map_locally(STATIC_MAP_LOCALLY), + replication_enabled(STATIC_REPLICATION_ENABLED) //-------------------------------------------------------------------------- { log_mapper.spew("Initializing the default mapper for " @@ -100,6 +102,7 @@ namespace Legion { INT_ARG("-dm:sched", max_schedule_count); BOOL_ARG("-dm:memoize", memoize); BOOL_ARG("-dm:map_locally", map_locally); + BOOL_ARG("-dm:replicate", replication_enabled); #undef BOOL_ARG #undef INT_ARG } @@ -360,6 +363,16 @@ namespace Legion { // This is the best choice for the default mapper assuming // there is locality in the remote mapped tasks output.map_locally = map_locally; + // Control replicate the top-level task in multi-node settings + // otherwise we do no control replication +#ifdef DEBUG_CTRL_REPL + if (task.get_depth() == 0) +#else + if ((total_nodes > 1) && (task.get_depth() == 0)) +#endif + output.replicate = replication_enabled; + else + output.replicate = false; } //-------------------------------------------------------------------------- @@ -709,7 +722,6 @@ namespace Legion { result.proc_kind = Processor::PROC_SET; result.variant = variants[0]; result.tight_bound = (variants.size() == 1); - result.is_inner = false; return result; } } @@ -869,6 +881,8 @@ namespace Legion { // a variant for a specific kind if (cache_result) { + result.is_replicable = + runtime->is_replicable_variant(ctx, task.task_id, result.variant); result.is_inner = runtime->is_inner_variant(ctx, task.task_id, result.variant); if (result.is_inner) @@ -1427,6 +1441,7 @@ namespace Legion { //-------------------------------------------------------------------------- { log_mapper.spew("Default map_task in %s", get_mapper_name()); + Processor::Kind target_kind = task.target_proc.kind(); // Get the variant that we are going to use to map this task VariantInfo chosen = default_find_preferred_variant(task, ctx, @@ -1436,7 +1451,6 @@ namespace Legion { output.postmap_task = false; // Figure out our target processors default_policy_select_target_processors(ctx, task, output.target_procs); - // See if we have an inner variant, if we do virtually map all the regions // We don't even both caching these since they are so simple if (chosen.is_inner) @@ -1644,6 +1658,114 @@ namespace Legion { } } + //-------------------------------------------------------------------------- + void DefaultMapper::map_replicate_task(const MapperContext ctx, + const Task& task, + const MapTaskInput& input, + const MapTaskOutput& def_output, + MapReplicateTaskOutput& output) + //-------------------------------------------------------------------------- + { + // Should only be replicated for the top-level task + assert((task.get_depth() == 0) && (task.regions.size() == 0)); + const Processor::Kind target_kind = task.target_proc.kind(); + // Get the variant that we are going to use to map this task + const VariantInfo chosen = default_find_preferred_variant(task, ctx, + true/*needs tight bound*/, true/*cache*/, target_kind); + if (chosen.is_replicable) + { + const std::vector &remote_procs = + remote_procs_by_kind(target_kind); + // Place on replicate on each node by default + assert(remote_procs.size() == total_nodes); + output.task_mappings.resize(total_nodes, def_output); + // Only check for MPI interop case when dealing with CPUs + if ((target_kind == Processor::LOC_PROC) && + runtime->is_MPI_interop_configured(ctx)) + { + // Check to see if we're interoperating with MPI + const std::map &mpi_interop_mapping = + runtime->find_reverse_MPI_mapping(ctx); + // If we're interoperating with MPI make the shards align with ranks + assert(mpi_interop_mapping.size() == total_nodes); + for (std::vector::const_iterator it = + remote_procs.begin(); it != remote_procs.end(); it++) + { + AddressSpace space = it->address_space(); + std::map::const_iterator finder = + mpi_interop_mapping.find(space); + assert(finder != mpi_interop_mapping.end()); + assert(finder->second < int(output.task_mappings.size())); + output.task_mappings[finder->second].target_procs.push_back(*it); + } + } + else + { + // Otherwise we can just assign shards based on address space + if (total_nodes > 1) + { + for (std::vector::const_iterator it = + remote_procs.begin(); it != remote_procs.end(); it++) + { + AddressSpace space = it->address_space(); + assert(space < output.task_mappings.size()); + output.task_mappings[space].target_procs.push_back(*it); + } + } +#ifdef DEBUG_CTRL_REPL + else + { + const std::vector &local_procs = + local_procs_by_kind(target_kind); + output.task_mappings.resize(local_cpus.size()); + unsigned index = 0; + for (std::vector::const_iterator it = + local_procs.begin(); it != local_procs.end(); it++, index++) + output.task_mappings[index].target_procs.push_back(*it); + } +#endif + } + // Indicate that we want to do control replication by filling + // in the control replication map with our chosen processors + // Also set our chosen variant + if (total_nodes > 1) + { + output.control_replication_map.resize(total_nodes); + for (unsigned idx = 0; idx < total_nodes; idx++) + { + output.task_mappings[idx].chosen_variant = chosen.variant; + output.control_replication_map[idx] = + output.task_mappings[idx].target_procs[0]; + } + } +#ifdef DEBUG_CTRL_REPL + else + { + const std::vector &local_procs = + local_procs_by_kind(target_kind); + output.control_replication_map.resize(local_procs.size()); + for (unsigned idx = 0; idx < local_procs.size(); idx++) + { + output.task_mappings[idx].chosen_variant = chosen.variant; + output.control_replication_map[idx] = + output.task_mappings[idx].target_procs[0]; + } + } +#endif + } + else + { + log_mapper.warning("WARNING: Default mapper was unable to locate " + "a replicable task variant for the top-level " + "task during a multi-node execution! We STRONGLY " + "encourage users to make their top-level tasks " + "replicable to avoid sequential bottlenecks on " + "one node during the execution of an application!"); + output.task_mappings.resize(1); + map_task(ctx, task, input, output.task_mappings[0]); + } + } + //-------------------------------------------------------------------------- void DefaultMapper::default_policy_select_target_processors( MapperContext ctx, @@ -2502,6 +2624,19 @@ namespace Legion { assert(false); } + //-------------------------------------------------------------------------- + void DefaultMapper::select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + log_mapper.spew("Default select_sharding_functor for Task in %s", + get_mapper_name()); + output.chosen_functor = 0; // use the default functor + } + //-------------------------------------------------------------------------- void DefaultMapper::map_inline(const MapperContext ctx, const InlineMapping& inline_op, @@ -2764,6 +2899,19 @@ namespace Legion { assert(false); } + //-------------------------------------------------------------------------- + void DefaultMapper::select_sharding_functor( + const MapperContext ctx, + const Copy& copy, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + log_mapper.spew("Default select_sharding_functor for Copy in %s", + get_mapper_name()); + output.chosen_functor = 0; // use the default functor + } + //-------------------------------------------------------------------------- void DefaultMapper::map_close(const MapperContext ctx, const Close& close, @@ -2851,6 +2999,19 @@ namespace Legion { assert(false); } + //-------------------------------------------------------------------------- + void DefaultMapper::select_sharding_functor( + const MapperContext ctx, + const Close& close, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + log_mapper.spew("Default select_sharding_functor for Close in %s", + get_mapper_name()); + output.chosen_functor = 0; // use the default functor + } + //-------------------------------------------------------------------------- void DefaultMapper::map_acquire(const MapperContext ctx, const Acquire& acquire, @@ -2885,6 +3046,19 @@ namespace Legion { assert(false); } + //-------------------------------------------------------------------------- + void DefaultMapper::select_sharding_functor( + const MapperContext ctx, + const Acquire& acquire, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + log_mapper.spew("Default select_sharding_functor for Acquire in %s", + get_mapper_name()); + output.chosen_functor = 0; // use the default functor + } + //-------------------------------------------------------------------------- void DefaultMapper::map_release(const MapperContext ctx, const Release& release, @@ -2931,6 +3105,19 @@ namespace Legion { assert(false); } + //-------------------------------------------------------------------------- + void DefaultMapper::select_sharding_functor( + const MapperContext ctx, + const Release& release, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + log_mapper.spew("Default select_sharding_functor for Release in %s", + get_mapper_name()); + output.chosen_functor = 0; // use the default functor + } + //-------------------------------------------------------------------------- void DefaultMapper::select_partition_projection(const MapperContext ctx, const Partition& partition, @@ -3051,6 +3238,32 @@ namespace Legion { assert(false); } + //-------------------------------------------------------------------------- + void DefaultMapper::select_sharding_functor( + const MapperContext ctx, + const Partition& partition, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + log_mapper.spew("Default select_sharding_functor for Partition in %s", + get_mapper_name()); + output.chosen_functor = 0; // use the default functor + } + + //-------------------------------------------------------------------------- + void DefaultMapper::select_sharding_functor( + const MapperContext ctx, + const Fill& fill, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + log_mapper.spew("Default select_sharding_functor for Fill in %s", + get_mapper_name()); + output.chosen_functor = 0; // use the default functor + } + //-------------------------------------------------------------------------- void DefaultMapper::configure_context(const MapperContext ctx, const Task& task, @@ -3163,6 +3376,56 @@ namespace Legion { return false; } + //-------------------------------------------------------------------------- + const std::vector& DefaultMapper::local_procs_by_kind( + Processor::Kind kind) + //-------------------------------------------------------------------------- + { + switch (kind) + { + case Processor::LOC_PROC: + return local_cpus; + case Processor::TOC_PROC: + return local_gpus; + case Processor::IO_PROC: + return local_ios; + case Processor::PROC_SET: + return local_procsets; + case Processor::OMP_PROC: + return local_omps; + case Processor::PY_PROC: + return local_pys; + default: + assert(0); + } + return local_cpus; + } + + //-------------------------------------------------------------------------- + const std::vector& DefaultMapper::remote_procs_by_kind( + Processor::Kind kind) + //-------------------------------------------------------------------------- + { + switch (kind) + { + case Processor::LOC_PROC: + return remote_cpus; + case Processor::TOC_PROC: + return remote_gpus; + case Processor::IO_PROC: + return remote_ios; + case Processor::PROC_SET: + return remote_procsets; + case Processor::OMP_PROC: + return remote_omps; + case Processor::PY_PROC: + return remote_pys; + default: + assert(0); + } + return remote_cpus; + } + //-------------------------------------------------------------------------- bool DefaultMapper::default_policy_select_must_epoch_processors( MapperContext ctx, @@ -3257,9 +3520,12 @@ namespace Legion { << " tasks to AS " << curr_as->first; // assign tasks in this group to processors in this space - for(std::set::const_iterator it2 = it->begin(); - it2 != it->end(); - ++it2) { + // we sort them by their index point in order to ensure that + // this process is deterministic for dynamic control replication + std::vector point_tasks(it->begin(), it->end()); + std::sort(point_tasks.begin(), point_tasks.end(), point_sort_func); + for (std::vector::const_iterator it2 = + point_tasks.begin(); it2 != point_tasks.end(); ++it2) { target_procs[*it2] = curr_as->second.front(); curr_as->second.pop_front(); n_left--; @@ -3345,6 +3611,23 @@ namespace Legion { return target_memory; } + //-------------------------------------------------------------------------- + void DefaultMapper::select_sharding_functor( + const MapperContext ctx, + const MustEpoch& epoch, + const SelectShardingFunctorInput& input, + MustEpochShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + log_mapper.spew("Default select_sharding_functor for Must Epoch in %s", + get_mapper_name()); + output.chosen_functor = 0; // use the default functor + // The default mapper currently doesn't support a collective + // map_must_epoch call as its algorithm requires global information + // about the must epoch launch to work correctly + output.collective_map_must_epoch_call = false; + } + //-------------------------------------------------------------------------- void DefaultMapper::map_must_epoch(const MapperContext ctx, const MapMustEpochInput& input, diff --git a/runtime/mappers/default_mapper.h b/runtime/mappers/default_mapper.h index f33042abde..4cbca3a9ab 100644 --- a/runtime/mappers/default_mapper.h +++ b/runtime/mappers/default_mapper.h @@ -92,12 +92,14 @@ namespace Legion { struct VariantInfo { public: VariantInfo(void) - : variant(0), tight_bound(false), is_inner(false) { } + : variant(0), tight_bound(false), + is_inner(false), is_replicable(false) { } public: VariantID variant; Processor::Kind proc_kind; bool tight_bound; bool is_inner; + bool is_replicable; }; enum CachedMappingPolicy { @@ -152,6 +154,11 @@ namespace Legion { const Task& task, const MapTaskInput& input, MapTaskOutput& output); + virtual void map_replicate_task(const MapperContext ctx, + const Task& task, + const MapTaskInput& input, + const MapTaskOutput& default_output, + MapReplicateTaskOutput& output); virtual void select_task_variant(const MapperContext ctx, const Task& task, const SelectVariantInput& input, @@ -170,6 +177,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Task& task, const TaskProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Inline mapping calls virtual void map_inline(const MapperContext ctx, const InlineMapping& inline_op, @@ -197,6 +209,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Copy& copy, const CopyProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Copy& copy, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Close mapping calls virtual void map_close(const MapperContext ctx, const Close& close, @@ -209,6 +226,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Close& close, const CloseProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Close& close, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Acquire mapping calls virtual void map_acquire(const MapperContext ctx, const Acquire& acquire, @@ -220,6 +242,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Acquire& acquire, const AcquireProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Acquire& acquire, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Release mapping calls virtual void map_release(const MapperContext ctx, const Release& release, @@ -235,6 +262,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Release& release, const ReleaseProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Release& release, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Partition mapping calls virtual void select_partition_projection(const MapperContext ctx, const Partition& partition, @@ -252,6 +284,17 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Partition& partition, const PartitionProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Partition& partition, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); + public: // Fill mapper calls + virtual void select_sharding_functor( + const MapperContext ctx, + const Fill& fill, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Task execution mapping calls virtual void configure_context(const MapperContext ctx, const Task& task, @@ -261,6 +304,11 @@ namespace Legion { const SelectTunableInput& input, SelectTunableOutput& output); public: // Must epoch mapping + virtual void select_sharding_functor( + const MapperContext ctx, + const MustEpoch& epoch, + const SelectShardingFunctorInput& input, + MustEpochShardingFunctorOutput& output); virtual void map_must_epoch(const MapperContext ctx, const MapMustEpochInput& input, MapMustEpochOutput& output); @@ -422,6 +470,8 @@ namespace Legion { const std::set ®ions); bool have_proc_kind_variant(const MapperContext ctx, TaskID id, Processor::Kind kind); + const std::vector& local_procs_by_kind(Processor::Kind kind); + const std::vector& remote_procs_by_kind(Processor::Kind kind); protected: // static helper methods static const char* create_default_name(Processor p); template @@ -447,7 +497,9 @@ namespace Legion { static inline bool physical_sort_func( const std::pair &left, const std::pair &right) - { return (left.second < right.second); } + { return (left.second < right.second); } + static inline bool point_sort_func(const Task *t1, const Task *t2) + { return (t1->index_point < t2->index_point); } protected: const Processor local_proc; const Processor::Kind local_kind; @@ -521,6 +573,9 @@ namespace Legion { // Whether to map tasks locally // Controlled by -dm:map_locally (false by default) bool map_locally; + // Whether to enable control replication + // Controlled by -dm:replicate (true by default) + bool replication_enabled; }; }; // namespace Mapping diff --git a/runtime/mappers/null_mapper.cc b/runtime/mappers/null_mapper.cc index 850e516034..9b0739c91c 100644 --- a/runtime/mappers/null_mapper.cc +++ b/runtime/mappers/null_mapper.cc @@ -118,6 +118,17 @@ namespace Legion { report_unimplemented(__func__, __LINE__); } + //-------------------------------------------------------------------------- + void NullMapper::map_replicate_task(const MapperContext ctx, + const Task& task, + const MapTaskInput& input, + const MapTaskOutput& default_output, + MapReplicateTaskOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + //-------------------------------------------------------------------------- void NullMapper::select_task_variant(const MapperContext ctx, const Task& task, @@ -177,6 +188,17 @@ namespace Legion { report_unimplemented(__func__, __LINE__); } + //-------------------------------------------------------------------------- + void NullMapper::select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + //-------------------------------------------------------------------------- void NullMapper::map_inline(const MapperContext ctx, const InlineMapping& inline_op, @@ -266,6 +288,17 @@ namespace Legion { report_unimplemented(__func__, __LINE__); } + //-------------------------------------------------------------------------- + void NullMapper::select_sharding_functor( + const MapperContext ctx, + const Copy& copy, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + //-------------------------------------------------------------------------- void NullMapper::map_close(const MapperContext ctx, const Close& close, @@ -306,6 +339,17 @@ namespace Legion { report_unimplemented(__func__, __LINE__); } + //-------------------------------------------------------------------------- + void NullMapper::select_sharding_functor( + const MapperContext ctx, + const Close& close, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + //-------------------------------------------------------------------------- void NullMapper::map_acquire(const MapperContext ctx, const Acquire& acquire, @@ -334,6 +378,17 @@ namespace Legion { report_unimplemented(__func__, __LINE__); } + //-------------------------------------------------------------------------- + void NullMapper::select_sharding_functor( + const MapperContext ctx, + const Acquire& acquire, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + //-------------------------------------------------------------------------- void NullMapper::map_release(const MapperContext ctx, const Release& release, @@ -383,6 +438,17 @@ namespace Legion { report_unimplemented(__func__, __LINE__); } + //-------------------------------------------------------------------------- + void NullMapper::select_sharding_functor( + const MapperContext ctx, + const Release& release, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + //-------------------------------------------------------------------------- void NullMapper::select_partition_projection(const MapperContext ctx, const Partition& partition, @@ -434,6 +500,28 @@ namespace Legion { report_unimplemented(__func__, __LINE__); } + //-------------------------------------------------------------------------- + void NullMapper::select_sharding_functor( + const MapperContext ctx, + const Partition& partition, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + + //-------------------------------------------------------------------------- + void NullMapper::select_sharding_functor( + const MapperContext ctx, + const Fill& fill, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + //-------------------------------------------------------------------------- void NullMapper::configure_context(const MapperContext ctx, const Task& task, @@ -453,6 +541,17 @@ namespace Legion { report_unimplemented(__func__, __LINE__); } + //-------------------------------------------------------------------------- + void NullMapper::select_sharding_functor( + const MapperContext ctx, + const MustEpoch& epoch, + const SelectShardingFunctorInput& input, + MustEpochShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + report_unimplemented(__func__, __LINE__); + } + //-------------------------------------------------------------------------- void NullMapper::map_must_epoch(const MapperContext ctx, const MapMustEpochInput& input, diff --git a/runtime/mappers/null_mapper.h b/runtime/mappers/null_mapper.h index a2b55eb1ac..806cf34e51 100644 --- a/runtime/mappers/null_mapper.h +++ b/runtime/mappers/null_mapper.h @@ -61,6 +61,11 @@ namespace Legion { const Task& task, const MapTaskInput& input, MapTaskOutput& output); + virtual void map_replicate_task(const MapperContext ctx, + const Task& task, + const MapTaskInput& input, + const MapTaskOutput& default_output, + MapReplicateTaskOutput& output); virtual void select_task_variant(const MapperContext ctx, const Task& task, const SelectVariantInput& input, @@ -84,6 +89,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Task& task, const TaskProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Inline mapping calls virtual void map_inline(const MapperContext ctx, const InlineMapping& inline_op, @@ -121,6 +131,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Copy& copy, const CopyProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Copy& copy, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Close mapping calls virtual void map_close(const MapperContext ctx, const Close& close, @@ -138,6 +153,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Close& close, const CloseProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Close& close, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Acquire mapping calls virtual void map_acquire(const MapperContext ctx, const Acquire& acquire, @@ -149,6 +169,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Acquire& acquire, const AcquireProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Acquire& acquire, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Release mapping calls virtual void map_release(const MapperContext ctx, const Release& release, @@ -169,6 +194,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Release& release, const ReleaseProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Release& release, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Partition mapping calls virtual void select_partition_projection(const MapperContext ctx, const Partition& partition, @@ -191,6 +221,17 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Partition& partition, const PartitionProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Partition& partition, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); + public: // Fill mapper calls + virtual void select_sharding_functor( + const MapperContext ctx, + const Fill& fill, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Task execution mapping calls virtual void configure_context(const MapperContext ctx, const Task& task, @@ -200,6 +241,11 @@ namespace Legion { const SelectTunableInput& input, SelectTunableOutput& output); public: // Must epoch mapping + virtual void select_sharding_functor( + const MapperContext ctx, + const MustEpoch& epoch, + const SelectShardingFunctorInput& input, + MustEpochShardingFunctorOutput& output); virtual void map_must_epoch(const MapperContext ctx, const MapMustEpochInput& input, MapMustEpochOutput& output); diff --git a/runtime/mappers/replay_mapper.cc b/runtime/mappers/replay_mapper.cc index cf9e0e8d17..0878a3ec65 100644 --- a/runtime/mappers/replay_mapper.cc +++ b/runtime/mappers/replay_mapper.cc @@ -301,6 +301,17 @@ namespace Legion { output.postmap_task = !mapping->postmappings.empty(); } + //-------------------------------------------------------------------------- + void ReplayMapper::map_replicate_task(const MapperContext ctx, + const Task& task, + const MapTaskInput& input, + const MapTaskOutput& def_output, + MapReplicateTaskOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } + //-------------------------------------------------------------------------- void ReplayMapper::select_task_variant(const MapperContext ctx, const Task& task, @@ -356,6 +367,17 @@ namespace Legion { // Nothing to do here } + //-------------------------------------------------------------------------- + void ReplayMapper::select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } + //-------------------------------------------------------------------------- void ReplayMapper::map_inline(const MapperContext ctx, const InlineMapping& inline_op, @@ -434,6 +456,17 @@ namespace Legion { { // Nothing to do } + + //-------------------------------------------------------------------------- + void ReplayMapper::select_sharding_functor( + const MapperContext ctx, + const Copy& copy, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } //-------------------------------------------------------------------------- void ReplayMapper::map_close(const MapperContext ctx, @@ -466,6 +499,17 @@ namespace Legion { // Nothing to do } + //-------------------------------------------------------------------------- + void ReplayMapper::select_sharding_functor( + const MapperContext ctx, + const Close& close, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } + //-------------------------------------------------------------------------- void ReplayMapper::map_acquire(const MapperContext ctx, const Acquire& acquire, @@ -494,6 +538,17 @@ namespace Legion { // Nothing to do } + //-------------------------------------------------------------------------- + void ReplayMapper::select_sharding_functor( + const MapperContext ctx, + const Acquire& acquire, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } + //-------------------------------------------------------------------------- void ReplayMapper::map_release(const MapperContext ctx, const Release& release, @@ -532,6 +587,17 @@ namespace Legion { // Nothing to do } + //-------------------------------------------------------------------------- + void ReplayMapper::select_sharding_functor( + const MapperContext ctx, + const Release& release, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } + //-------------------------------------------------------------------------- void ReplayMapper::select_partition_projection(const MapperContext ctx, const Partition& partition, @@ -572,6 +638,28 @@ namespace Legion { // Nothing to do } + //-------------------------------------------------------------------------- + void ReplayMapper::select_sharding_functor( + const MapperContext ctx, + const Partition& partition, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } + + //-------------------------------------------------------------------------- + void ReplayMapper::select_sharding_functor( + const MapperContext ctx, + const Fill& fill, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } + //-------------------------------------------------------------------------- void ReplayMapper::configure_context(const MapperContext ctx, const Task& task, @@ -600,6 +688,17 @@ namespace Legion { output.size); } + //-------------------------------------------------------------------------- + void ReplayMapper::select_sharding_functor( + const MapperContext ctx, + const MustEpoch& epoch, + const SelectShardingFunctorInput& input, + MustEpochShardingFunctorOutput& output) + //-------------------------------------------------------------------------- + { + assert(false); // TODO + } + //-------------------------------------------------------------------------- void ReplayMapper::map_must_epoch(const MapperContext ctx, const MapMustEpochInput& input, diff --git a/runtime/mappers/replay_mapper.h b/runtime/mappers/replay_mapper.h index c0e88f8956..91f46c0ee2 100644 --- a/runtime/mappers/replay_mapper.h +++ b/runtime/mappers/replay_mapper.h @@ -146,6 +146,11 @@ namespace Legion { const Task& task, const MapTaskInput& input, MapTaskOutput& output); + virtual void map_replicate_task(const MapperContext ctx, + const Task& task, + const MapTaskInput& input, + const MapTaskOutput& default_output, + MapReplicateTaskOutput& output); virtual void select_task_variant(const MapperContext ctx, const Task& task, const SelectVariantInput& input, @@ -164,6 +169,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Task& task, const TaskProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Task& task, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Inline mapping calls virtual void map_inline(const MapperContext ctx, const InlineMapping& inline_op, @@ -191,6 +201,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Copy& copy, const CopyProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Copy& copy, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Close mapping calls virtual void map_close(const MapperContext ctx, const Close& close, @@ -203,6 +218,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Close& close, const CloseProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Close& close, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Acquire mapping calls virtual void map_acquire(const MapperContext ctx, const Acquire& acquire, @@ -214,6 +234,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Acquire& acquire, const AcquireProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Acquire& acquire, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Release mapping calls virtual void map_release(const MapperContext ctx, const Release& release, @@ -229,6 +254,11 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Release& release, const ReleaseProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Release& release, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Partition mapping calls virtual void select_partition_projection(const MapperContext ctx, const Partition& partition, @@ -246,6 +276,17 @@ namespace Legion { virtual void report_profiling(const MapperContext ctx, const Partition& partition, const PartitionProfilingInfo& input); + virtual void select_sharding_functor( + const MapperContext ctx, + const Partition& partition, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); + public: // Fill mapper calls + virtual void select_sharding_functor( + const MapperContext ctx, + const Fill& fill, + const SelectShardingFunctorInput& input, + SelectShardingFunctorOutput& output); public: // Task execution mapping calls virtual void configure_context(const MapperContext ctx, const Task& task, @@ -255,6 +296,11 @@ namespace Legion { const SelectTunableInput& input, SelectTunableOutput& output); public: // Must epoch mapping + virtual void select_sharding_functor( + const MapperContext ctx, + const MustEpoch& epoch, + const SelectShardingFunctorInput& input, + MustEpochShardingFunctorOutput& output); virtual void map_must_epoch(const MapperContext ctx, const MapMustEpochInput& input, MapMustEpochOutput& output); diff --git a/runtime/realm/indexspace.h b/runtime/realm/indexspace.h index 556a4faddf..a147d09051 100644 --- a/runtime/realm/indexspace.h +++ b/runtime/realm/indexspace.h @@ -283,7 +283,7 @@ namespace Realm { // index-based: Event create_equal_subspace(size_t count, size_t granularity, - unsigned index, IndexSpace &subspace, + unsigned index, IndexSpace& subspace, const ProfilingRequestSet &reqs, Event wait_on = Event::NO_EVENT) const; diff --git a/runtime/realm/logging.h b/runtime/realm/logging.h index ff42541fc3..c002c2ca51 100644 --- a/runtime/realm/logging.h +++ b/runtime/realm/logging.h @@ -154,6 +154,7 @@ namespace Realm { LoggerMessage& vprintf(const char *typeName, LoggerMessageID messageID, const char *fmt, va_list ap); bool is_active(void) const; + void deactivate(void); std::ostream& get_stream(void); diff --git a/runtime/realm/logging.inl b/runtime/realm/logging.inl index b535ff5f06..aa0c44c232 100644 --- a/runtime/realm/logging.inl +++ b/runtime/realm/logging.inl @@ -542,6 +542,11 @@ namespace Realm { { return active; } + + inline void LoggerMessage::deactivate(void) + { + active = false; + } inline std::ostream& LoggerMessage::get_stream(void) { diff --git a/runtime/runtime.mk b/runtime/runtime.mk index 0f4d669e43..63f5c7b0a5 100644 --- a/runtime/runtime.mk +++ b/runtime/runtime.mk @@ -805,6 +805,7 @@ LEGION_SRC += $(LG_RT_DIR)/legion/legion.cc \ $(LG_RT_DIR)/legion/legion_constraint.cc \ $(LG_RT_DIR)/legion/legion_mapping.cc \ $(LG_RT_DIR)/legion/legion_redop.cc \ + $(LG_RT_DIR)/legion/legion_replication.cc \ $(LG_RT_DIR)/legion/region_tree.cc \ $(LG_RT_DIR)/legion/runtime.cc \ $(LG_RT_DIR)/legion/garbage_collection.cc \ diff --git a/test.py b/test.py index abee7180c6..d02e237140 100755 --- a/test.py +++ b/test.py @@ -95,6 +95,7 @@ legion_network_cxx_tests = [ # Examples ['examples/mpi_interop/mpi_interop', []], + ['examples/mpi_with_ctrl_repl/mpi_with_ctrl_repl', []], ] legion_openmp_cxx_tests = [ @@ -378,7 +379,8 @@ def run_test_external(launcher, root_dir, tmp_dir, bin_dir, env, thread_count, t # SNAP # Contact: Mike Bauer snap_dir = os.path.join(tmp_dir, 'snap') - cmd(['git', 'clone', 'https://github.com/StanfordLegion/Legion-SNAP.git', snap_dir]) + # TODO: Merge deppart branch into master after this makes it to stable Legion branch + cmd(['git', 'clone', '-b', 'ctrlrepl', 'https://github.com/StanfordLegion/Legion-SNAP.git', snap_dir]) # This can't handle flags before application arguments, so place # them after. snap = [[os.path.join(snap_dir, 'src/snap'), diff --git a/tools/legion_spy.py b/tools/legion_spy.py index bfc55e94ba..07d8944984 100755 --- a/tools/legion_spy.py +++ b/tools/legion_spy.py @@ -2913,7 +2913,7 @@ def __init__(self, state, iid, fid, tid): self.children = dict() self.name = None self.parent = None - self.logical_state = dict() + self.logical_state = dict() # only for top-level regions self.verification_state = dict() # only for top-level regions self.index_space.add_instance(self.tree_id, self) self.node_name = 'region_node_'+str(self.index_space.uid)+\ @@ -2933,6 +2933,13 @@ def add_child(self, color, child): def has_all_children(self): return len(self.children) == len(self.index_space.children) + def has_ancestor(self, target): + if self is target: + return True + if self.parent is None: + return False + return self.parent.has_ancestor(target) + def get_index_node(self): return self.index_space @@ -2989,7 +2996,7 @@ def get_num_children(self): return self.index_space.get_num_children() def reset_logical_state(self): - if self.logical_state: + if self.logical_state is not None: self.logical_state = dict() def reset_verification_state(self, depth): @@ -3058,6 +3065,35 @@ def close_logical_tree(self, field, closed_users): return self.logical_state[field].close_logical_tree(closed_users) + def get_logical_state(self, field, point): + # Should always be at the root + assert not self.parent + key = (field,point) + if key not in self.logical_state: + result = LogicalVerificationState(self, field, point) + self.logical_state[key] = result + return result + return self.logical_state[key] + + def perform_logical_verification(self, op, req, field, logical_op, + previous_deps, point_set = None): + if point_set is None: + # First get the point set + return self.perform_logical_verification(op, req, field, logical_op, + previous_deps, self.get_point_set()) + elif self.parent: + # Recurse up the tree to the root + return self.parent.parent.perform_logical_verification(op, req, field, + logical_op, previous_deps, point_set) + else: + # Do the actual work + for point in point_set.iterator(): + state = self.get_logical_state(field, point) + if not state.perform_logical_verification(op, req, logical_op, + previous_deps): + return False + return True + def get_verification_state(self, depth, field, point): # Should always be at the root assert not self.parent @@ -3325,6 +3361,12 @@ def add_child(self, color, child): def has_all_children(self): return len(self.children) == len(self.index_partition.children) + def has_ancestor(self, target): + if self is target: + return True + assert self.parent is not None + return self.parent.has_ancestor(target) + def get_index_node(self): return self.index_partition @@ -3445,6 +3487,11 @@ def close_logical_tree(self, field, closed_users): return self.logical_state[field].close_logical_tree(closed_users) + def perform_logical_verification(self, op, req, field, logical_op, + previous_deps): + return self.parent.perform_logical_verification(op, req, field, logical_op, + previous_deps, self.get_point_set()) + def compute_current_version_numbers(self, depth, field, op, tree): self.parent.compute_current_version_numbers(depth, field, op, tree, self.get_point_set()) @@ -3509,6 +3556,89 @@ def print_tree(self): for child in itervalues(self.children): child.print_tree() +class LogicalVerificationState(object): + __slots__ = ['region', 'field', 'point', 'last_write', + 'current_epoch_users', 'previous_epoch_users'] + + def __init__(self, region, field, point): + self.region = region + self.field = field + self.point = point + self.last_write = None + self.current_epoch_users = list() + self.previous_epoch_users = list() + + def perform_logical_verification(self, op, req, logical_op, previous_deps): + dominates,success = self.perform_epoch_analysis(self.current_epoch_users, + op, req, logical_op, previous_deps) + if not success: + return False + if not dominates: + _,success = self.perform_epoch_analysis(self.previous_epoch_users, + op, req, logical_op, previous_deps) + if not success: + return False + else: + self.previous_epoch_users = self.current_epoch_users + self.current_epoch_users = list() + self.current_epoch_users.append((op,req,logical_op)) + if req.is_write(): + self.last_write = req.logical_node + return True + + def perform_epoch_analysis(self, epoch_users, op, req, logical_op, previous_deps): + dominates = True + for prev_op,prev_req,prev_log in epoch_users: + dep_type = compute_dependence_type(prev_req, req) + if dep_type is NO_DEPENDENCE: + dominates = False + continue + # Interfering operations should have been caught earlier + assert prev_op is not op + assert prev_log is not logical_op + # Deletions do no need close operations for now + if op.kind == DELETION_OP_KIND: + # Deletions of the same thing can race + if prev_op.kind == DELETION_OP_KIND: + continue + need_close = False + # Now determine whether we need to have a close operation along the + # path between these two operations + elif prev_op.owner_shard != op.owner_shard and prev_op is not prev_log: + # Operations from two different shards with control + # replication always need a close operation if the + # first one is an index space operation. If the first + # one is an individual operation then we know its mapping + # dependences are explicitly broadcast across the shards + # Note that the converse is not true, individual ops do + # not perform collective incoming dependences + need_close = True + elif req.is_read_only(): + # If we're reading from a reduction we always need a close + if prev_req.is_reduce(): + need_close = not req.logical_node.has_ancestor(prev_req.logical_node) + else: + # Otherwise if we're not reading below the most + # recent write then we also need a close + need_close = not req.logical_node.has_ancestor(self.last_write) + elif req.is_reduce(): + # If we're reducing over a read we always need a close + if prev_req.is_read_only(): + need_close = not req.logical_node.has_ancestor(prev_req.logical_node) + else: + # Otherwise if we're not reducing below the most + # recent write then we also need a close + need_close = not req.logical_node.has_ancestor(self.last_write) + else: + # If the previous thing is not directly above use then we need a close + need_close = not req.logical_node.has_ancestor(prev_req.logical_node) + if not logical_op.has_verification_mapping_dependence( + logical_op.reqs[req.index], prev_log, prev_log.reqs[prev_req.index], + dep_type, self.field, need_close,previous_deps): + return dominates,False + return dominates,True + + class LogicalState(object): __slots__ = ['node', 'field', 'open_children', 'open_redop', 'current_epoch_users', 'previous_epoch_users', @@ -4004,10 +4134,7 @@ def perform_close_checks(self, close, closed_users, op, req, # Check for replays if prev_op is op: # If it is a previous registration of ourself, skip it - # This will only happen during replays - if prev_req.index == req.index: - continue - assert False + continue if perform_checks: if not close.has_mapping_dependence(close_req, prev_op, prev_req, ANTI_DEPENDENCE if prev_req.is_read_only() @@ -5242,12 +5369,12 @@ class Operation(object): 'eq_incoming', 'eq_outgoing', 'eq_privileges', 'start_event', 'finish_event', 'inter_close_ops', 'inlined', 'summary_op', 'task', 'task_id', 'predicate', 'predicate_result', - 'futures', 'index_owner', 'points', 'launch_rect', 'creator', - 'realm_copies', 'realm_fills', 'realm_depparts', 'version_numbers', - 'internal_idx', 'partition_kind', 'partition_node', 'node_name', - 'cluster_name', 'generation', 'transitive_warning_issued', + 'futures', 'owner_shard', 'index_owner', 'points', 'launch_rect', + 'creator', 'realm_copies', 'realm_fills', 'realm_depparts', + 'version_numbers', 'internal_idx', 'partition_kind', 'partition_node', + 'node_name', 'cluster_name', 'generation', 'transitive_warning_issued', 'arrival_barriers', 'wait_barriers', 'created_futures', 'used_futures', - 'intra_space_dependences', 'merged', "replayed"] + 'intra_space_dependences', 'merged', 'replayed'] # If you add a field here, you must update the merge method def __init__(self, state, uid): self.state = state @@ -5279,6 +5406,7 @@ def __init__(self, state, uid): self.predicate = None self.predicate_result = True self.futures = None + self.owner_shard = None # Only valid for tasks self.task = None self.task_id = -1 @@ -5328,6 +5456,9 @@ def set_name(self, name): for point in itervalues(self.points): point.set_name(name) + def is_index_op(self): + return self.launch_rect is not None + def __str__(self): if self.name is None: return OpNames[self.kind] + " " + str(self.uid) @@ -5514,6 +5645,10 @@ def set_predicate_result(self, result): if not result: self.fully_logged = True + def set_owner_shard(self, shard): + assert self.owner_shard is None + self.owner_shard = shard + def add_future(self, future): if not self.futures: self.futures = set() @@ -5614,7 +5749,7 @@ def get_equivalence_privileges(self): key = (point,field,req.tid) if key not in self.eq_privileges: self.eq_privileges[key] = req.priv - elif self.launch_rect is None: + elif not self.is_index_op(): # If we have aliased region requirements # then they shouldn't interfere with each other # However, some privileges can appear to interfere @@ -6178,6 +6313,35 @@ def analyze_logical_requirement(self, index, perform_checks): req.priv = REDUCE return True + def verify_logical_requirement(self, index, logical_op, previous_deps): + assert index in self.reqs + req = self.reqs[index] + # Special out for no access + if req.priv is NO_ACCESS: + return True + assert index in logical_op.reqs + # Destination requirements for copies are a little weird because + # they actually need to behave like READ_WRITE privileges + if self.kind == COPY_OP_KIND and len(self.reqs)/2 <= index: + if req.priv == REDUCE: + copy_reduce = True + req.priv = READ_WRITE + else: + copy_reduce = False + else: + copy_reduce = False + # See if we are restricted in any way + assert logical_op.context + # Now do the traversal for each of the fields + for field in req.fields: + if not req.logical_node.perform_logical_verification(self, req, field, + logical_op, previous_deps): + return False + # Restore the privileges if necessary + if copy_reduce: + req.priv = REDUCE + return True + def analyze_logical_fence(self, perform_checks): # Find all the operations since the previous fence and then make sure # we either depend on them directly or we have a transitive dependence @@ -6271,6 +6435,51 @@ def perform_logical_analysis(self, perform_checks): return False return True + def perform_op_logical_verification(self, logical_op, previous_deps): + if self.replayed: + return True + # We need a context to do this + assert logical_op.context is not None + # If this operation was predicated false, then there is nothing to do + if self.predicate and not self.predicate_result: + return True + # See if there is a fence in place for this context + if logical_op.context.current_fence is not None: + if logical_op.context.current_fence not in logical_op.logical_incoming: + print("ERROR: missing logical fence dependence between "+ + str(logical_op.context.current_fence)+" and "+str(logical_op)) + if self.state.assert_on_error: + assert False + return False + if self.reqs is None: + # If this is a fence, check or record dependences on everything from + # either the begining or from the previous fence + if self.kind == FENCE_OP_KIND: + assert logical_op is self + # Record dependences on all the users in the region tree + if not self.analyze_logical_fence(True): + return False + # Finally record ourselves as the next fence + logical_op.context.current_fence = self + elif self.points is not None: + # For index space operations we'll perform all their operations + # separately so everything gets updated individually + if self.kind == INDEX_TASK_KIND: + for point in sorted(itervalues(self.points), key=lambda x: x.op.uid): + if not point.op.perform_op_logical_verification(logical_op, previous_deps): + return False + else: + for point in sorted(itervalues(self.points), key=lambda x: x.uid): + if not point.perform_op_logical_verification(logical_op, previous_deps): + return False + elif self.launch_rect is None: + # This is a single operation if it doesn't have a launch rectangle + assert len(self.reqs) >= len(logical_op.reqs) + for idx in xrange(0,len(logical_op.reqs)): + if not self.verify_logical_requirement(idx, logical_op, previous_deps): + return False + return True + def has_mapping_dependence(self, req, prev_op, prev_req, dtype, field): if self.incoming: for dep in self.incoming: @@ -6324,6 +6533,133 @@ def has_transitive_mapping_dependence(self, prev_op): queue.append(next_op) return False + def has_verification_mapping_dependence(self, req, prev_op, prev_req, dtype, + field, need_close, previous_deps): + tree_id = req.logical_node.tree_id + # Do a quick check to see if it is in the previous deps + if prev_op in previous_deps: + if need_close: + # Check to see if the previous dependence had an intermediate close + if (field,tree_id) in previous_deps[prev_op]: + return True + else: + # We already found this prev_op as a previous dependence + return True + self.has_verification_transitive_mapping_dependence(prev_op, need_close, + field, tree_id, previous_deps) + # Did not find it so issue the error and return false + if prev_op not in previous_deps: + print("ERROR: Missing mapping dependence on "+str(field)+" between region "+ + "requirement "+str(prev_req.index)+" of "+str(prev_op)+" (UID "+ + str(prev_op.uid)+") and region requriement "+str(req.index)+" of "+ + str(self)+" (UID "+str(self.uid)+")") + if self.state.assert_on_error: + assert False + elif need_close and (field,tree_id) not in previous_deps[prev_op]: + assert need_close + print("ERROR: Missing close operation on "+str(field)+" of tree "+ + str(tree_id)+" between region requirement "+str(prev_req.index)+ + " of "+str(prev_op)+" (UID "+str(prev_op.uid)+") and region "+ + "requriement "+str(req.index)+" of "+str(self)+" (UID "+ + str(self.uid)+")") + if self.state.assert_on_error: + assert False + else: + return True + return False + + def has_verification_transitive_mapping_dependence(self, prev_op, need_close, + field, tree_id, previous_deps): + # If we don't need a close then we can do BFS which is much more efficient + # at finding dependences of things nearby in the graph + if not need_close: + next_gen = self.state.get_next_traversal_generation() + self.generation = next_gen + queue = collections.deque() + queue.append(self) + while queue: + current = queue.popleft() + if not current in previous_deps: + previous_deps[current] = set() + if current is prev_op: + return True + if not current.logical_incoming: + continue + for next_op in current.logical_incoming: + if next_op.generation == next_gen: + continue + next_op.generation = next_gen + queue.append(next_op) + else: + # Otherwise start the traversal and look for a path that contains the close + # Since it's likely that the operation we're looking for is nearby in the + # graph we'll use a timeout to find it efficiently search ever increasing + # depths until we notice that we do not get a timeout + max_depth = 4 + close_map = dict() + timeout_counter = list() + timeout_counter.append(0) + while True: + timeout_counter[0] = 0 + assert len(close_map) == 0 + if self.has_mapping_dependence_with_close(prev_op, field, tree_id, + previous_deps, close_map, max_depth, timeout_counter): + return True + # If we didn't have any timeout it means we explore the whole graph + elif timeout_counter[0] == 0: + break; + # Increase the depth for the next pass + max_depth *= 2 + return False + + def has_mapping_dependence_with_close(self, prev_op, field, tree_id, + previous_deps, close_map, timeout, timeout_counter): + # Record our dependence set in the previous deps + if not self.kind == INTER_CLOSE_OP_KIND: + if self not in previous_deps: + previous_deps[self] = set() + if close_map is not None and len(close_map) > 0: + for key in iterkeys(close_map): + previous_deps[self].add(key) + # See if we arrived at the node we're looking for + if self is prev_op: + # Check to see if we have a close for our field and tree + if (field,tree_id) in close_map: + return True + else: + return False + # If there's no where to traverse we're done + if not self.logical_incoming: + return False + # If we timed out record it in the count + if timeout <= 0: + timeout_counter[0] += 1 + return False + # Otherwise continue the traversal + if self.kind == INTER_CLOSE_OP_KIND: + # If we're a close op, add ourselves to the close map + assert self.reqs is not None and len(self.reqs) == 1 + close_tid = self.reqs[0].logical_node.tree_id + for close_field in self.reqs[0].fields: + key = (close_field,close_tid) + if key in close_map: + close_map[key] += 1 + else: + close_map[key] = 1 + for next_op in self.logical_incoming: + if next_op.has_mapping_dependence_with_close(prev_op, field, tree_id, + previous_deps, close_map, timeout-1, timeout_counter): + # No need to prune close map entries, this is the fast path back + return True + if self.kind == INTER_CLOSE_OP_KIND: + # Remove our entries on the way back + for close_field in self.reqs[0].fields: + key = (close_field,close_tid) + close_map[key] -= 1 + if close_map[key] == 0: + del close_map[key] + return False + def analyze_previous_interference(self, next_op, next_req, reachable): if not self.reqs: # Check to see if this is a fence operation @@ -6892,7 +7228,8 @@ def remove_restriction(self, index, req, perform_checks): return True def verify_physical_requirement(self, index, req, perform_checks): - if req.is_no_access() or len(req.fields) == 0: + # We can end up with no mappings in control replicated cases + if req.is_no_access() or len(req.fields) == 0 or self.mappings is None: return True assert index in self.mappings mappings = self.mappings[index] @@ -6946,22 +7283,29 @@ def perform_op_physical_verification(self, perform_checks): # If we were predicated false, then there is nothing to do if not self.predicate_result: return True + # If we're part of a control replication environment only do + # this operation if we are the owner + if self.owner_shard is not None: + assert self.context.shard is not None + if self.owner_shard != self.context.shard: + return True prefix = '' if self.context: depth = self.context.get_depth() for idx in xrange(depth): prefix += ' ' # If we are an index space task, only do our points - if self.kind == INDEX_TASK_KIND: + if self.kind == INDEX_TASK_KIND and self.points: for point in itervalues(self.points): if not point.op.perform_op_physical_verification(perform_checks): return False return True # Handle other index space operations too - elif self.points: - for point in sorted(itervalues(self.points), key=lambda x: x.uid): - if not point.perform_op_physical_verification(perform_checks): - return False + elif self.is_index_op(): + if self.points: + for point in sorted(itervalues(self.points), key=lambda x: x.uid): + if not point.perform_op_physical_verification(perform_checks): + return False return True if perform_checks: print((prefix+"Performing physical verification analysis "+ @@ -7017,6 +7361,37 @@ def perform_op_physical_verification(self, perform_checks): elif self.kind == DELETION_OP_KIND: # Skip deletions, they only impact logical analysis pass + elif self.task and self.task.replicants: + # Special case for if we are (control) replicated + if self.reqs is not None: + self.compute_current_version_numbers() + assert self.mapping is None + # We have to do verification for all our replicatnts first + for shard in itervalues(self.task.replicants.shards): + self.mapping = shard.op.mapping + for index,req in iteritems(self.reqs): + if not self.verify_physical_requirement(index, req, perform_checks): + return False + self.mapping = None + # Then we do the registration for all our replicants + for shard in itervalues(self.task.replicants.shards): + self.mapping = shard.op.mapping + for index,req in iteritems(self.reqs): + if not self.perform_verification_registration(index, req, + perform_checks): + return False + self.mapping = None + # Last decided how to analyze each of the shards depending + # on whether we are control replicated or not + if self.task.replicants.control_replicated: + # Traverse it like a single logical task + if not self.task.perform_task_physical_verification(perform_checks): + return False + else: + # Can verify each of these separately + for shard in itervalues(self.task.replicants.shards): + if not shard.perform_task_physical_verification(perform_checks): + return False else: if self.reqs: # Compute our version numbers first @@ -7189,6 +7564,11 @@ def print_event_graph(self, printer, elevate, all_nodes, top): # If we were predicated false then we don't get printed if not self.predicate_result: return + # If this is in a control replication context see if we should print ourself + if self.owner_shard is not None: + assert self.context.shard is not None + if self.owner_shard != self.context.shard: + return # Do any of our close operations too if self.inter_close_ops: for close in self.inter_close_ops: @@ -7390,9 +7770,9 @@ def set_invertible(self, invertible): class Task(object): __slots__ = ['state', 'op', 'point', 'operations', 'depth', - 'current_fence', 'used_instances', 'virtual_indexes', - 'processor', 'priority', 'premappings', 'postmappings', - 'tunables', 'operation_indexes', 'close_indexes', 'variant'] + 'current_fence', 'used_instances', 'virtual_indexes', 'processor', + 'priority', 'premappings', 'postmappings', 'tunables', + 'operation_indexes', 'close_indexes', 'variant', 'replicants', 'shard'] # If you add a field here, you must update the merge method def __init__(self, state, op): self.state = state @@ -7413,10 +7793,14 @@ def __init__(self, state, op): self.operation_indexes = None self.close_indexes = None self.variant = None + self.replicants = None + self.shard = None def __str__(self): if self.op is None: return "Root context" + elif self.shard is not None: + return str(self.op)+" (Shard "+str(self.shard)+")" else: return str(self.op) @@ -7440,6 +7824,11 @@ def set_variant(self, variant): assert not self.variant self.variant = variant + def set_shard(self, shard, original): + assert not self.shard + self.shard = shard + self.op.set_context(original, False) + def add_premapping(self, index): if not self.premappings: self.premappings = set() @@ -7554,6 +7943,106 @@ def flatten_summary_operations(self): flattened.append(op) self.operations = flattened + def reset_logical_state(self): + # Just need to reset the fence for now + self.current_fence = None + + def perform_task_logical_verification(self): + # If we are a shard then we don't need to do anything as + # the original version of ourself will do the analysis + if self.shard is not None: + return True + # If we don't have any operations we are done + if not self.operations and self.replicants is None: + return True + # If this is the top-level task's context, we can skip it + # since we know there is only one task in it + if self.depth == 0: + assert len(self.operations) == 1 + return True + print('Performing logical dependence verification for %s...' % str(self)) + success = True + if self.replicants is not None: + # We need to do a verification for the logical analysis in each shard + for logical_shard in itervalues(self.replicants.shards): + print('Verifying shard %s...' % str(logical_shard.shard)) + logical_shard.reset_logical_state() + for idx in xrange(len(logical_shard.operations)): + logical_op = logical_shard.operations[idx] + if logical_op.inlined: + continue + if not logical_op.fully_logged: + print(('Warning: shard %s has operation %s which is '+ + 'not fully logged and therefore being skipped. '+ + 'This is likely the result of a crash in a run.') % + (str(logical_shard.shard),str(logical_op))) + if logical_op.state.assert_on_warning: + assert False + continue + # Run this analysis for all the points from each shard + for shard in itervalues(self.replicants.shards): + # Handle cases where shards have different numbers + # of operations because of a crash + if idx >= len(shard.operations): + continue + op = shard.operations[idx] + if op.is_index_op(): + if op.points is not None: + if op.kind == INDEX_TASK_KIND: + for point in itervalues(op.points): + if not point.op.fully_logged: + assert not op.fully_logged + break + point.op.owner_shard = shard.shard + else: + for point in itervalues(op.points): + if not point.fully_logged: + assert not op.fully_logged + break + point.owner_shard = shard.shard + elif shard is not logical_shard: + # Skip individual operations not from our shard + # as we only need to verify individual operations + # once in each logical dependence pattern + continue + if not op.fully_logged: + print(('Warning: shard %s has operation %s which is '+ + 'not fully logged and therefore being skipped. '+ + 'This is likely the result of a crash in a run.') % + (str(shard.shard),str(op))) + if op.state.assert_on_warning: + assert False + continue + # Can finally do the verification for this shard + print('Verifying '+str(op)+' of shard '+str(shard.shard)) + previous_deps = dict() + if not op.perform_op_logical_verification(logical_op,previous_deps): + success = False + break + if not success: + break + # Clear out the logical analysis for the next shard + self.op.state.reset_logical_state() + if not success: + break + else: + if self.op.state.verbose: + print(' Analyzing %d operations...' % len(self.operations)) + self.reset_logical_state() + # Iterate over all the operations in order and + # have them perform their analysis + for op in self.operations: + # Keep track of the previous dependences so we can + # use them for adding/checking dependences on close operations + previous_deps = dict() + if not op.perform_op_logical_verification(op, previous_deps): + success = False + break + # Reset the logical state when we are done + self.op.state.reset_logical_state() + print("Pass" if success else "FAIL") + return success + def perform_logical_dependence_analysis(self, perform_checks): # If we don't have any operations we are done if not self.operations: @@ -7586,43 +8075,6 @@ def perform_logical_dependence_analysis(self, perform_checks): print("Pass" if success else "FAIL") return success - def perform_logical_sanity_analysis(self): - # Run the old version of the checks that - # is more of a sanity check on our algorithm that - # doesn't depend on our implementation but doesn't - # really tell us what it means if something goes wrong - if not self.operations or len(self.operations) < 2: - return True - print('Performing logical sanity analysis for %s...' % str(self)) - # Iterate over all operations from 1 to N and check all their - # dependences against all the previous operations in the context - for idx in xrange(1, len(self.operations)): - # Find all the backwards reachable operations - current_op = self.operations[idx] - # No need to do anything if there are no region requirements - if not current_op.reqs and current_op.kind != FENCE_OP_KIND: - continue - reachable = set() - current_op.get_logical_reachable(reachable, False) - # Do something special for fence operations - if current_op.kind == FENCE_OP_KIND: # special path for fences - for prev in xrange(idx): - if not prev in reachable: - print("ERROR: Failed logical sanity check. No mapping "+ - "dependence between previous "+str(prev)+" and "+ - "later "+str(current_op)) - if self.op.state.assert_on_error: - assert False - return False - else: # The normal path - for prev in xrange(idx): - if not current_op.analyze_logical_interference( - self.operations[prev], reachable): - print("FAIL") - return False - print("Pass") - return True - def find_enclosing_context_depth(self, child_req, mappings): # Special case for the top-level task depth = self.get_depth() @@ -7664,19 +8116,22 @@ def find_enclosing_context_depth(self, child_req, mappings): def perform_task_physical_verification(self, perform_checks): if not self.operations: - return True + if not self.replicants: + return True + assert self.replicants.control_replicated # Depth is a proxy for context depth = self.get_depth() assert self.used_instances is None self.used_instances = set() # Initialize any regions that we mapped if self.op.reqs: - for idx,req in iteritems(self.op.reqs): + # A small helper function for initializing a requirement state + def initialize_requirement(task, idx, req): # Skip any no access requirements if req.is_no_access() or len(req.fields) == 0: - continue - assert idx in self.op.mappings - mappings = self.op.mappings[idx] + return + assert idx in task.op.mappings + mappings = task.op.mappings[idx] # If we are doing restricted analysis then add any restrictions # We treat all reduction instances as restricted to eagerly flush # back reductions to this instance for now @@ -7689,21 +8144,61 @@ def perform_task_physical_verification(self, perform_checks): if inst.is_virtual(): assert not add_restrictions # Better not be virtual if restricted continue - req.logical_node.initialize_verification_state(depth, field, inst, + req.logical_node.initialize_verification_state(depth, field, inst, add_restrictions) + if self.replicants: + # Control replicated path + for shard in itervalues(self.replicants.shards): + for idx,req in iteritems(shard.op.reqs): + initialize_requirement(shard, idx, req) + else: + # Normal path for non-control replicated + for idx,req in iteritems(self.op.reqs): + initialize_requirement(self, idx, req) success = True - for op in self.operations: - if op.inlined: - continue - if not op.fully_logged: - print(('Warning: skipping physical verification of %s '+ - 'because it was not fully logged...') % str(op)) - if op.state.assert_on_warning: - assert False - continue - if not op.perform_op_physical_verification(perform_checks): - success = False - break + if self.replicants: + # Control-replicated path + num_ops = -1 + for shard in itervalues(self.replicants.shards): + shard_ops = 0 + for op in shard.operations: + if not op.fully_logged: + break + else: + shard_ops += 1 + if num_ops == -1: + num_ops = shard_ops + elif num_ops != shard_ops: + print(('Warning: shard %s has %s operations which is '+ + 'different than %s operations in other shards. '+ + 'This is likely the result of a crash in a run.') % + (str(shard.shard),str(shard_ops),str(num_ops))) + if self.state.assert_on_warning: + assert False + num_ops = min(shard_ops,num_ops) + # Perform all the operations in order across the shards + for idx in range(num_ops): + for shard in itervalues(self.replicants.shards): + op = shard.operations[idx] + if not op.perform_op_physical_verification(perform_checks): + success = False + break + if not success: + break + else: + # Normal path + for op in self.operations: + if op.inlined: + continue + if not op.fully_logged: + print(('Warning: skipping physical verification of %s '+ + 'because it was not fully logged...') % str(op)) + if op.state.assert_on_warning: + assert False + continue + if not op.perform_op_physical_verification(perform_checks): + success = False + break # Reset any physical user lists at our depth for inst,fid in self.used_instances: inst.reset_verification_users(depth) @@ -7835,7 +8330,48 @@ def print_dataflow_graph(self, path, simplify_graphs, zoom_graphs): def print_event_graph_context(self, printer, elevate, all_nodes, top): if not self.operations: - return + # Check to see if we were replicated + if self.replicants: + # If we're control replicated we need to alias all the single + # operations across shards so they have the same operation name + num_ops = -1 + for shard in itervalues(self.replicants.shards): + shard_ops = 0 + for op in shard.operations: + if not op.fully_logged: + break + else: + shard_ops += 1 + if num_ops == -1: + num_ops = shard_ops + elif num_ops != shard_ops: + print(('Warning: shard %s has %s operations which is '+ + 'different than %s operations in other shards. '+ + 'This is likely the result of a crash in a run.') % + (str(shard.shard),str(shard_ops),str(num_ops))) + if self.state.assert_on_warning: + assert False + num_ops = min(shard_ops,num_ops) + for idx in range(num_ops): + owner_op = None + # See if we have an owner op + for shard in itervalues(self.replicants.shards): + op = shard.operations[idx] + if op.owner_shard is not None and \ + op.owner_shard == op.context.shard: + owner_op = op + break + # We should only have owner ops for single operations + if owner_op is not None: + # Alias all the node names to the owner node name + for shard in itervalues(self.replicants.shards): + op = shard.operations[idx] + if op is not owner_op: + op.node_name = owner_op.node_name + # Now we can do the normal event graph print routine + for shard in itervalues(self.replicants.shards): + shard.print_event_graph_context(printer, elevate, all_nodes, top) + return if not top: # Start the cluster title = str(self)+' (UID: '+str(self.op.uid)+')' @@ -8086,6 +8622,30 @@ def is_simult(self): def is_relaxed(self): return self.coher == RELAXED +class Replicants(object): + __slots__ = ['repl', 'orig', 'shards', 'control_replicated'] + def __init__(self, repl): + self.repl = repl + self.orig = None + self.shards = dict() + self.control_replicated = None + + def set_original(self, orig, ctrl): + assert not self.orig + self.orig = orig + self.control_replicated = ctrl + + def add_shard(self, sid, shard): + assert sid not in self.shards + self.shards[sid] = shard + + def update_shards(self): + assert self.orig + self.orig.replicants = self + for sid,shard in iteritems(self.shards): + shard.set_shard(sid, self.orig) + shard.merge(self.orig) + class SpecializedConstraint(object): __slots__ = ['kind', 'redop'] def __init__(self, kind, redop): @@ -9963,6 +10523,12 @@ def generate_html_op_label(self, title, requirements, mappings, color, detailed) prefix+"Point Point (?P[0-9]+) (?P[0-9]+)") index_point_pat = re.compile( prefix+"Index Point (?P[0-9]+) (?P[0-9]+) (?P[0-9]+) (?P.*)") +replicate_pat = re.compile( + prefix+"Replicate Task (?P[0-9]+) (?P[0-9]+) (?P[0-1])") +shard_pat = re.compile( + prefix+"Replicate Shard (?P[0-9]+) (?P[0-9]+) (?P[0-9]+)") +owner_shard_pat = re.compile( + prefix+"Owner Shard (?P[0-9]+) (?P[0-9]+)") intra_space_pat = re.compile( prefix+"Intra Space Dependence (?P[0-9]+) (?P[0-9]+) (?P.*)") op_index_pat = re.compile( @@ -10750,6 +11316,23 @@ def parse_legion_spy_line(line, state): index = state.get_operation(int(m.group('index'))) index.add_point_op(point, index_point) return True + m = replicate_pat.match(line) + if m is not None: + repl = state.get_repl(int(m.group('repl'))) + repl.set_original(state.get_task(int(m.group('uid'))), + True if int(m.group('ctrl')) == 1 else False) + return True + m = shard_pat.match(line) + if m is not None: + repl = state.get_repl(int(m.group('repl'))) + task = state.get_task(int(m.group('uid'))) + repl.add_shard(int(m.group('shard')), task) + return True + m = owner_shard_pat.match(line) + if m is not None: + op = state.get_operation(int(m.group('uid'))) + op.set_owner_shard(int(m.group('shard'))) + return True m = intra_space_pat.match(line) if m is not None: point = state.get_operation(int(m.group('point'))) @@ -10978,14 +11561,14 @@ class State(object): __slots__ = ['temp_dir', 'verbose', 'top_level_uid', 'top_level_ctx_uid', 'traverser_gen', 'processors', 'memories', 'processor_kinds', 'memory_kinds', 'index_exprs', 'index_spaces', - 'index_partitions', 'field_spaces', 'regions', 'partitions', 'top_spaces', + 'index_partitions', 'field_spaces', 'regions', 'partitions', 'top_spaces', 'trees', 'ops', 'unique_ops', 'tasks', 'task_names', 'variants', 'projection_functions', 'has_mapping_deps', 'instances', 'events', 'copies', 'fills', 'depparts', 'indirections', 'no_event', 'slice_index', 'slice_slice', 'point_slice', 'point_point', 'futures', 'next_generation', - 'next_realm_num', 'next_indirections_num', 'detailed_graphs', + 'next_realm_num', 'next_indirections_num', 'detailed_graphs', 'assert_on_error', 'assert_on_warning', 'eq_graph_on_error', 'config', - 'detailed_logging'] + 'detailed_logging', 'replicants'] def __init__(self, temp_dir, verbose, details, assert_on_error, assert_on_warning, eq_graph_on_error): self.temp_dir = temp_dir @@ -11035,6 +11618,7 @@ def __init__(self, temp_dir, verbose, details, assert_on_error, self.point_slice = dict() self.point_point = dict() self.futures = dict() + self.replicants = dict() # For physical traversals self.next_generation = 1 self.next_realm_num = 1 @@ -11133,6 +11717,9 @@ def post_parse(self, simplify_graphs, need_physical): # Flatten summary operations in each context for task in itervalues(self.tasks): task.flatten_summary_operations() + # Hook up any replicated tasks + for replicant in itervalues(self.replicants): + replicant.update_shards() # Create the unique set of operations self.unique_ops = set(itervalues(self.ops)) # Add implicit dependencies between point and index operations @@ -11208,6 +11795,7 @@ def post_parse(self, simplify_graphs, need_physical): print("Found %d region trees" % len(self.trees)) print("") print("Found %d tasks" % len(self.tasks)) + print("Found %d replicated task" % len(self.replicants)) print("Found %d operations (including tasks)" % len(self.ops)) print("") print("Found %d instances" % len(self.instances)) @@ -11561,19 +12149,17 @@ def has_aliased_ancestor_tree_only(self, one, two): return (True,parent_one) def perform_logical_analysis(self, perform_checks, sanity_checks): - # Run the full analysis first, this will confirm that - # the runtime did what we thought it should do for task in itervalues(self.tasks): # If we're only performing checks then we might break out early if not task.perform_logical_dependence_analysis(perform_checks): return False - # If we're doing full on sanity checks, run them now - if perform_checks and sanity_checks: - if not task.perform_logical_sanity_analysis(): - return False + if sanity_checks: + for task in itervalues(self.tasks): + if not task.perform_task_logical_verification(): + return False return True - def perform_physical_analysis(self, perform_checks, sanity_checks): + def perform_physical_analysis(self, perform_checks): assert self.top_level_uid is not None top_task = self.get_task(self.top_level_uid) if perform_checks: @@ -11950,6 +12536,13 @@ def get_task(self, uid): self.tasks[op] = result return result + def get_repl(self, repl): + if repl in self.replicants: + return self.replicants[repl] + result = Replicants(repl) + self.replicants[repl] = result + return result + def get_future(self, iid): if iid in self.futures: return self.futures[iid] @@ -12065,14 +12658,6 @@ def reset_logical_state(self): # Definitely run the garbage collector here gc.collect() - def reset_physical_state(self, depth): - for region in itervalues(self.regions): - region.reset_physical_state(depth) - for partition in itervalues(self.partitions): - partition.reset_physical_state(depth) - # Definitely run the garbage collector here - gc.collect() - def reset_verification_state(self, depth): for region in itervalues(self.trees): region.reset_verification_state(depth) @@ -12353,7 +12938,7 @@ def error(self, message): else: # Doing verification so we still need the equivalence class graphs state.compute_equivalence_graphs() - state.perform_physical_analysis(physical_checks, sanity_checks) + state.perform_physical_analysis(physical_checks) # If we generated the graph for printing, then simplify it if need_physical and simplify_graphs: state.simplify_physical_graph(need_cycle_check=False)