From 6e5e374613c913d27af53cb5057933f3addba4e7 Mon Sep 17 00:00:00 2001 From: Florian Lackner Date: Wed, 26 Jan 2022 14:20:18 +0100 Subject: [PATCH 1/3] [SYSTEMDS-3280] Add homomorphic encryption functionality to Parameter Server. This patch adds homomorphic encryption functionality to ParameterServer. It allows a federated parameter server to encrypt the data of each client and do the accumulation step using this encrypted data and homomorphic operations. The data never leaves the clients in plaintext. --- src/main/cpp/build.bat | 4 + src/main/cpp/build.sh | 5 + src/main/cpp/he/CMakeLists.txt | 64 ++++ src/main/cpp/he/he.cpp | 279 +++++++++++++++ src/main/cpp/he/he.h | 111 ++++++ src/main/cpp/he/libhe.cpp | 294 +++++++++++++++ src/main/cpp/he/libhe.h | 144 ++++++++ src/main/cpp/lib/libhe-Linux-x86_64.so | Bin 0 -> 168696 bytes src/main/cpp/systemds.cpp | 8 +- .../java/org/apache/sysds/common/Types.java | 2 +- ...arameterizedBuiltinFunctionExpression.java | 3 +- .../org/apache/sysds/parser/Statement.java | 2 +- .../context/ExecutionContext.java | 17 +- .../federated/FederatedData.java | 2 + .../federated/FederatedLocalData.java | 1 + .../federated/FederatedStatistics.java | 34 ++ .../federated/FederatedWorker.java | 14 + .../federated/FederatedWorkerHandler.java | 45 ++- .../paramserv/FederatedPSControlThread.java | 337 +++++++++++++++--- .../paramserv/HEParamServer.java | 194 ++++++++++ .../paramserv/LocalParamServer.java | 2 +- .../paramserv/NativeHEHelper.java | 100 ++++++ .../paramserv/NetworkTrafficCounter.java | 23 ++ .../controlprogram/paramserv/ParamServer.java | 61 ++-- .../dp/DataPartitionFederatedScheme.java | 1 + .../homomorphicEncryption/PublicKey.java | 36 ++ .../homomorphicEncryption/SEALClient.java | 88 +++++ .../homomorphicEncryption/SEALServer.java | 112 ++++++ .../instructions/cp/CiphertextMatrix.java | 39 ++ .../runtime/instructions/cp/Encrypted.java | 53 +++ .../runtime/instructions/cp/ListObject.java | 27 ++ .../cp/ParamservBuiltinCPInstruction.java | 103 ++++-- .../instructions/cp/PlaintextMatrix.java | 39 ++ .../org/apache/sysds/utils/NativeHelper.java | 92 ++--- .../org/apache/sysds/utils/Statistics.java | 2 + .../utils/stats/ParamServStatistics.java | 51 +++ .../apache/sysds/test/AutomatedTestBase.java | 10 +- .../EncryptedFederatedParamservTest.java | 253 +++++++++++++ .../homomorphicEncryption/InOutTest.java | 116 ++++++ .../EncryptedFederatedParamservTest.dml | 61 ++++ 40 files changed, 2642 insertions(+), 187 deletions(-) create mode 100644 src/main/cpp/he/CMakeLists.txt create mode 100644 src/main/cpp/he/he.cpp create mode 100644 src/main/cpp/he/he.h create mode 100644 src/main/cpp/he/libhe.cpp create mode 100644 src/main/cpp/he/libhe.h create mode 100644 src/main/cpp/lib/libhe-Linux-x86_64.so create mode 100644 src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/HEParamServer.java create mode 100644 src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NativeHEHelper.java create mode 100644 src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NetworkTrafficCounter.java create mode 100644 src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/PublicKey.java create mode 100644 src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALClient.java create mode 100644 src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java create mode 100644 src/main/java/org/apache/sysds/runtime/instructions/cp/CiphertextMatrix.java create mode 100644 src/main/java/org/apache/sysds/runtime/instructions/cp/Encrypted.java create mode 100644 src/main/java/org/apache/sysds/runtime/instructions/cp/PlaintextMatrix.java create mode 100644 src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java create mode 100644 src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java create mode 100644 src/test/scripts/functions/federated/paramserv/EncryptedFederatedParamservTest.dml diff --git a/src/main/cpp/build.bat b/src/main/cpp/build.bat index 93c3819c3be..316c4f393df 100644 --- a/src/main/cpp/build.bat +++ b/src/main/cpp/build.bat @@ -34,5 +34,9 @@ cmake . -B OPENBLAS -DUSE_OPEN_BLAS=ON -DCMAKE_BUILD_TYPE=Release cmake --build OPENBLAS --target install --config Release rmdir /Q /S OPENBLAS +cmake he\ -B HE -DCMAKE_BUILD_TYPE=Release +cmake --build HE --target install --config Release +rmdir /Q /S HE + echo. echo "Make sure to re-run mvn package to make use of the newly compiled libraries" \ No newline at end of file diff --git a/src/main/cpp/build.sh b/src/main/cpp/build.sh index df67aba5392..e40ec895a3c 100755 --- a/src/main/cpp/build.sh +++ b/src/main/cpp/build.sh @@ -66,3 +66,8 @@ ldd lib/libsystemds_mkl-Linux-x86_64.so | grep -v $gcc_toolkit"\|$linux_loader\| echo "Non-standard dependencies for libsystemds_openblas-linux-x86_64.so" ldd lib/libsystemds_openblas-Linux-x86_64.so | grep -v $gcc_toolkit"\|$linux_loader\|"$openblas echo "-----------------------------------------------------------------------" + +# compile HE +cmake he/ -B HE -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++ +cmake --build HE --target install --config Release +rm -R HE \ No newline at end of file diff --git a/src/main/cpp/he/CMakeLists.txt b/src/main/cpp/he/CMakeLists.txt new file mode 100644 index 00000000000..373ba3a5d99 --- /dev/null +++ b/src/main/cpp/he/CMakeLists.txt @@ -0,0 +1,64 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +cmake_minimum_required(VERSION 3.8) +cmake_policy(SET CMP0074 NEW) # make use of _ROOT variable +project (he LANGUAGES CXX) + +# All custom find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") + +# Build a shared libraray +set(HEADER_FILES libhe.h he.h) +set(SOURCE_FILES libhe.cpp he.cpp) + +# Build a shared libraray +add_library(he SHARED ${SOURCE_FILES} ${HEADER_FILES}) + +set_target_properties(he PROPERTIES MACOSX_RPATH 1) + +# sets the installation path to src/main/cpp/lib +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/.." CACHE PATH "sets the installation path to src/main/cpp/lib" FORCE) +endif() + +# sets the installation path to src/main/cpp/lib +# install(TARGETS he LIBRARY DESTINATION lib) +install(TARGETS he RUNTIME DESTINATION lib) + +# unify library filenames to libhe_<...> +if (WIN32) + set(CMAKE_IMPORT_LIBRARY_PREFIX lib CACHE INTERNAL "") + set(CMAKE_SHARED_LIBRARY_PREFIX lib CACHE INTERNAL "") +endif() + +set(CMAKE_BUILD_TYPE Release) +set_target_properties(he PROPERTIES OUTPUT_NAME "he-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}") + +find_package(SEAL 3.7 REQUIRED) +target_link_libraries(he SEAL::seal_shared) + +# Include directories. (added for Linux & Darwin, fix later for windows) +# include paths can be spurious +include_directories($ENV{JAVA_HOME}/include/) +include_directories($ENV{JAVA_HOME}/include/darwin) +include_directories($ENV{JAVA_HOME}/include/linux) +include_directories($ENV{JAVA_HOME}/include/win32) diff --git a/src/main/cpp/he/he.cpp b/src/main/cpp/he/he.cpp new file mode 100644 index 00000000000..f9bad7e9846 --- /dev/null +++ b/src/main/cpp/he/he.cpp @@ -0,0 +1,279 @@ +#include "he.h" +#include "libhe.h" + +#ifdef _WIN32 +#include +#else +#include +#endif + +unique_ptr get_stream(JNIEnv* env, jbyteArray ary) { + size_t size = env->GetArrayLength(ary); + jbyte* data = env->GetByteArrayElements(ary, NULL); + + // FIXME: this copies string data once. maybe implement a custom stream + // idea: implement a custom stream that wraps a jbyteArray, which calls ReleaseByteArrayElements in its d'tor + string data_s = string(reinterpret_cast(data), size); + unique_ptr ret = std::make_unique(std::move(data_s)); + env->ReleaseByteArrayElements(ary, data, JNI_ABORT); + return ret; +} + +jbyteArray allocate_byte_array(JNIEnv* env, ostringstream& stream) { + string data = stream.str(); // FIXME: this copies string content. maybe implement custom ostream + jbyteArray ret = env->NewByteArray(data.size()); + env->SetByteArrayRegion(ret, 0, data.size(), reinterpret_cast(data.data())); + return ret; +} + +void my_assert(bool assertion, const char* message = "Assertion failed") { + if (!assertion) { + throw logic_error(message); + } +} + +template jbyteArray serialize(JNIEnv* env, T& object) { + ostringstream ss; + object.save(ss); + return allocate_byte_array(env, ss); +} + +void serialize_uint32_t(ostream& ss, uint32_t n) { + n = htonl(n); + ss.write(reinterpret_cast(&n), sizeof(n)); +} + +uint32_t deserialize_uint32_t(istream& ss) { + uint32_t ret; + ss.read(reinterpret_cast(&ret), sizeof(ret)); + ret = ntohl(ret); + return ret; +} + +Ciphertext deserialize_ciphertext(istream& ss, const SEALContext& context) { + Ciphertext ret; + ret.load(context, ss); + return ret; +} + +void serialize_plaintext(ostream& ss, Plaintext plaintext) { + plaintext.save(ss); +} + +template T deserialize_unsafe(JNIEnv* env, const SEALContext& context, jbyteArray serialized_object) { + auto ss = get_stream(env, serialized_object); + T deserialized; + deserialized.unsafe_load(context, *ss); // necessary bc partial public keys are not valid public keys + return deserialized; +} + +template T deserialize(JNIEnv* env, const SEALContext& context, jbyteArray serialized_object) { + auto ss = get_stream(env, serialized_object); + T deserialized; + deserialized.load(context, *ss); // necessary bc partial public keys are not valid public keys + return deserialized; +} + +JNIEXPORT jlong JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_initClient + (JNIEnv* env, jclass, jbyteArray a_ary) { + double scale = pow(2.0, 40); + GlobalState gs(scale); + + // copy a to global state + size_t byte_size = env->GetArrayLength(a_ary); + my_assert(byte_size % sizeof(uint64_t) == 0); + size_t size = byte_size / sizeof(uint64_t); + uint64_t* a = reinterpret_cast(env->GetByteArrayElements(a_ary, NULL)); + gsl::span new_a(a, size); + + vector new_a_buf; + new_a_buf.assign(new_a.begin(), new_a.end()); + gs.a.set_data(new_a_buf); + + // release a without back-copy + env->ReleaseByteArrayElements(a_ary, reinterpret_cast(a), JNI_ABORT); + + Client* client = new Client(gs); + return reinterpret_cast(client); +} + + +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_generatePartialPublicKey + (JNIEnv* env, jclass, jlong client_ptr) { + Client* client = reinterpret_cast(client_ptr); + return serialize(env, client->partial_public_key().data()); +} + + +JNIEXPORT void JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_setPublicKey + (JNIEnv* env, jclass, jlong client_ptr, jbyteArray serialized_public_key) { + Client* client = reinterpret_cast(client_ptr); + client->set_public_key(deserialize(env, client->context(), serialized_public_key)); +} + + +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_encrypt + (JNIEnv* env, jclass, jlong client_ptr, jdoubleArray jdata) { + Client* client = reinterpret_cast(client_ptr); + size_t slot_count = get_slot_count(client->context()); + size_t num_data = env->GetArrayLength(jdata); + const double* data = static_cast(env->GetDoubleArrayElements(jdata, NULL)); + + std::ostringstream ss; + // write chunk size + uint32_t num_chunks = (num_data - 1) / slot_count + 1; + serialize_uint32_t(ss, num_chunks); + for (size_t i = 0; i < num_chunks; i++) { + size_t offset = slot_count * i; + size_t length = min(slot_count, num_data-offset); + gsl::span data_span(&data[offset], length); + Ciphertext encrypted_chunk = client->encrypted_data(data_span); + encrypted_chunk.save(ss); + } + env->ReleaseDoubleArrayElements(jdata, const_cast(data), JNI_ABORT); + return allocate_byte_array(env, ss); +} + + +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_partiallyDecrypt + (JNIEnv* env, jclass, jlong client_ptr, jbyteArray serialized_ciphertexts) { + Client* client = reinterpret_cast(client_ptr); + auto input = get_stream(env, serialized_ciphertexts); + std::ostringstream ss; + + // read num of chunks + uint32_t num_chunks = deserialize_uint32_t(*input); + + // write chunk size + serialize_uint32_t(ss, num_chunks); + for (int i = 0; i < num_chunks; i++) { + Ciphertext ciphertext = deserialize_ciphertext(*input, client->context()); + Plaintext plaintext = client->partial_decryption(ciphertext); + plaintext.save(ss); + } + + return allocate_byte_array(env, ss); +} + + +JNIEXPORT jlong JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_initServer + (JNIEnv *, jclass) { + double scale = pow(2.0, 40); + GlobalState gs(scale); + Server* server = new Server(gs); + return reinterpret_cast(server); +} + + +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_generateA + (JNIEnv* env, jclass, jlong server_ptr) { + Server* server = reinterpret_cast(server_ptr); + uint64_t* data = server->a().data(); + size_t size = server->a().size() * sizeof(data[0]) / sizeof(jbyte); + jbyteArray ret = env->NewByteArray(size); + env->SetByteArrayRegion(ret, 0, size, reinterpret_cast(data)); + return ret; +} + + +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_aggregatePartialPublicKeys + (JNIEnv* env, jclass, jlong server_ptr, jobjectArray partial_public_keys_serialized) { + Server* server = reinterpret_cast(server_ptr); + size_t num_partial_public_keys = env->GetArrayLength(partial_public_keys_serialized); + std::vector partial_public_keys; + partial_public_keys.reserve(num_partial_public_keys); + + for (int i = 0; i < num_partial_public_keys; i++) { + jbyteArray j_data = static_cast(env->GetObjectArrayElement(partial_public_keys_serialized, i)); + partial_public_keys.push_back(deserialize_unsafe(env, server->context(), j_data)); + env->DeleteLocalRef(j_data); + } + + server->accumulate_partial_public_keys(gsl::span(partial_public_keys)); + return serialize(env, server->public_key()); +} + + +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_accumulateCiphertexts + (JNIEnv* env, jclass, jlong server_ptr, jobjectArray ciphertexts_serialized) { + Server* server = reinterpret_cast(server_ptr); + size_t num_ciphertext_arys = env->GetArrayLength(ciphertexts_serialized); + + // init streams + vector> buf; + buf.reserve(num_ciphertext_arys); + for (int i = 0; i < num_ciphertext_arys; i++) { + jbyteArray j_data = static_cast(env->GetObjectArrayElement(ciphertexts_serialized, i)); + auto stream = get_stream(env, j_data); + buf.emplace_back(std::move(stream)); + env->DeleteLocalRef(j_data); + } + + // read lengths of ciphertext arys and check that they are all the same + uint32_t num_slots = deserialize_uint32_t(*buf[0]); + for (int i = 1; i < num_ciphertext_arys; i++) { + my_assert(deserialize_uint32_t(*buf[i]) == num_slots); + } + + // read ciphertexts in chunks and accumulate them + ostringstream result; + serialize_uint32_t(result, num_slots); + for (int chunk_idx = 0; chunk_idx < num_slots; chunk_idx++) { + vector ciphertexts; + ciphertexts.reserve(num_ciphertext_arys); + for (int i = 0; i < num_ciphertext_arys; i++) { + Ciphertext deserialized; + deserialized.load(server->context(), *buf[i]); + ciphertexts.emplace_back(deserialized); + } + Ciphertext sum = server->sum_data(std::move(ciphertexts)); + sum.save(result); + } + + return allocate_byte_array(env, result); +} + + +JNIEXPORT jdoubleArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_average + (JNIEnv* env, jclass, jlong server_ptr, jbyteArray ciphertext_sum_serialized, jobjectArray partial_decryptions_serialized) { + Server* server = reinterpret_cast(server_ptr); + size_t slot_size = get_slot_count(server->context()); + size_t num_plaintext_arys = env->GetArrayLength(partial_decryptions_serialized); + + // init streams + vector> buf; + buf.reserve(num_plaintext_arys); + for (int i = 0; i < num_plaintext_arys; i++) { + jbyteArray j_data = static_cast(env->GetObjectArrayElement(partial_decryptions_serialized, i)); + auto stream = get_stream(env, j_data); + buf.emplace_back(std::move(stream)); + env->DeleteLocalRef(j_data); + } + + // read lengths of ciphertext arys and check that they are all the same + uint32_t num_slots = deserialize_uint32_t(*buf[0]); + for (int i = 1; i < num_plaintext_arys; i++) { + my_assert(deserialize_uint32_t(*buf[i]) == num_slots, "number of plaintext slots is different"); + } + + auto encrypted_sum_stream = get_stream(env, ciphertext_sum_serialized); + my_assert(deserialize_uint32_t(*encrypted_sum_stream) == num_slots, "number of ciphertext slots is different"); + + // read ciphertexts in chunks and accumulate them + jdoubleArray result = env->NewDoubleArray(num_slots * slot_size); + for (int chunk_idx = 0; chunk_idx < num_slots; chunk_idx++) { + Ciphertext encrypted_sum = deserialize_ciphertext(*encrypted_sum_stream, server->context()); + + vector partial_decryptions; + partial_decryptions.reserve(num_plaintext_arys); + for (int i = 0; i < num_plaintext_arys; i++) { + Plaintext deserialized; + deserialized.load(server->context(), *buf[i]); + partial_decryptions.emplace_back(deserialized); + } + vector<double> averages = server->average(encrypted_sum, move(partial_decryptions)); + env->SetDoubleArrayRegion(result, chunk_idx*slot_size, averages.size(), averages.data()); + } + + return result; +} \ No newline at end of file diff --git a/src/main/cpp/he/he.h b/src/main/cpp/he/he.h new file mode 100644 index 00000000000..c7b0ad05d5f --- /dev/null +++ b/src/main/cpp/he/he.h @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <jni.h> +/* Header for class org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper */ + +#ifndef _Included_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper +#define _Included_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: initClient + * Signature: ([B)J + */ +JNIEXPORT jlong JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_initClient + (JNIEnv *, jclass, jbyteArray); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: generatePartialPublicKey + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_generatePartialPublicKey + (JNIEnv *, jclass, jlong); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: setPublicKey + * Signature: (J[B)V + */ +JNIEXPORT void JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_setPublicKey + (JNIEnv *, jclass, jlong, jbyteArray); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: encrypt + * Signature: (J[D)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_encrypt + (JNIEnv *, jclass, jlong, jdoubleArray); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: partiallyDecrypt + * Signature: (J[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_partiallyDecrypt + (JNIEnv *, jclass, jlong, jbyteArray); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: initServer + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_initServer + (JNIEnv *, jclass); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: generateA + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_generateA + (JNIEnv *, jclass, jlong); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: aggregatePartialPublicKeys + * Signature: (J[[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_aggregatePartialPublicKeys + (JNIEnv *, jclass, jlong, jobjectArray); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: accumulateCiphertexts + * Signature: (J[[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_accumulateCiphertexts + (JNIEnv *, jclass, jlong, jobjectArray); + +/* + * Class: org_apache_sysds_utils_NativeHelper + * Method: average + * Signature: (J[B[[B)[D + */ +JNIEXPORT jdoubleArray JNICALL Java_org_apache_sysds_runtime_controlprogram_paramserv_NativeHEHelper_average + (JNIEnv *, jclass, jlong, jbyteArray, jobjectArray); + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/src/main/cpp/he/libhe.cpp b/src/main/cpp/he/libhe.cpp new file mode 100644 index 00000000000..5f8a929972c --- /dev/null +++ b/src/main/cpp/he/libhe.cpp @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <cassert> +#include <algorithm> +#include <optional> +#include <gsl/span> + +#include "libhe.h" + +#include "seal/seal.h" +#include "seal/util/common.h" +#include "seal/util/rlwe.h" +#include "seal/util/polyarithsmallmod.h" + +using namespace std; +using namespace seal; + +RawPolynomData::RawPolynomData(const SEALContext& context) { + // Extract encryption parameters + auto &context_data = *context.key_context_data(); + auto &parms = context_data.parms(); + auto coeff_modulus = parms.coeff_modulus(); + size_t coeff_modulus_size = coeff_modulus.size(); + size_t coeff_count = parms.poly_modulus_degree(); + _size = util::mul_safe(coeff_count, coeff_modulus_size); +}; + +void RawPolynomData::set_data(vector<uint64_t >& data) { + assert(data.size() == _size); + _data = move(data); +}; + + +gsl::span<Ciphertext::ct_coeff_type > data_span(Ciphertext& c, size_t n) { + size_t poly_size = util::mul_safe(c.poly_modulus_degree(), c.coeff_modulus_size()); + return { c.data(n), poly_size }; +} + +RawPolynomData generate_a(const SEALContext& context) { + auto ciphertext_prng = UniformRandomGeneratorFactory::DefaultFactory()->create(); + + auto &context_data = *context.key_context_data(); + auto &parms = context_data.parms(); + + RawPolynomData rpd(parms); + vector<uint64_t > a_poly_data(rpd.size()); + util::sample_poly_uniform(ciphertext_prng, parms, a_poly_data.data()); + rpd.set_data(a_poly_data); + return rpd; +} + +EncryptionParameters generateParameters() { + EncryptionParameters parms(scheme_type::ckks); + + size_t poly_modulus_degree = 4096; + parms.set_poly_modulus_degree(poly_modulus_degree); + parms.set_coeff_modulus(CoeffModulus::Create(poly_modulus_degree, { 54, 54 })); + return parms; +} + +size_t get_slot_count(const SEALContext& ctx) { + // slot count is only half of it. but every slot can take one complex number or 2 doubles. so in the end we get twice + // the space + return ctx.first_context_data()->parms().poly_modulus_degree(); +} + +// returns a vector filled with random double values between 0 and 1 +vector<double> random_plaintext_data(size_t count) { + // this example is just copied from the CKKS example of SEAL + vector<double> data; + data.reserve(count); + for (size_t i = 0; i < count; i++) + { + data.push_back(sqrt(static_cast<double>(rand()) / RAND_MAX)); + } + return data; +} + +GlobalState::GlobalState(double _scale) : context(generateParameters()), a(generate_a(context)), scale(_scale) {}; + + +PublicKey Client::generate_partial_public_key(const SecretKey &secret_key, const SEALContext &context, RawPolynomData& a) +{ + PublicKey public_key; + Ciphertext& destination = public_key.data(); + + // We use a fresh memory pool with `clear_on_destruction' enabled. + MemoryPoolHandle pool = MemoryManager::GetPool(mm_prof_opt::mm_force_new, true); + + auto &context_data = *context.key_context_data(); + auto &parms = context_data.parms(); + auto &coeff_modulus = parms.coeff_modulus(); + size_t coeff_modulus_size = coeff_modulus.size(); + size_t coeff_count = parms.poly_modulus_degree(); + auto ntt_tables = context_data.small_ntt_tables(); + size_t encrypted_size = 2; + + // If a polynomial is too small to store UniformRandomGeneratorInfo, + // it is best to just disable save_seed. Note that the size needed is + // the size of UniformRandomGeneratorInfo plus one (uint64_t) because + // of an indicator word that indicates a seeded ciphertext. + size_t poly_uint64_count = util::mul_safe(coeff_count, coeff_modulus_size); + + destination.resize(context, context.key_parms_id(), encrypted_size); + destination.is_ntt_form() = true; + destination.scale() = 1.0; + + // Create an instance of a random number generator. We use this for sampling + // a seed for a second PRNG used for sampling u (the seed can be public + // information. This PRNG is also used for sampling the noise/error below. + auto bootstrap_prng = parms.random_generator()->create(); + + // Sample a public seed for generating uniform randomness + prng_seed_type public_prng_seed; + bootstrap_prng->generate(prng_seed_byte_count, reinterpret_cast<seal_byte *>(public_prng_seed.data())); + + // Set up a new default PRNG for expanding u from the seed sampled above + auto ciphertext_prng = UniformRandomGeneratorFactory::DefaultFactory()->create(public_prng_seed); + + // Generate ciphertext: (c[0], c[1]) = ([-(as+e)]_q, a) + uint64_t *c0 = destination.data(); + uint64_t *c1 = destination.data(1); + + // copy a into c1 + assert(a.size() == poly_uint64_count); + copy(a.data(), a.data()+poly_uint64_count, c1); + + // Sample e <-- chi + auto noise(util::allocate_poly(coeff_count, coeff_modulus_size, pool)); + util::SEAL_NOISE_SAMPLER(bootstrap_prng, parms, noise.get()); + + // Calculate -(a*s + e) (mod q) and store in c[0] + for (size_t i = 0; i < coeff_modulus_size; i++) + { + util::dyadic_product_coeffmod( + secret_key.data().data() + i * coeff_count, c1 + i * coeff_count, coeff_count, coeff_modulus[i], + c0 + i * coeff_count); + + // Transform the noise e into NTT representation + ntt_negacyclic_harvey(noise.get() + i * coeff_count, ntt_tables[i]); + + util::add_poly_coeffmod( + noise.get() + i * coeff_count, c0 + i * coeff_count, coeff_count, coeff_modulus[i], + c0 + i * coeff_count); + util::negate_poly_coeffmod(c0 + i * coeff_count, coeff_count, coeff_modulus[i], c0 + i * coeff_count); + } + + public_key.parms_id() = context.key_parms_id(); + return public_key; +} + +Client::Client(GlobalState global_state) : _gs(move(global_state)), _encoder(_gs.context) { + KeyGenerator keygen(_gs.context); + _partial_secret_key = keygen.secret_key(); + _partial_public_key = generate_partial_public_key(_partial_secret_key, _gs.context, _gs.a); +}; + +Ciphertext Client::encrypted_data(gsl::span<const double> plain_data) { + if (!_encryptor) { + _encryptor = make_unique<Encryptor>(_gs.context, *_public_key); + } + + // reinterpret plain data as complex<double> + assert(plain_data.size() % 2 == 0); + gsl::span complex_plain_data(reinterpret_cast<const complex<double>*>(plain_data.data()), plain_data.size() / 2); + + Plaintext plaintext; + encoder().encode(complex_plain_data, _gs.scale, plaintext); + Ciphertext ciphertext; + encryptor().encrypt(plaintext, ciphertext); + return ciphertext; +} + +Plaintext Client::partial_decryption(const Ciphertext& encrypted) { + using namespace seal::util; + + // c = (c0, c1) + // dec(c) = c0+c1*s + // we need: c0 + c1*sum(s[i]) + // so we return c1*s[i]*e[i] and add c0 at the server. e[i] is a noise term necessary for security + + // adapted from Decryptor::decrypt + + auto &context_data = *_gs.context.get_context_data(encrypted.parms_id()); + auto &parms = context_data.parms(); + auto &coeff_modulus = parms.coeff_modulus(); + size_t coeff_count = parms.poly_modulus_degree(); + size_t coeff_modulus_size = coeff_modulus.size(); + size_t rns_poly_uint64_count = mul_safe(coeff_count, coeff_modulus_size); + + Plaintext plaintext; + // Since we overwrite destination, we zeroize destination parameters + // This is necessary, otherwise resize will throw an exception. + plaintext.parms_id() = parms_id_zero; + + // Resize destination to appropriate size + plaintext.resize(rns_poly_uint64_count); + + // Do the dot product of encrypted and the secret key array using NTT. + RNSIter destination(plaintext.data(), coeff_count); + ConstRNSIter secret_key_array(_partial_secret_key.data().data(), coeff_count); + ConstRNSIter c1(encrypted.data(1), coeff_count); + + SEAL_ITERATE( + iter(c1, secret_key_array, coeff_modulus, destination), coeff_modulus_size, [&](auto I) { + // put < c_1 * s > mod q in destination + dyadic_product_coeffmod(get<0>(I), get<1>(I), coeff_count, get<2>(I), get<3>(I)); + }); + + // for security we need to introduce noise here + // this part is based on rlwe.cpp:encrypt_zero_symmetric() + auto prng = parms.random_generator()->create(); + MemoryPoolHandle pool = MemoryManager::GetPool(mm_prof_opt::mm_force_new, true); + auto noise(allocate_poly(coeff_count, coeff_modulus_size, pool)); + SEAL_NOISE_SAMPLER(prng, parms, noise.get()); + auto ntt_tables = context_data.small_ntt_tables(); + + for (size_t i = 0; i < coeff_modulus_size; i++) + { + // Transform the noise e into NTT representation + ntt_negacyclic_harvey(noise.get() + i * coeff_count, ntt_tables[i]); + + add_poly_coeffmod( + noise.get() + i * coeff_count, plaintext.data() + i * coeff_count, coeff_count, coeff_modulus[i], + plaintext.data() + i * coeff_count); + } + + // Set destination parameters as in encrypted + plaintext.parms_id() = encrypted.parms_id(); + plaintext.scale() = encrypted.scale(); + return plaintext; +} + +Server::Server(GlobalState global_state) : _gs(move(global_state)) {}; + +void Server::accumulate_partial_public_keys(gsl::span<const Ciphertext> partial_pub_keys) { + // sum only the first poly of the ciphertexts + // the second poly is always the same, see GlobalState.a + Ciphertext sum = sum_first_polys(context(), partial_pub_keys); + _public_key.data() = sum; + assert(is_valid_for(_public_key, context())); +} + +Ciphertext Server::sum_data(vector<Ciphertext>&& data) const { + Evaluator e(_gs.context); + Ciphertext result; + e.add_many(data, result); + return result; +} + +vector<double> Server::average(const Ciphertext& encrypted_sum, gsl::span<const Plaintext> partial_decryptions) const { + // the partial decryptions were of the form c1*s[i]. we need c0 + sum(c1+s[i]) + // so we need to add c0 once here. + + // FIXME: this copies encrypted_sum, which is unnecessary + uint64_t num_coeffs = util::mul_safe(encrypted_sum.poly_modulus_degree(), encrypted_sum.coeff_modulus_size()); + gsl::span<const Plaintext::pt_coeff_type> es_data(encrypted_sum.data(0), num_coeffs); + Plaintext c0(es_data); + c0.parms_id() = context().first_parms_id(); + c0.scale() = encrypted_sum.scale(); + + sum_first_polys_inplace(_gs.context, c0, partial_decryptions); // c0 + sum(c1+s[i]) + + // decode sum + size_t slot_count = context().first_context_data()->parms().poly_modulus_degree() >> 1; + CKKSEncoder encoder(context()); + vector<double> result(slot_count * 2, 0.0); + gsl::span<complex<double>> result_destination(reinterpret_cast<complex<double>*>(result.data()), slot_count); + encoder.decode(c0, result_destination); + + // divide by N for average + for (double& x : result) { + x /= static_cast<double>(partial_decryptions.size()); + } + return result; +} + diff --git a/src/main/cpp/he/libhe.h b/src/main/cpp/he/libhe.h new file mode 100644 index 00000000000..25774a80a43 --- /dev/null +++ b/src/main/cpp/he/libhe.h @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef LIBHE_H +#define LIBHE_H + +#include <cassert> +#include <algorithm> +#include <optional> +#include <gsl/span> + +#include "seal/seal.h" +#include "seal/util/common.h" +#include "seal/util/rlwe.h" +#include "seal/util/polyarithsmallmod.h" + +using namespace std; +using namespace seal; + +class RawPolynomData { + vector<uint64_t > _data; + size_t _size; + +public: + explicit RawPolynomData(const SEALContext& context); + + SEAL_NODISCARD inline const size_t& size() const { return _size; }; + SEAL_NODISCARD inline uint64_t* data() { return _data.data(); }; + SEAL_NODISCARD inline gsl::span<uint64_t > data_span() { return { data(), size() }; }; + + void set_data(vector<uint64_t >& data); +}; + +gsl::span<Ciphertext::ct_coeff_type > data_span(Ciphertext& c, size_t n); + +RawPolynomData generate_a(const SEALContext& context); + +EncryptionParameters generateParameters(); + +size_t get_slot_count(const SEALContext& ctx); + +// returns a vector filled with random double values between 0 and 1 +vector<double> random_plaintext_data(size_t count); + +struct GlobalState { + SEALContext context; + RawPolynomData a; + double scale; + + explicit GlobalState(double _scale); +}; + +class Client { + GlobalState _gs; + CKKSEncoder _encoder; + SecretKey _partial_secret_key; + PublicKey _partial_public_key; + std::optional<PublicKey> _public_key = std::nullopt; + std::unique_ptr<Encryptor> _encryptor = nullptr; + + SEAL_NODISCARD static PublicKey generate_partial_public_key(const SecretKey &secret_key, const SEALContext &context, RawPolynomData& a); + +public: + explicit Client(GlobalState global_state); + + SEAL_NODISCARD inline const SEALContext& context() const { return _gs.context; }; + SEAL_NODISCARD inline const PublicKey& partial_public_key() const { return _partial_public_key; }; + SEAL_NODISCARD inline const CKKSEncoder& encoder() const { return _encoder; }; + SEAL_NODISCARD inline CKKSEncoder& encoder() { return _encoder; }; + SEAL_NODISCARD inline const Encryptor& encryptor() const { assert(_encryptor != nullptr); return *_encryptor; }; + SEAL_NODISCARD inline const PublicKey& public_key() { return *_public_key; }; + inline void set_public_key(const PublicKey& pk) { _public_key = make_optional(pk); }; + + Ciphertext encrypted_data(gsl::span<const double> plain_data); + + Plaintext partial_decryption(const Ciphertext& encrypted); +}; + +// adds b to a in place +template<typename T> void sum_first_poly_inplace(const SEALContext& context, T& a, const T& b) { + auto &context_data = *context.get_context_data(a.parms_id()); + auto &parms = context_data.parms(); + auto &coeff_modulus = parms.coeff_modulus(); + size_t coeff_count = parms.poly_modulus_degree(); + size_t coeff_modulus_size = coeff_modulus.size(); + + // by dereferencing we get only the first poly + auto summand_iter = *util::ConstPolyIter(b.data(), coeff_count, coeff_modulus_size); + auto sum_iter = *util::ConstPolyIter(a.data(), coeff_count, coeff_modulus_size); + auto result_iter = *util::PolyIter(a.data(), coeff_count, coeff_modulus_size); + // see Evaluator::add_inplace + util::add_poly_coeffmod(sum_iter, summand_iter, coeff_modulus_size, coeff_modulus, result_iter); +} + +// This function adds the first polys in summands to sum (either Ciphertext or Plaintext). +template<typename T> T sum_first_polys_inplace(const SEALContext& context, T& sum, gsl::span<const T> summands) { + for (size_t i = 0; i < summands.size(); i++) { + sum_first_poly_inplace(context, sum, summands[i]); + } + return sum; +} + +// This function sums the first polys in summands (either Ciphertext or Plaintext). +template<typename T> T sum_first_polys(const SEALContext& context, gsl::span<const T> summands) { + T sum = summands[0]; + sum_first_polys_inplace(context, sum, gsl::span(&summands.data()[1], summands.size() - 1)); + return sum; +} + +class Server { + GlobalState _gs; + PublicKey _public_key; + +public: + explicit Server(GlobalState global_state); + + SEAL_NODISCARD inline RawPolynomData& a() { return _gs.a; }; + SEAL_NODISCARD inline const SEALContext& context() const { return _gs.context; }; + SEAL_NODISCARD inline const PublicKey& public_key() const { return _public_key; }; + + void accumulate_partial_public_keys(gsl::span<const Ciphertext> partial_pub_keys); + + Ciphertext sum_data(vector<Ciphertext>&& data) const; + + vector<double> average(const Ciphertext& encrypted_sum, gsl::span<const Plaintext> partial_decryptions) const; +}; + +#endif //LIBHE_H diff --git a/src/main/cpp/lib/libhe-Linux-x86_64.so b/src/main/cpp/lib/libhe-Linux-x86_64.so new file mode 100644 index 0000000000000000000000000000000000000000..5d55922788b8ab95c41042ce8bce3e5b7d6e784e GIT binary patch literal 168696 zcmeFa33yaR)&|^R(Wp2blxW;$qDBoam_UF;#YqT(8)z^RK-9rbNCJr_BqSXO;u1`v zv~7<Wl@a&2euy|SE{F;eKnOaE0XNifh%2{^N<N(t)Y1I!d#Y}y61x4wZ{~ac|9Sq_ zPTlj~s#B*<ojP@@Zr$eE9MAZEF){sJ`qSTazAFaTnRN=t)OcN>pT5F9SGwyE{Qaft z1j#!fd}sfQ_DY30sVh#=tYaWAs}$WgeSa=<xLi(ZmXrEA)O}Z<^steNpOc#94#KVa zV^urqPw$<sI7F4(rc<XJbFp0M@hX4mnZ45#MY^0cR<UtWMt;-v{l00E&gZ1e%TM0N zzG7Y{eyZBnH(jNB%1NDa7or^H{HY(^4X&xWzG!->mrjD-bgH#3Z5x8hD9(f_(_Q(m z<XUfy^&e9F*=LJSsN4L@x&3aNN?984EytJj9g6Rf8Toe%8Rr`Gh^s8F@$k$evg@yN zxe{E-u9(b`Cl%&gn&4_3kbPs~Kon>g5Z6EUq|9d5fSCG2uD*VDzX4On9g^pAopi|R z%($%no|leje(4u+uGH%*mRykF8gS0Q`j%vuE7q0VFVj`uTwi};*6#)+pR}!|U`9&H z?&7q}%*^z}0hDDPzVq>|!Pkc`Kib?~ixeq;uf*T0@V!Q3*CMUQ7ZS^-34gE0_Xb7E zU%$S-5othQ-=tF}L40q<_jmZ-f-gU};=2sr-{X5bzIWow&s{1B|GgVm_uzZ4Mp|_G zpiUps>BC4L!FL6|EAd^0?;r4e0$+Zf#P?}@pCN(I8kM-7MY<N>=kR@AV@|Tpxj_1& zam~wCd^hp}pH28~kt_A54av*+{u$rDD#-P!PMK`ScL%<&lfdUKeBZ|RJ$&ECx9{fz z03T`a6Qs7j{uJpK`r1kM;GzrPFE#KFr2oWsuclq)SAM)?b^NX=8>Zgc_}(+|pLZrK z-?{I}*qh%<dLVn+qTQRGJ>2`q-eKRAR;ArJ;qo`TUp~BRSW(HRe;&3l`G@V7FUq^J z--4HpKK$I@9M=8DO;24u=Ze?g^uGPwrODd|C4Txt^Lck1^=SO;aZ5h`=AqA977m{G zWcr<Nt$pXQfp-jERI%`sSuq6%)o=Iz`O1BbXYM-VjjuPHGjr$WuV&Pqu;q-WpS@w# z{o_tLbFQy!?<E`FKBVilcXxii>d={$r`KgBmY(oVU0%hW<k$CpG_&C2Z7Z)ov1!WP z<5tCPef!0CI%jk}8N1fsf5H5Xp+_q|KlbF8KRf)Ma|@EUyYdE|6*^}2xOLZ@w&CW1 z8=v`P{Pga#Gc)4XJaG5CS1+Ej_2~8QJod%l%htYl>t*F9Uw6dk)|8`4K97I7HE{H& zKRmwplv(e!{x<HZFY?{_OS?B5;%eUW$nW2s_{RI^>{>qK?Y;NBw(5<~I@5pf{=F{m zm@nr(c-q4&Q^()+=)&J#cv#nQ?{~i4;*GuJtCEa2e*cdrW1FVVA6j?KkdGdC_WqFr zQ{K64^^Gk@EqUs2?};xiJZ`{~Hy=KE%Mj0dUH&m&5A9t1yE%*U&UyT_?y0XF)p&+I z^`bcgPX40g`XfI7^U<089=LA%oimP%@pazi%US*6#r>+Ey6)pMkA1oIxr>sw|K+$( ze|Tlh^0p0oUp)Ki&e#7kxY_@Y`p@oZzWD0rURyis#jS-!H{HDB&6V+ql{e&0edn~R zF8X^geb-y7-gs(i?FrTw>%Avj_0hm_EAAinUCgL!9{+Vp%(NBNr``Wy`qG8R&n<M! zCQ>@`Y?teZ2si$6cx3#AgCgU9cSpu+e&uovj=+CblzLA-I5MBdqSRXe{*mP03osJ? z%f!g?JOeyZ{_bIs@z*gaM9Tm9kjVI2OeB%=kB^OvUvfrd{JGO2<E|qk<Kv<GlOpIp zeNg25*F}+kB_`}h@@JnA8DAU49!5s#-}ET`dv+8(^pB$d4Zn=szkfu06iNR4vm)d3 zkBZFare8$Pf6MWa@gos`M5_0aev$d7MA1(m3jceDM&`dIN_&ruf_tO%*E9^?Nb=ti zWn52*Vka9gjw1Pi6QZ<tCG-=i-n=OCUmJ!0ucG*|$D-IzQxtn19i`t#L!L<O`dbt| zJQ+owu~F*1HVU7EkB(e#WfVX5ZWQ}DD2m^BJBnYuHHtppK|md;zdplwjD)`tWxRW% z@IPZ{WImgt=;u#S{N$GqCXzhw#z)3okS9|ACP%T`x1-cMI*K27FbX~~A+q0|A4Ly= zDC76wDEuFbVsD?LUm~@a>mZocexKwh{5z5(>+|F&a(;z=j8yNKDE;z!l<~L$@<ig( z5XEooh=Ly$Wn9ES1Bv@-!R2a-qK7l1<bOQMJlhau{@NWyo*SazmqEx#cK%fqeY#Gi zy$y1;Kh73nU854$84|Yue?eeb_wZ_*^y45`%i{_khW_B^<Fi=rAXnae#bBZ4b0QQ$ z{#~0C&<Q)^XCO%81>Y!IXLqeiQGB{?P{;>5e<Jk8{LQB+{Ai3%etwDj#8bXk^q(}N z7vYyqbp65=_n`tXtmN~?aVme_!wMg!@rmLeApe^Rn1gwOpLdQWpCPVvy^rGXIcB8d z-@04z8KJVf?uH&HPgT3ZD`4OJ<il?ecXcb;k#p}cihpb${yEaWgIuj0D*w|c&(HD` z6`$5WD*l(Eo&5Yw{Kg<x$Ab!o+n3Lf)0IA3$0<BU^QppJ(OVz=kB1!+uh;rHPxI-A z!NvZK+p2&Z%+vh*S+{HT0LA}aEl=(#TF#XUU!n7VE&d1nm9Fp$6&qLRG^K~cK6>~L z<BRen{!3*%Ocir^b$eU;wD)*j@9IZY#zS=eal;k=x-*m@^lSWF4EtfbT+0>h$n!qh zA^Q1N;SfVUhao<sU3Gk*7@w{2V-HjL+qHmt*tm9U{p7`}{0@FC?2Y`J2Pynp*cm_E zgCU-&`^E8Tdrwk$$A78(dN{d0!#d)A*D<d2G}SJU^0^)UjdmNW+Y7fWpPNore7e>t zK92qDO4aT9SmA{_|E<u&{b+CEM+$fJHd2q5)>l==8}vB6TKo*`$5#0Ly1-`=hrpkV zRD57&^5HmOd#iMN^{{q%(GcP;t<S|e|2p7oZ%Uu`=E-;*;@YX(>x|PUp%(J#(ERnV zcBLSWVn5bt{{XippGhftyl4k@rk3BM?XXMR!&#~suCZ_+!e2YieX1K=)4`2+N}us{ zFARqE)?Td`poQ}JW0Z095ZWt#^)8ifk>>B$c3YtBcDLs9s@Ufc*J{0A?C2p|+B?Lx zP2)~KeyZ`VKKUypPKMuZQH+cAxZW9MJU*fOw@UXf%t=0H!+<G&!8S$DLOc1nUAMR0 zrS$N3%|9hg+3kux{rI9D*HwC+nx)vdwk0Y(w?3@+<Z66RlEVA?fq{CS%DhbF=X}r4 zz!O#exJMN3jIVD-DLcswDLh+ccby|~GRCj=W2fqVnWpvDevis{v~Jhmv>$Gtr1+Fz z{P1%g#y#z%`5}cjDcDtWs+Ru=g=;f%U6!o$*>#@6C+TsQ7-c-3c%0&ssP*H_ci(F} z?9g`jhMuQB)Z?hDkN)FeFzm;^^ZwD=pLgkgKVR#ibGYiqrJ9dpZ^b7?)^jPELp}73 z2Y#*fUtp{DW@-I2YP?nB^EF-~_VWu@fu5IVX#Ce0Xkv#CDc~IVM}BU@#7#fBypNo} zhF)d-ey{S;Pw~?ab}QrY4n=>h@m6jBtNZx3>!DZaU+p&>yS?p9g?GHH7zA`b&d`3T zS^K4<R5h+rJzo084`+$|zi_pDqWBc){NHIjQR~4OkMXca%G0F{DpTjrh+?-_M;WJl zK14qCn$JYdrv-YZK2x+l(=|R#+t2EC3b;h;|533M^p}?Ze4YPxG@Sl9Q|kfUDxWtr zpT2R(RZ;xI&05ZOEhm~OpYNl@IhP)z+U4pq9!oHO*{-}k@kz%}We=IzO8?hrd$>Wy z5rSiFpEGs6H|qXv)$RI?#y4oc*V<>?^+(6i-a3A(7{5D26}S`oW7L0MtHPJ-aTkvU zO26oNGe`5kMEk?O@xy)K&vupevHyD=|Fc^0AE4W1Id~txgy&dFo>-O9iE}bXtA1?J z{<B8Q<CS`0w;!wg=V_eN1m*7=|2zS|M?CQvmH%bfB|lYq{@U4R{+h1GU8Wv)jvmez zyBgwZ=`#-=qvem&0izbn<??#xR#eXO*7ypmeO|B2n>%f?x45LbWNulFucUg~<m~dw zc_q^dXP1|#dy(&Wi|Pu!bIRrwmX}?LjF(JFt0^fgPaQsaa$a@ioC)QXvkS||4bNE+ z?N+L~mG4VUshw9gzqZ6%<*UwBTt+D-Ik=ftTr#Jyw%i8^%S(JEx%sKyoE(Sm&nq0R zdH4u#ewDYVvUZ-Yq?pxv%jQ*;7Z#OhG16*%Wo%JNMP>D(yvoY*$<uQ33v=^_dviQf zGN8tJwa`Lc>Ts{8vgitLRb_cu(W0E;rhNaoD*p4WNKNs=Fv@DGDr-uj_G9|o5+7P# zS)Nl7?rEjBQFFxxybB7e%V<nBsp%T25QVBz;X-{ZCq-&3F3(#aeDi((IZdjnru+x1 zNKJz#sVHT{)j3lq<@+*xi>gXsadRr6gG*3-Y6i`*qVS3mZ%t`oHB8c1ICrEs-|H*O zS&%c*OA=Y9<wwcts_|79Ra8-<|H-z9Rg|Ev%0=Nm3iqOP{ph^PG7PEe+<af!>_xtk z1v#UuF?tKN85I}i^>_s-kX}(+?yV`DQ<7V;ASY*<7b6V+!%@-46qc7)78UwRyd`x- zB~`w%%6Y1Vsj20abIXdnCDqlH)!C^zd6SAF+!&|E$u!8)tAJ+iE3K|vsITW#m-MP+ z`n-i@^NPJwOKNH>;NiS;>*~B!CDk>R^m)ET-UTVWo1N}Rnp-gu9hNh$uwv?z{1k6$ zN{<K08CP||v=lT=I($^wyanJ@>@BRGi#q1{dNpIeyJlPVyBU^ccuLN^qUuGWsl3AK z!V0*}>KeyoikFU(MY|N`$W!xkvOL+9=*K#r6vn8nDe;A?N$r(E7fnr@TDUN;vV76J z%8GG?zQS~r@D}4DXKKE0)Pj;CUuAV}1$<~lP7XL_@Rqlxs&L*^J$<BPmsORPRI^fX zC>$mw)fJd#U`ASN!z*ij-pV=N>cV+*OL}Rn*Bw(^y>5Wb@Kh8ouB`CF-P2U-l$3Ne z^A=XPc+-XUR#Yx1an;PPW~(uzQZNK*FW&N!d2@ZG%JQ_?rKYMWFW(1A9cA|hd#g8l z(xiNJR3&C!C%aa>d8b#I;gd=hO(>aHQe6l&_Ab=>mRTx&+@g6|)zyWIpz~g3dfm_^ zdOAt<$LPtG#kJ+NH8SmS<}J*p?;1e^#()~_C}egaY?$NJ>7!BR%sLw*KljIHOqpv; zCxOXAwX~$ihRJ-w3@6eUp+?o@!g+;rOR6&_l=yVi0C!V?o<$Ub{0El1cdIjKy%?ai zCC22II-;tux}wI5QF~=cbtTPMm5My5Qb!e4R#er(eL2PGG{v#atu3sE+KT4amQ^!? znl?iZf?hEcDll?u6=@8nR##>W$>G_RC3EKJJ{y%?U4jfb6+N9=hLKmsk!Nr$oLwpQ zt?KUWzu}F0dG0Br@=K~0lvHOA&*^2_)PZv=b^lwF^u2UAQsoUU=#fdc*&O&f#_CaR zMh<&{R*&I6b<&i4Z!hm787Il;>B*U1SX_+BblxJFY#rajqLHeaD)Y@HQ!p>X6s^LV zQXgX$%o-yL)!^||q~#zWO{*#_tInN-FqAp<Jk7{;l5U9U7E@Cl2TB)+;s_qIyg8Vj z)4hlVe1!-vQ@r_@8;g9YH9oLUFPUFkSnl;zN}!ACN+F6|Psz{rrlxz*xytc*d8cgN zTyV?w!ZnT4ak6;DV!7+w%BwD`DDhU8u!Iw(3cJxgE%o}U3(I^pxt`gnIiA@m_@+YO zoHRuE#c<vj@$iQ_x)_yRUWUnP_-JQ{(zbnNkh`jOHez7!6(!WVQb<OANl|r)53Wmk z4%4Bjp`59Grwkbg|H1gh7-%X^?H%9Kl$9Sg%w$w`NexyxkxZ|Tf%F8y!-r3wS2m}z zx?-x#G-1CT_G>v6c{)A{*Eu{Ra+&dkoH!Syrs+=Bw1i(uTGf=1HH8&b<#08afxShu zi@AU@Z4}lYO{^T%xX4Z{C$FbnQq%Ae#fu7y(bMp<#kEB~ETZ6eDk_U%aw*=?c^D6j z|MDhbp3hJ7R>+tQPbO0^dUEqr2w;dmJawMW=bcwFx3FkY5zPoe@B)}kZ<Z)VZYkc3 zDbuFODz&D^Q7AQx=FnF<_}__@I$VbdjACnYoc<DJ=s^(474AQeM|WE)Z?EY)dS>P0 zQhVh_2g?AHK_}C%u#8~aoKk&Wn537@QR`kcu{mwo-*=~`u}n>QCA%2wWM@?O9>g?Y z?c+zJ6c!cLBA&<86TvOi<V+bcx28OeF<;na8Y3;oQ<0jUmNsg1T3X8J5u;NwMvhD! zl{ylhC_G3^|E<Ci6;JndB<Pvw((ad8<`vy>M&!$CQ&Ylzt}r^s{mY^sIp89O<2kcA z6QN8Ig2@FX@c38^GJEPM71nUO!Gh-~FX?Hy>eCE!kYnqFKIY#%(0fhp&|@)`-y2L- zpthXUbY})AhB2zON00W+{_W|;R3pw!9a$owFqXsc8(3dq;vLPvsl22vx0s8m1$mRE zc@fI=nvRRHzVc?|IWc@R{b8Lr95D}!vbbo3s~UD(?4q-xENV!JW=DhD*ut8U4734N zbJruUB6^oNO;3a9Y%<pCveGdPy!c1W*$G)x`O50lQJPg)4azASEu1}@O*V%>v_?Dg zSOyju6faurL}v;e%EZk{I$8%&DaG8ch}J=xQMfbmrg@_)PmPw5Wi{T}wR7f_R5Q#j zL!hCA3G?U?W;C+OYEdniVf6cDmcsixCaFp~cz6cZ+gJ;ic>gyOO$3Wc9ia^d3wf-x zU^YKuH%>G>x)8Ep=@NOIm{ugSU5}Ojs7)b+EUSc8VaI8?^UAoRpx0!*ac6bR&7@TC zWGqnXq|pDcPD6*Nt&{^RGGhwjTr4{33TKxsNaeb*sJyVIM!cSEfzj+_W9vW{j*dV- zr(AETxV*0^i!7LmQk-pRsdIlQUqQV$GNoa9KRLo3sQ*8<v8h$_ALNfc^HMA_iy3=( z5#?jfN2T1#9PJ%nJFkeFb%^3Pi&f{2UobVl5);)(?^vuUa>q|r>N1?$r11+7*I^?M zAtQJU_rm6jN=hrsu}6~YDU$_BP6oD8F^ns(;K0H5kT+}IBJm6;dcc+Ua-{#R>eG7K z&S983!pm8OYf)}ga7p=N8laM(0X&+BrUBV2(cHMN)}sO_B!}g&OjMP5E3si&sP{}c zaDPhGvb2q;e68ODZD3z5|DQH7O6C7!BcoVwnI7b_h@FAS1SZaXfgZ(TabwsttR8b( zYM)gvY?V)u^766056=~`r?ISu(c@_nK0jvk7wD}EaQWZJ+)GdYM^bZ3p=uGV3bNyQ zn%-RP+0|Ejs~HuGyqGMoD?D2Psn}M=GbKKF^wuIE88O`JgH2s7=2<y%VRad{y^ysW z!0?pfl1RDBBy(9B-B(VYWKG3>pTjo1FATo>`6(7v0nCVeT|&lK>ga65jn(<wDyZ?5 zRmkoeW1x`?530QgIZITaPYycz3(xDR!)p)`c%hToFe?mfJ{O8uPg*`Q!sgQFXdkP6 zYfd>#A!2>AT@woz6nZPG=XwjP3X4kdglSPtagDbcOFHfi6=9#hy0RRr`MG#tz;#<8 zHv6j=c&DIY3regUtE3#B1kbQ!_qilXwCCeIj_04LsUzX&uwchSA?!#YqRTBpkERx3 z0q^W96>;;cDEw#?d!2~z54^Zcr!&C3r^b@PiYPTQGpZTI=SSrwI^y|x_3FZ03Sv3z zgVj{zjh;9qH)q}gZyo}u%ekbMZFW^>IM~WokHDuv<_x7IglCb|BQ2X|O6TNPT2+-s zDybB!8|xH`Qb}4FVnS|YL{cC&_#u7ou5tJskPuqqJ+bG{=;_gz=W7<tE5hKckPR0j zvUQ$g{45h17k=K`d;CQ9>^a%Pdrg}M<lRSm^V6ew_nfeIUm(vsmEGV;aX4@~(BMHi z?csf<nyL9^Q{WcVx+pwh<uSmB7G;?;&yNNhxzU3S81he3Z|eW0dViWo`}O#N%#HsC zeI7v^r-SwL2FxD+AM5o9ZHgK+9k`0akAKv(ZEi^MnN069ML#8TcvbxqGUE{%=TiL; zvFFJboUnQHDZdQBe#f2gxQ-Hz=ymGJsq}z3>?ihta){QJ{bHp5Nv%eR9L^e+4o>I) ze+1_|f*V<cqchdvr^d5!PH&zU2V#m!^>bW2T{BLxl;FsUzAIZbJqr`DeM9eRzS!@9 z96g5v8uJAfOgNPjd8LvSp3DACaVDb_gx1&~<FvTnpiL{UMARN(zYlINVu94*EYTBN zbDOdEPN6u=KC@&GtKM;bnupczKfyUX!&=LM%4YX)iM-v}%k%YQj<(_1D|-YLfWz?N z=9;G{e4++zL;<DEUJqgajlHHG4MT_syA8cW|9`Rd;n+ubKfI?yBKKazo%3jYXJn4t zb6l<HQ&I=&nX&kvjA#6E3I`ixm~ni$6J_4WSzsPkMJPTGPmJ)e9;;+INt7$W;7HCe zIGm<8(Ht@MAAI4nKoM)4DwF9?s1p7}Cbei)8ji}DApVUN{(F|gdIvvjBRw+ze1~JK zX@;At>gX#!3DK)duw_w%7^4CkA6ze`W;n+Y)#F}yG7(<0;we(k@>{R*Du4PKuV<$u z;zQh=-b>=1TE)7iMOx(^K;;n^s0UD9@4T4a2+vfp^&)3HBJQ8Cht+<KSt|05&Cgb> z>@oa%4Aa@C$y4WYdc4`chrNDG`EQyw3g^znD+bss>nm-|zo`QKY`T2WIQ4M)-^8}( zphM50)f!j|pJNSA^IlX^cm*7C4PRc6qu$ur!-idM#OEH|5|<-aERPlO|Em0f)_)*V z@y{0WXI1i(T4D~#96ZE1a9#c=hXa@4XEpUFb$TyBe^xbU4W`k)wi(?hWk(RQ9eAOi zT+@NH<G{o@V9q}-b1&^{L;DE=MJ;v!wH<Ive!R<iiFClcA$RmurPd1`VgBeNHaf}Z zOb$e(JXZ8mmZr)x_o=j(CVsx`z1aLmq%SX<%@^#3)l?1}F>JJpq_4Q>j57#~B3Ti> znp=cNF{Dz7>u2<&Q%Dzu^OcsI<tdw2TX$An`Y7+HG~liYp4_oGPCRV*u#v6_+1cI^ zD8n>uSPHN7-PAN!_RN`CV{^T!5GKPQRF;`DQ83NC9yZ*0U01#2X#C}Cp8Z|@@O!5H z@!vpP=@-le=Wo9rjC*+bQGNxMdDV4)q4|=rx;mIIY|1?(!iAjpQm%fHl(6t-JqJmS zgN0T#^KL(>jr9+})j_zc-mX=(;HMjyNy<32l2-iHU-FLm|No1q0qV*ncRt5Ew0wU) z5#QQl%g{-Ox=zq9R#SU#?05fQ<9<KaE`2`*<?~Kmgl;*^H5BE^v*41`z;B={HO#M{ z>w6>rZNK|Hx--^wuFkLOA7Jnw;M%J5gPm*9_9wvq5Z8E}|FtKs7)V>p*R*fwEik<P zuFEt%f8Xseg?_HfLG$DM&M~c%pRU_gx>Ucf>);)~S9o(DJWl6d-UnZ;@q|9V8{*K; z`@np=_#}L(8-ASM%jk<c?@RZ^o!^(~i#xxA&=+@pcc3r6Ld#G5w<3x9<NU4%alR|> z$mzWQ+!uF#*P}1KLcj0b7kA!2?~6OXtI`*DekY<Y?);unU)=e9pT4;B`$~Or=Y9LW z_*z3h?-}}OHSn(uyxqX>H}DPv-(ldkfj?&8`waXO19utvTxsBO2L2BNPcZN$2A*i( z9R{9k;BOmvx`D4U@Js`5GjPkmTMRtU!0$HjnFby-@B#yW)4;0?{J#yn-oW)xl)KFa z{x>6ki-Esx;HwS%4Fg|m;GY@zHUodfz}pQxVBj4FzR19B1K(oc`waYJ19$1)iJ;y7 z*}&rre4BwM82FP0o^0SR8+eL=?>6vE17B<4mVw`A;4=-p%fJf^e3OAMAEst*&hc!U z^Jy_~zXKv&Vc;Ja_-X^k1QGtMHSjOPsLR!A;Fw^-pKS&nA4XlSb_2i7z;_zBo<ihq zhk<`+<hKnRKlu~>bQ$=)VbtZ?XW%Cqxa;g5y`5^{u?D`(z~c=38Us%-aObytcstR+ zZ!q#F8+eL=rx<vhfu|dIt$}A6c$$G*2ENX~^9+2nfzLGX3<EDP@N*2j)WF{{@G1je zVBmEIKHk9V4SceJFE#M147}OEmmBzU1HaI~TMT@vfv+&|YYlw0fnQ|cYYlv+fwvm? zB?i9Dz<*=l?FQaz;5!Z6Yv3IQUS#05ftMI~mx0eU@O=h;g@GsPxQjj`&%hJ(KSXgb zFf~Hd%|ru#$jF~;;GY_Jih=j6i16=O7&uo#&L`8r>C2ptW#F9-h&0c@DXa6DY2Z$5 z%6kO{jsQFSDK&7lT2i;F4E*_UZu~nB29BRr3V-Si-1*Hs-dt+n4~BEQT+IfKUnL5E zmK(V98+g3gV&G1UNqmKY4<?{Ls}1}J2SmEoz>hTWRs%0L@NEV@#K7APe1(DUH1MMg zyu-kcHgMa(6AZk|!0{7M;m<w;KQ@fwB2`%{Z2>>A7XHK<`0-&Bak_!)pYoHt2?maz zQVV|)4IDpJ75*d}cw!iJxl#=LWCKq(@FWAzH1Jal+%oVd3_Q=kPc!hD27Zr$7Z~`J z23~65$p&6!;2#-yoq-QE@OlG3-N2U`_!$P?Y~Ym!zTCiXGVm4yf6c&G82Fh6zS_X? z6NBN;S_8*V?u9?C2Hq4#vEDTBR}H+~z|GB}od!PK$lqb$_$j#X$2RbPhEc4u4SchK z?=$d`2JRZ(WB;QJJl4Qh8+e?7hYUQyz|##p(ZIJGc(Q@tY~U#dey4$_8~C{fo@wCc z8MtNOV+=gcz<+JvGYve`zzYm~kAasO_`3#PW#Cx`UT5HA4ZPmK+YNlFfq!n`%?AFI zfiE}kYy)pG@Novd!oYJ3e6@j3Fz~eoZW(y1f&b3Hw;6b@fwvp@L<8Sx;9nVdhk-w5 z;I@HJGVm?~f84<L8Tele+%=-d{yhdBYv4B;c$|SRHSh!j-)i8A20q2WlMVcR15YvV z3k*En!1E0})4-=0xMkpb4Lr}lryKZ81Han93k-aQftMQi9R^-y;1?Tsoq=C!;PnPR z%fOc!_+<v(Y~Te3zTChY47|m_YYcpaf&bCKR~vYRfv+|2g$CYg;DrXh&A?|Hc)NjD z8Td{EFE;QF1OJnO+Xmif;9Ukj$H4a)c&UNAbUe@ar_8`(4g7Kgk2CP~27aKIR>Nnp zR$$;88)95mQ>(9Ece~Z_Qf$wR)jje9-0D8%Bm8y$Dic?vMe<NbH$JDlLz>HwP`l7O zNOLfUT7`avbbr#Th2BJZ0O=N?pC>(#bhFS;lI9X3R4?>O(o{>RO6Uhj$C54(`cBe^ zkj@kO7Se~3&J_Aa(ua{w5&AmPgGeU|eI@C`NyiCYO`0Kn$R+gUq}`;uegJUFY|;$j zLmfh2LV7UicA=+|X2>3D6?!7+BT26odMs&%;-MCy&mkR8x>@Mqq#1gL>V-a?G(+xC zmCz@VW~d!15c*is46#FbLLWhzp>-%z=tD^#M><95exw;nhZ2SU_C?Sqkd71jOVSLH zLoT5|CC#mxP}lct{|BTKNp}eS4(Ss~w+p?4^hu;!g?@$f$)r~cy@_-Z=@y}%C(V#K z)GYLqq)#PXFZ4>%43R@sLO(z{nRJ2Bcak1TI#1|ZNS{tRQ|KE>Gjt852z?!ChMJ*7 zp|2!;7U?*lt4VV!DC83Qa?)p$?)sPXKj{?G9YSA1I+b+0&{Ih>bPTl$J(2VX(yN6Y zOPX6gp%$UfA<a-Q)GYLH(hLDZ^+KOcdNk=Op-&>s&@NOU^s%HfNaqQC1nF}~X9|5N zX@+W{6ruZ(W{4I_6#CospvRDo6Z%Wi=aY5`{V8dNXrZp}r2k20lI{@t9nx8(+lAgi zdMxQyp<f}LO?tJ^n@Eo%-6Hh!q#62znuUIn^mx+sLa!vv5GYh7^aG?V(gi}_NjjHw zp3t|Do=7@V=o?8hGzp~$eI03r7@<U=uOvO0bez!Dq!}WFTtZ(?njt`_>s#r6(s`si zguaCIg{0eso=SQu=~kg9l4ht7S}pWg(hLzoEkd6|njt`_S?J-UImL(Sg+871MWm~Q zK8f^9(gi{vOPW)BC{O4kNMAxaQ|LoUbE*%e2;GnLZ%8K!{Vo3Gg3?)}<AnZ_G^hHI zOXyEYb89Qq^^NpD=>pOnLcc?rQ*NkT=pCdv)rMMyeuZ=q>D5ASB3(?nMd;^Amym82 z`bpB9GDG!3uOvN}bd}H#kmdvwDiHck(q*LcguaFJ<)kx(zLE45q*H{xj&wQcM4_)F z%@89LCv-LGd8A!JUrxG`bk{!Vf6@#YLLEY1LV7;wcA=+|t|r|o^hDAOc|xm&9!uIs zx<%-7NY|2X7J4}81*GeRKAki}olup~Cy}lrT_E(aq!*FS6Z#0!SCY;Y`cTq~Nv8<i zkMvcf6NUcv1<+TMjuZMz($|o73H>SQYe{$QW&3Yg6XTlG3|c-`ApV$V5jzA%UdhD2 z%bti?189GR7Hc$3p#v2<N~6CdiZWkA)0TgaJw?->X!?9jzp3eA!sTj&D&4+;_G;KC zOD6zrw)|VHz*R|!>ee#e3Y?dus2g;~v*FNL&2FZF#hL2vqhuaftV>=hB^Sr4dqG|C zGg2ac4%&{U-iO5F-)(z=G98Uvol77==N)8kLvNI;UKRQVKe3HMm#{4Eo=()i$K(Il z?ncH?5tIsjy{2-dY1RXJa^0cX{Z7j=RIz<W6*y8Ccn!{)62wxVM_45ILMFm8`|_{J z|Ak~+t2RByn^2WB@ez9^$j<v9fnPD)goHe<14E|A;=TuRZ-O_Z{3~@uM_R$fmVFh} z)!D&(!==u}rS>=fB>%;Es<ugHZO_{0v9{Hy?KoB2G^wrI9wJ<K*{fh<9)GtzPUy{S zPPhFQ!NB@b^mA_T+EV*=)Q}svR@V^1T`O?)a#i2Eq@eu%WDtnI>S<B_dq~LXacP8R zTQ3FIZ=+F(LEWc-wF;<BvI6U8;@ZAJ;jG_kXtV4aq$hUS_ko*zIc|2ofvR#<RoCgN z#+p^dBNLT4i_H&M{>_~aPHy_hcP|w;lnibIgZ8IH)>O?Ph72G9?L)+KQH<dFc8YY{ zKVUk)R;IIBm3tov+xW6raQt-&{ZOM%0p;BQ%pr8LD(F>NT6LCYDY$;0YQDSizgdyw z-%ThGf40isEcy4?oPU5`sL&e)-EB{ligw#4(e8pTbYUrAfASS`zOWP5Xo|aO1}_5Z zl4gdDGD+pOm^&Ft^7!|L$M<sHgYr0k<@#TuP3*BZ)5?PDlIr1x?60XAHKx8}^??Ul zu;PG*-Hn*-T)C*K8R;DNp4Ke);KrmnRj<48EnWl;&J8@Bv>cb+dC8LltCFg4Il1X` zU#=%GX7Q6TxbAj0z6r2lTX$EMdt**)gB??QMs6Sn3g6S67bh9^7#RjJ!=d2}36vla z_rntKXZ@*)#+!MeB;XL$67X5DBf&TLr>UfZdjbBa1pAB(eI;PKs4H-FHx(meZeUze zT`v4URqEGPaL5+ibJ^dcqXf$hHYT+I$!%*)TFP7a@VcHC_*fZj@W8{+36lfmNwJQs zlO0)?8lpth)^$dPzOuf!0yjA{egXdVFDyr2x_pCROfzw1AEHd9ou=|VGK5|dU%4gp zTcq~8Xo0LOsB%gnT0Rv9oK?RrsRe+`-E=f9`*$pRIBx`UVuN|zS^nv<p_4(gWP($2 zo-Uc9N_tuHB9@%4N)A#b2Zat*bePknplJY>e_T>5E>gI@o&bv)ihlwE{pp+4@N##T zd!rTG5Q?cCW(8s`e;YH$bzew>vd}+qzI>f&5HlU3GId{=z!C6AwA4f8hm!k<5(7G5 z#Rlhh`>SF*Tg>*&ms^A6)}YRnxD{@|Lfpb|z7GEq1*WSPkg%4;&~|vxP5F{pu1bot zZ=;&85+^NF*+2dp^ts}52b{vynftJ2D=;SbhZvW=6Zbe&w*j^%!Ew<)7fCN-202H7 zJ@!-rh#OS(=SlUu?SDbl;c4wvGV<@T{GIj?)Ze)n`R_sv5cgk9Anu=;KM>!f&`&hF zMxm7oZP#dvLN8Y6^BTQTq1g(3NTWWYaNFplJ@y330V64JjO3T_r*<J#Fod2JS&~Q` zEX3weH#G~;t;@AT5n`5K;f6hwSwl+s168f(>so&h-E8iBg{8N5vDT}SQtYGA4V`PC z-eLIHo9sozq0zxw#CO?#+<;a-LBdX413pZDi>#b?BuJPq(m26X5h-x_ldf_nrk<q~ zN~WGBse3(lOcaoY?72F?a@(hZQf~RPJ;)JQPm5F;Cu!+w$TzT_v6{V*o%c4gh504v zY#uFXBeEbvy5_UW?vD(e4+C<9df)CsU7hc+T{9&2RY?h~(`~=S8`4v=a79Nlk{Qst z&n2TWMdp3QevElH^WOV3ZZLZpd0@Noi&lvapF~0y)lzhQof5hBLd?W*xV8`dT<LZ; z7(u*P-K0eOLUKMvjE3cpQ1?5ifEj2EBAFC>jMn;eFt-y~U<GRKeoSgkmzww5`$RAa zTK3w1`a+84*|&Vg9=JALsre(oJwec$q|l@vWW2^NRQz@#;k>#+@e5rEQAyV!cc_AL z2jYuW)@3^D)r!F=h2E~wxthU1&0sVcNZ5cnKNWrKwXZ{_$xUCo8;{hvvY6{2Rn5ov zXAEJV<H;9gjuM8u>@RfL@v7PlDp#Vewu{ON#J{i5V+Gx9ze7v(Z&iWs<6r=W9{*SN zm#nUc_7jL-po+YOgbn^L#i&4`8#VenpzQZ03VvS0R{};MnUn0R!IT0&Ob&DhWS7oL zgIBM%yFX=ZP9%CE;lO%}EtWx)sf4pjh2tK(6+Orxn$o7B?{kA3+^5q{P{mlC>rUnZ zJzUYA;KK#rXg2`v{0ohL5wA8w@NJKZQe#B$J@!T#nNrLi`!-cqseP%)=tSp_6E<bM z8vx{yy;j`l8KA5WK)Py;yYT`FF%MuUhfw$qqJo*dn7SJim^GVOe<4}pcVm7>?^~1? z4M>2XVc*)>2?CP^BOv<^A+lbVWGjaW-v8olAj`wZr$nHiBgiEXA5gqMV*=jmsLeq9 z;|l#mqmPJ=wKrcREU!w+vo}D-&g<Y;-^9NPX`iSWKc`q+u2?*%SrjSsc!gfA(aDOj zC-_j(YDlPhbf)h2D!IAGxoNvlLFeC4-CR@`MpsG)jY~?m0(xw3(*ma0_ku&`O6bZA zo^P>tqX<?N0cYk#gB!sTD*l)Wl=F<X+aQ@kEgzG{TvDnm^AV|YT~e|$p)F>YH3JRJ zS6!B-_$?EDd+ZV^AOA~*UaQdyq?SGQX)JiLDmX%ADc4!flq{Q_)^_?KjYHUb(A^=Q zD)$}!;XpJ%>V?4n5FP3sYZJ4$Dru&DI8+nbqXzlQszg6s(<ZaTGF>9YewVV{rIyeO zRf!c$prmG&2#k3f|FmCNq~j^lNLGLh7my2t`7g;BpQws2k>Y!7j(7TxLlk_u=6a6E z0sGX2FIR;>L@j9FIGt;VLjR)BVH*8`BMBA1X{(BdX<l13{YzEqc2#O06Xf0|<1c=R zLO;;xlbY9!y6gv<zEaaWRH;d-)GA%-Dus?z=<hW;N0&Wcm;Jq_hiUqHDK#!hV&21_ zXZ&vY{|?#kcnSKOYu<KSaC?b5&kxM7%N{Qo;y2(Qjf0N(Z!Hq-(gSkaZl#Sx<_a#3 zvv0r+MA)_!@FgKMKEv{-^|#uxV^J;258U=3*&u4Q8d~Ele|DYy^~a*u7wc*7h$bJQ z+zhR98(xa@WbB5c$(N?=vR{zFhY(&5N5n-3cjv`+wquK9rj>D3QZh21D&+^>t}9B^ z6~)<~(|ihP(f&5u^$F8G_Huw+FddG$!eupVNwG5a)DE-<!!qr^p>3W2zzDZDfn=-C zfdDTu(xk8d9=AE=G)cPKzFJhV%Z^7T;wPX`=O^?I``C6&J8#=BquS0LEPa6puuJ@f z?B2X6jof3u$+`mZH;DWr`ELL-VyPDi^p;XS%LcK%bL>aV>r5c!grPaTUp7l>*kd=L zC~FMT=tcE11O9;9+=SRF*>>5NA=`^9U>>aQL}}9=`%&CP#EUlQMVN2BJ&+h8-uRCm z5R*88l7jzx!hg)EK<)YL)S#Lg?2oi!{>2PV+<r52P+0p73V<5&)Q*I&&3+K~I$Kf0 zb*hFJn4pHmQbT;2LYE1;%PvvqFBN*N&MW>Q5Fe}1GL2>^8Lzt^GX6!B2cu!v9HdN% zojhHM$@#*vlPSB?kmEpf{+WxxOfDeRVDk7sfyOXaf>+1cL$&e7(rI<D+qp;eAvN{{ zrwVXd3)C+BK()DrS)@Cj5NXyW#VReh8)>Qb9Y8}DiOp^fy(PXCU7-~<(7uv|usgs_ z5?R|~#)O_m0PegMZQ*!2l`PH{7JKYtNZhP^^eIpRM)YM!*yX<z(Z|efQ687;B2Vz? zB<?QT4}9oEL<wjLjk`(|c@<_xHn5*`p`OAVRible+7|CXsV=c@ZNImI9e0H7cPM{g zv7TF35%zadb!P3IO|XPahy`Kqr*sf@zL?aO&P^o7p<L%$nTgSxeTzfD6uS@>6*`AX z2s9?Oiqf6_>v4vivP<327H7|9hy9U8NkLQx{S~!13JHCqyJBXzEB3HQ(G{7xE0RO= zq`LU)?-R*)Gl3$W6;;KrQ0V)D?y>Jz=p_n$0SOxx5OmDz*lcoYwvR?Vp)1uS$nKp$ zy1%jznpYy~xBGRlr7$K6Bj%N-4UU_73OOH$!!<Niw<lBGb~nmyzj)Ws2&oL?`+X?o zZl$BmaaP7IINRsQ78ZA+NQyoF2ItOw`p&^}M{QPd(8v^1)icH92CuQ~P1G$KeB~~( z+hy-ScX9m1+UepZIC}*=FgAaqG|&oU+Lr&O0c<QLm705EB)qFqzP?d580~vBvnFBD zBSBEyPLBkkMbut!ajg9(F-Oc@{#DYC9{;KV(w~8Wai{`$syzM=ol3BE<g(LQiCFp+ z7KZFcu{t&Mu8?lf!vbq;nXI;(CU^w9Y&C4GM;u+Y^6$_#*4SJ~bBIx&oy*L!Z_3S{ zf}MEi(wNi|-MFDou!^%Aq3s^V<H0yIgDMEbAHh}zMowaaUOh~jqyxRbv!#%h6M=pI zd(5v^2FjfFh$7Y{S?m&SrqU+uw}JKy7Ih0|vq^Y0O3UfD5<=z34FN8`TLgF>38$<Z zMbh|eg+8QFpP=KCs%THguqCmE7O)K5W2dqLZi#U47sGIsH@^f|?4u(@e0nd__3yDS zA!j_ec?qKY^)3<RY+Y$L?Ku#?R-wmeG^Ef66xv^-uMwp|T`9M*cPP;it+$>!5bJ$< zfVv*|%$=wRrsddG7Zu#`Hz-br?c)^phmo*twW`ix3cXpQUR@QH;%;<Noftp`J&Fl^ zCSyEamzyi)cH5^(xiM7`!V_F2{?2|1>d6gWZP}-51GelcY7Uw%u-|-FU6tD9ybEn6 zk%d}-@3x;sS%wweuwTSfdCEGr6V>cHAA%lPA1rM<tHr>-Upi(p7ty<IA2M?hJua!h z3f9HiX;QZumaTA59{)oFn2J0Y{=PfFtD!Bdz!dIG-v=>6)5NOu{0L)_;Z^|s_NawS z5R77kje*VRRq?tc;ZHLjEG#&3)qaV8vzU_882ezBdR0{0cV_YUe}E<s8R#nF>_efk z&JAe(qZBzX@&hL5(g#G(@s}!e9TMu{CPCrF!#?{)SOw=dHbE`UP&fym6<n8;PP>VH zhiX^TfKvSo-UhM{w`E|S0b%V-prI2fHac!2B-*uHB+3+tcG+>NZ#lz!Oido#Ta)g_ zmrz!;=)^yvCq)TkP6Z=-KFe(SzPo$QF!!$-;gayUrCH9m1;?BZ?_$S6ZCKyMu)2nq zVy$5OQQ~1oCL#%aiQ?h>r!arV-zEQcN~!XHg2845M;^@ln^gYbnD+%7_zshdeZF{D z$7L*nzCWb9!~LszO0rp}R&dN86_b@rhW=n}{Gq?)@3J=9zqVqwTHAl{4FQA6nn7%L z#~k;sa9YMGUq5C(%DHOKo^CaaIp}uk*Z(n|jReL_1a8lQ3hh$pZR@~X&@r=|a!wjr zjGH|QYUTqet8GFYWyECR9(=GBILh)LZ3U*pT5UXr2lyiFrCxvq&=jnmW+1%?=>@!w zV<QJ)Z4*j@Hh4a8^PC>$q^-aODOS_Vz6+*X{%!fv8++%bS53D38^vin#0t(`m|XK4 zD{w7#Bfj<xwDx{dRcX1O_-{Ow+G{m@)9pLfYT5z4vJd>*E@hissXK7Fc}}z4`Br!L z28wUTyp8S(jy!z=^vWkGo9!oPa8>Aam7!y&WVpB|!(}KODhX%UpfcFw!yGEeApsdu zcZ9Bj=HRG9lRz@f*XgB5(MihUthLC+&#$sBl81dWvZiNEL*u_^<CCdhN81@&-CS_F za{d2AyKC9LWXo?~IxFipS+lY(%kr|Fo?y~H*$Et*U2Gtij0oyg{7%6Co21yU+}B`* z>WTR<C-o!CKPzsIyS&vsILFm_{Fm-H+((EM`=xvErTcI*7Hiy?$r&H}Znp1xlalXg z#dpV-?$cYpbYC{`xQQ{YFWnbzo8unVjv}6<SoaNIv6BL`Vy$y$#nnD;;h|Qp|L?g& zx60Eh4~=(Ue|O^_nJYL1j|toMK>=1q`-1uO${8Z%#@v`T2vZDUuu6}c7#x?1-h|X& zj&%=CO0_b+@)cSBS3UkO?G>;k+K0F>zb7fd-pY<aFqU9}jFZ;JdHh$$xsL(^HZ*h^ zY7~3hfD3+pWo3N3FcpSVJ0&NzJN4zx>tq~R4RF0K-?N?ft8taJfx86awB^1QK9mg> z_O&#R{<y-Z{V)f^>#tClwXtJBE-d~;;{MH{a?Bh(Hjlo}xBTCsuQRQ{S--s%r)JmW zxEe7%u4FF;PPr6NZXhiQlNf9`a1fpzZ_SSDC*u%%!PvG)Kt>my$+l%@@(O7w)3)p? zCR?+8F8AQ<M0`{5weX#3kB6Y}8aIG9_<rTCghf>5#kmJhs&fy%bt#J3uOUnQ;y73B z)oL)j_6D0Z(l<`q`WAZ$`YGf=YebK5@-FlPC-(7Zsi?`GC-ua?dJ86tz$wj0Fq2pf zo8of)1JI&4`+IT?jJb?=g7MG7KfzVow?l2>&Ea3!J`7jR;hEsZb_;K-4Iv8~=pH&C z$sQ;b9ezDWVlWS`x;!cIMOAJZrcPAqhxJT@G5J5RP`CXw<{&nYR+4B1reRyA*n07? zmFi>p+pVG7V5S4$SGHOW`!W}nTN&GZa}fh2TK+HWCb7|7_T%he*j=I;U{i!dNGHQB zlT^b?nU;Si2PuYUCOrOj-}6eNOI{a^UXm>u#WPd;Y?y5*5iUe!C{P(5?a43<QimRb znnPp6=C_1Obo#JPNAu<bnmAq3&7s?of`8~4jn^TCC+9qoHH{ke1g~#_DjieAh^hPq zzB6DE2O+&I5r5{S;Lp5F{8^Ca9_&VXO#xhdm3#0-$mWF-udfFWEAV^Tk9+W3=KpQ8 zTvukg2QNgaYYQ+DR9S&*>MVbQ<n}MdrC6MMaDbVuwuTlY{CF}px*Ly^z@nA2TxL2( zz}~%9P!}DrJ#hUB<dd-wIP1b@D6;K(?)39xwT-WXw{-a%n8&^5`c(o-LviI=+8{vS z`qdyj{_#~1iI->xw?Ev2<px~P_3cFc4McIXl}Vel6<53=NZ|Ty>IPfS5Aw7!#RqwK z6YjBryqpnk;~K+a(ap{X-|dWWjLlA<w9DA74f~J}>>6Bb4nEK;duV1HFs`2YL5VwL zXx_>_c?{y61pN&Opsc`vB=_LESRG2lIVCWFVx?%?fFw!%4T;R%){qj;(vaSBnb~v6 z1(Yh<kf$%VHq7LeOwvufXm}}3tX|m}BghVn2l|KDuz#5H8i!B(6TcPzP=%zAe|T>@ zGmJT(cY^V^!(Y)qEZhMje;J`1{6paPEvzjVki<U;-FPc6Lv4uc)Hq5Yli<iJ(Ev>J z9B7|Fl8}wj#!<0_lLhAIZ945l+U(5FS<Za?hPde^pS6Ow2kr&)$sYR)2!bAbD;VR- za<3VXM8AC_4Av7E`5>^|z$nXacIr`VhjeOoA{3pS3})<3el57E{5<sI_OUUptzzQn zN5p#BnW_UPp}(_JSORIfO84u+Y|aQXb*2KHmg=-hr*)FDD>6B{{O`b}u{pnfmD*mF z)QtNUHk_|S%(xQmaJd`V<DNiCiYHKu!zdrsCOPpbPf`8To(>h`;jPay^7uaN%4)!I zlbG6fa$$jpNHBK^G$^LFEev6f78XY+7Pafuyo+sur4R%28rB8%xXN9!r3Fd&=+Si| zth#66q|tKR<cHU606(6LJy^-FZD+H}`LzFY-^uK+n|PuIhrND}JZoW%JXeE5TKi($ zjmrR8OI~VaqWm_4EBI~|uZ6{q%fi}EorRf}Gzdj&hlKAX^vv(NaU&suJlsw~Wu}gH zk&QY7gt1OGf9bpnahy70gcC*-s_6=pVi*N(a(e6|-?d?}S+~0>*UXV1+lZn^#(||6 zW$f%Z91$Gy_M4)+_|XB5yOH~D<gnVU=H-JpEzkKY>l@RC8yI$paI7Bx*R~6#Ldjqv z^0Dpy>4Q4!)gXK7FDMcm89ydG$d;hm(CeJwF}Oa^>0X_FtJ57wjU_-cBHxd*E>fc| z-&*q7a@d^}IH8u3_~k^W<$tD*t2qC%0!e}v68D<^9D6&l0Ko`44Y?6=<pxH3f)&lF ze3qdiC)~T0oC3j8mJLi_Znfoz;a~vf(W%C6%}OK-&##fKElUQdd+;HaKPwX*(Ci+% ztcBGLz7T^kOAp4Zbm$bhbHwbB)w28-=HdQKg=ZD$RFA`~Doxih-MXY+NcPY@2fJJw zc-Ky24<K&$H_b$SGQ+K<f8`f00YUM=xBUnf5!rE1hS8nn?+^4{g=R_-+}WC)5Q747 zRuEr*Q<b#4NmSRCohLQxfD!u`n9Q?hT0zV#{=eIW6oFEW8Az?xVeHrOIfT1$D1nAC zH=`<-yYUz(y{lnDY>a*LpW#sh6Vh|fn2=&U9}cDvM&$-BNX`vx*h%T#Pi(UML8=iw znkPw9GZQTP4zc#W@2+o`3Ps57nuO=!qhO}bFi&?)oE5l-o!4Cj$FbGpe&QK&>aK#w zyX+}6!ocA-!w4q_=H_k9PK?1klxX=UW#$IF7LPJv*)-<Or6PDgR0Ds6@fTwS`qwYE z(8^3IX*oE$o2CrF@yeP^E4Y3&XlSInCcV2RrMo7%yCxBGL6+{CSZtjj#0wviy6I1p zK$QqCi-Tk@W<q>B3Ci>YAMXMudfJag&}T?mfhJ}{HLXG|OCS{pCL-|!@Sx(JWCH%C zJOP3!k}OLXaM?_u?#YzoVN1YZfh6~EU7t10;}2HxBH!a*R>zB}9{;TRtP4H<lBM|< zPR+}LM}fq09EZWGgsxj&5y8|bu}ZB#UJClGo&q-QzzGfPUU9{RFnMoby1PCTsuEM_ z=6s?@SHq&@829z#c`Klb1_z)>(^ApFSJuW4`*UZs$&zNPb6S{+0x9-eq8()-O)tYI z3s+?%w2vc&r;H5gG4vWIpF?vuelf<~bQJHB(KA(2(;n__HL-m%Cfb@haEU&_%Au(p zms^``0wVwE(y*qT08qw7JqBnIECP?i?Cv#93}BW@DP&qLx1@JKtyL7e1q^*h!rDr& z7i+r@jp%B~gslbFvnz9h3*+qfwz3A}%(mxc01fM#!3IMs9wouF1r$b53Rdemm=HI1 zAS!T1a-OPiITE#h(!^f6Ab92iD<HlqcjLzcCI*jB1Rv3SOeojCb7C-!KphJQ>he4> zyX~pc<#Jjasv|$5<=a%%5R#2YQ9g@OoINoiX8A@&{RJrqUK`d^RM*77c|&~}j^O@{ zm@lkA+H0w;x&Cjlb>UvtnmhDMYwu3aiS1a7y@rV=CO2by?Yl5lX==mP7*GF)t0>$O z8H|ieJsIKw+)c+wuP7JhIP&#7ks0p00G{wfq*aOTq2d_agXd$@i9SnIiQTGaI*=8| zrqQBptU>JkjW#M`{n{|_k^*J|u-D3UQz>r-OLkuB?6=Iq{%<cM3^ieLwAw_^G_X)S zXpes%m;WDqt6c$y|0D1c-L>0#TniXc<K4Bb-L-4GYgc#IuIR39>8@SgUEAE<9=hW@ z@P&W77v<=Y6TWd~@<HeyM9>utoYf8=#strGu)?<dF}eO%Iho%fHa6zl>u@g^pAX-M zO<#Ki8WnnHAN+whz|gmlkJcHQ#Wtwk7R9{|x$4);5OFuPQ6EcDXJ|dO5m?s61TLXz zBif5r$dFOvCA8yTV2ODl^dZHKLk2sz1=)im*NmW?2*um%b7AnIs}WhK4CN}rvYrgb zGQ%&z878R=FZE=2c{4Jc63%eC%8=iaVJGEHgacMp9HKG|@5%5iGaMi0V80|n4(-XX z24f?X8|KiaGo*!Ev5XmJz-_Usi|M|w(=%VEtvWp!3_>aR!<&E8;3=AZTGQufI$Ng= z8oyQ3rAR|%8l0ok%aH2zyzC<C^}JrZ=#|SpI3Ns-Q`MrK>-u?&cornO2cL%Y8m`wb z%XANx#pRql-Y>wi9e(2adg)xfu)mfW*E9pdvLE^9Vqsrd@M8=6r<fV*$`&O2$ilw* zP+)M8|I@<0XGQM`J|zK!UfAOi|Ib|8_g=oQg+zK?zyG4YRo&dir0x2pNcicsw#PZ8 zUdwws8Rj8ToG17K&q3I80eAvuCDh|hIPRZ62VbNjxxiU>T}vaf5YQbSi~C<?xht=h zztiE$G83&JBF(_z191x^CSe3}f;<M}1ES(QKc?^?HU={h>$PR-s4i27n+@rZfgj5s zQ0u*0<q5mLK~A%@HSmCd+;3|r=(#NIxvc8BtkaiRpz;b5%ybWyR$|W~8IZHNac(Sw zwo~rDMsIH5!5X&;Fi9A}_$M4<f!pN0o~;cl$dL+|C4Wn}zp+}^M;+smPmjq4+Zjt) z+29A46K!i)>*Q-|XboRB3S-qYRZgz`4ec!Czjdc7+Yv6%VE0^h^<3`jxx`QB%4cgs zEUzSJI0%-!wL#uGk%Jn0XzDtjE&d8F|EX4O6x@H9u?e#k8d@RtAl(*31RJ&s-m6vp zuT10ft9AdQ^TWL!IQ)^}G#(g8{71|%)Q6{$@crx+i#DNc!9BD*3G0!bb+Nf#W?k5^ z#BNX(Tjm~$ldZ(CagOrJhVW3Brtco=Nh;ubK#AN}uS<Q|@;{waj~nFIf|MU#uTYnA zy;@&yZIok0kQX(5ndOcfmxPz6%9AW?i{Z^8dC%w%SW55}l`1RnR63MC$6fgwNFKO# zCJ2xJF=8IQK>;UpYXWGbfn=mG36j{bYsf@u1;m(9NXiBtECmRD#ZZwyKms{`lT=4` z%PBUR(!n%vlZfvLG+1QgzfF!7djfZH6nFx+)**2Zy)CH~na8=;1UL@dLvNC&(pKPw zJdkKnW0Fl~P+Y8->w^>#_+69@LKPr^9OTIwsU$oy7A5lGDVgK8)V1a&QJm%XB~>kz z>R6&)a@?l&YWx9_6)?NW^5ab=q<jbrj}f3q?xE9?@}&Qix+xi(?eX8Onu~T)h;d6* zZ64gO2hHA$Lksxfb%Jr7sISqp$-LzKeUejMcj;^U2G~BI{?-GquO#8eYZ!R&yc_#= ztko{T9PVDzsC(dreUKP^rId^d^~@g~)+n8^R9ppp0hP=r7Hr|*$Ce|3qgc%ZH(5Vg zs~iXRZ#7I2k*igcY!S!~O|nrSZIX-|1F{xH-1roSzQm1<@)+9_cx)#!sWpXyz?kOv zw5S6YH|fq-`T<w6U^6pOiF@!<T`0d)90^oN44`8$fp*;Fsn4_U3fQSv>R0*(0$htY z(Rr<qJ66On`8Tq5>2u3IY6EnPeY0lVl2)98it)JC;(Lq0*o@$M>c{SUky-Cf>R`K@ z-t{fOie=3r%Gc<_Ur(S2>;5yi<(H_Q^X&5Ad3(uvbtD!0ea?N~N6vQNyWx$n@OIy} zFoadRa5!0UsC`lI52L?!qCU0X*G^%UY@um{_xo;nLA6Ql!P0Ex;fGd7;pJNP`?jr9 zO)4c9H0caz@S{4d!6N}G4Z1gGt1Z7H06F@jDrV|(*eV>e$|-m9{^d?$z9ld1gF=mm zu&O=b_U^w=I4FEC0sGfV8}SgC9<cErh^GymcI>x3_}9*d*bl}#db0mF0Oi83kE-`e zv(V=~wxYHP-$QU8&Xy4RD@Y}8Ae)<z{(QCH_%);Y09Kj1WV;c=U`Uu5)TVfCH!>xJ zHiP26U@ty@H?k<9)<Ice^(#Kl8JQA7k10Ng671D4s@2Xjdogv!qJQmQiVehHyMX(a zr(_~wG=Uk<d7k+Aa}<2cFL);yKX)<LI-l4Ro=42Q09=0Tc|p}b)qVI*XVmP(0){Vv zg2qdruYj^|f&}tIEqedk<F|1X1G^5{$TTrH;BzrU<#@5pB~vBee@uIpf{o+xK<qb3 z*sz4=11OVF0$<fcBt)O=UNbHU14~V**2WL|FU6xy->Duy9-zN!KMzw0-^3ma^0AOJ zPRJ2a<9UOvm~A{>GR$hQV=<~QVJBNd_jyF5jdp+L7oDDi?(j$<JPJ)_zC?Lop2!Ck z5Nn;x5Ax*P<HvSBeFz7pd#Dmnoh6`GTT3zYJmOx1=dL_fu=F;;nY)?yvE05wP8nhW zep6Blpj^~rv6(m$wvXoN!<3JPZkNthc|xC{_tgG|ydf@5tr|<#`WGY)b2pWMu)jd} zhxm&OVShLoD~x*^TWjz1V6FM8Scl@+xl})A@L-N?dbzfwrz=7=VuSUNjpHo*!t3)e zfIznFd*=_5UkjSk+(X;As}`6m)}XJ0BM&{5<|)sc7NE~UF&mu>H>eETl5~bG_H<@= zg6oDjRBF#r8E)>$FoY)js7#$(LjA#sPp*=XTG(7)ukpP|iJu5O)K6#faj}X_M{9bq zrgtFo9PGCT7pCxF8jqVK*cC7aG0qa^3d~Hw&L8%C-*q?M3%PJY;qDratVyuo&wVvr z4Kdg`9g7T~`;LNu@*HJw!?!WMqp&HOfRi`Pf&AFu#BTqjSfI6^T7e^#{ll?L#lFnv zRC=uM9H(slAZEk1ijfT_h&-KtP=2Z*J2s{>(DQsHkUvQH58}zvvB{k`G#gJ7;E$YV z33#-<gztEvM|hTw?-?$^X1jaMSXg`Iu@=tM_+P`->^MXjV>6fP!3UP^p?4?Mv+;<C zR)`YDrnrYbqysN~J~S{kU4p(=ol%BDVBm}M5tXlz;~Wi$BQT%b1O(3|una;xv`yIY zNQV{3FW|ru#6KhngK9vMI+cTYk=ZEmBaj#mGGxz_doCNe(+NHd&^UnDwiNDmsyNLF zP4=NYKTcGNV8ewH;EAoINsMpMv@QZDm`GlDlME2cF_yIDR(=7;I0HR#Ns0$H>;pBW z_Ng?RU=Hkz&TYwebub0IeGdLj$H^lZI52C;X4b(C%!bw~o~wxkcX&j1<0vrI{uIHU zj&0!vQX~cxw}CSj>5Q`3R7T$v)86omw$?i&yeUPbQc7cVH#Pyl4Dm%3t#vwnZU7Ja zE(X@s&@TqEilO%7#MyWl(A+?6N^W3sy0zAMW-&QX0{?WbHT1u&jo%EwBa3a~7`e@l z<7?Y;_kQHA-z$;fDy1GgyO;&{|1nyKR-Ko`Lt5h2;WCqR{kF#+^7ucHZnXeq(x1Y# z#&LCncM#@S?rXN%Psk*JV@tUWTT-l;K{%-V7YmP26Ju(h&BdlDRJauUJ?Hvxim8e7 zFT11#ki^LK>O#E_A;EGh5SVfbl*PVX0n$Bmm9iIfaJt4<3;wj~d*GQGrxlA?=MlgS zOKuF@Juht5=ZX48(+U|j^&DWhEvW;gY10x;^|fggo57~PlD+&Mo0jt)!QWt&CI-zn zz%c$x9a~<CO8B8IBQ$ftmYWHQExREMXRRvAS`XL?oWlu;g*YMA(VAHblM+jD!tG#T znS8bj^0f4)J&mRXm5~eUW_?O#j2T%R;JE-AN7iui-BV#Mck37Y-}McJ?NvibVs~Yb z%CR{eaq{;Kl|N(eWADasF@Lv~v~jrGuRp>1*U7MS>gNra`1nuu=xCl}R2C(7s|6^+ zT|VLdXN9i(A!O}S`GYLI<fRpm+z4Bp=lQ*!GGFKv%~7G6{AiGqBO&w>raWD)AtyE_ z{LDJ^0B)%EAwJ5*<a4Lw9OuGwZRff1AJqKo-)?OTjd3@oedkPmWo&wI<Si%2?1-t$ zKKd!lXXS_n887SnI}y67$udLbpWTyx_mjvUC;6Rz3wdNthuX#rEyu~cX-})zvJAn3 ziY+&-rpDt>#$B!#u0ZX9G2b(h`0`+GrB9Jw8S@0uV0<imIk(bnprO;iNS<?OcZ{`T zQ@Zq%_Ng#Qc2?*~WbB#0P{$qMX1|0c(jBjcqLe(tl{~lNZmu6Iw=wgzEEHtRA1LgY zQ<WgsRw+Te6l5NCA3o7c`zoN(H2qo38+$KdtA|8tWMKP6`*e;9chlGGEyv(tgd8$r zcnMHC0%93)?xsUQ*^42j{PLk=&*8VP)qw+KccD1A&SZ8Evroa5wj2A>Rid}v>jw1< zBE0^9^X3nbImW2hAL-!YEK(9e{F$u%)*ttxAx;*oKR)s$_u+B1ks%>e$#$#r*WtLX z^DVK{FfG3_stm6cqQg)_$A%4c9$WV~;#QU6K9%9<<HH#yGsD_&hHF%Yua667II}m0 zSt`T5Jyo1R+wQl{sp349;cAtEdy9uM!-t5d_>|#MNw<VH>GTtgcR_^EZ}5kiF4n1! z;8mJ%((?$e(BQ*5^&(aMoi&}_Nu5}>{5XESH4c$v8|HWXPHbCF9J&p@I>%)nNsFb0 zkAXFK0tG3EE|bMS^2r=_$H$hz(1SPxBkSNsfcsC@3>2(Db~=KN6ze%>vWEY)hVHfS zY;oha129)_muJ*pqGoIDboCi%Wo&Zaa2bl*??7YFldvGGj4pTM4(9cL>G8jh5b#h8 zc!b0oQ{01d+d)t5z;TZbTwK!S9(-{Z&MDWvjYSn2Fey2F%HdFJD9_}x`b`#;EFZUp z2b?|UAcEsE`|~ugK39t6;ntor4{a2|{ls4_|KtQ(>ZypJ@tV+X<)TCXc#Ni*2`P87 z%l3*lgFj_1l?6wXK=VL1TGTWQ0#LxzuR9-Qx$vF?h5>%4N(}@2G!@P;$^OA3U>MGm z5W3yrubvL}+Jy-5H-Vh}l3u5vI-j=BeAKbb2XKYa@u#E3wlN9Vd5^=CZsr;cEqGF= z%a9r&yL0}>3N|I830cz-NM%jS_5TZ<%n*JXEx_ND01@mK+J4|3=HuZFcT=TAWLVm+ zPsDwm-S9^_w84m5{a8hVbL3$M<joD%#^I%+D!T;gRw0G3tP^V$7x*3Hm4-35&Ve8> z=J45<TTvWV{O1z5;$Tf2jsvP=+~<M;=9Ali%Xy^rDG-fEk|4T7{ymS2VcBt>z!eE- zUp+Fhqp*4(lgz4egE_J8#uve?3+F%~{8HqAKserZ3~Tm3PQbke$AJR*=^T?i<725h zIT6T4Ao_;qj4QBwks}4DhYy{(h0!L*7+`{kh2+pP-?s4m%6h(M{It7opnm!I`gV3L zssV#IFvyMBg1}ZTA;)SUvU1Ut1Qz3c^>2M%%l{1w9>eGmjx9I}jISviUl(G0&EWXD z7~|_QjIUQPzMR30arLd$@M(&*@l!RlVJ`B)*m}hZPMeAm^}zrurX8}ZK+Vu0PG6~G z7=di;BQ&)V)kiVjE0$jCM)=jmhux^OXy#=9zUS)Eau6>_cpZG4>W7YLWzVAoJlMnA zaE%;SlzRMMxCcMJQ;{_|FE2-DaO=V4lJglGw@@G!V%le|5JpM{c2@a+3DAN(s`rjl zIoNv&n5c53hk*<Mo}n;2&9{=K8JN|=oy|osk3-m|+M`3`5p3xF#P94aD8t_jn+bhj zRdg;D@KIMoDGp=4Cns9)zKe7im*F^o1m5Dba1Jt0*5J>oT-{S)jo7dK&aP1$YxO!D zhvAl6fvcObnsa^&?!O<VU@|m$2i~9S4eZb6v`9LTiTWM5Q&k(2p(54j>5tI;V82N$ zx2-7!cUTCP%Hb^zy^PV6I4&S~T>}pNZ9>80-(XhAY5xE=x?!H+c@X`b)UO@Wgah&T z*ORffPBOO7qtM(%Yf2RTDyN_ZAi!kyf;zI~*w7ub#fJWbzhXm{{}mjL<gmz-R^6kz z@?BpN+(890%Y3#D*~77i+ID^cP&3Y>I32rP^k@8LmW!={o1zRd($8iNoIvsTJB&jp zDvstcrTsfKkSE(Cmh|b7;XXCuL)E*>z*&AXFo~7**i1(tPxf!{qjT7p+0nUN&p$_e z$0>mG4|a)z>oZYG&bfI4RdMz<m~^)~q^UeH-LM?S>VBpaIlG@JkiRqKZ=U?M<S+ST zb-x+%e(S^+)awfHW90A*P!$^=C8~;FSRk=_IumCfSmN|Y9%L)V9E=-?(+lX65vON9 zg!yM7m>^D+qb}b|E9z-ILZ_m79si}ijJJS7A0wz#aZc~}bUZQ|apmVeWtSK|IyODU z$dC{^x`%HT7fPYY;r7Y-m<`{A>kY@5ozL1G_rpNn#UIX>8xYnpZPRIoPP=t_wN9_s z>0fnvpH9C-+B~PZ_mL3j2eVaiQX*FQazfi)$La!S4fJw_uVyfw_Jk?#K=g!-5-1dh z-+MjA$A&W>+9vZ!%r+_$AFjY#k#Z6n4KGy$hI$fv88av+F^}Hw;?cc9cy=@|7TYn$ z%M`Op_;4Q_6EAm%o#|ku_y??MGP$ALfeNs_(T)T&8}|v)DMJ@%esYI?PcgV`mTmf; ziS3j=6I-U9*j{{qt!IU-LUyy@KCq{SDGo2%>(w~kmD81TVw<BTHf=YU2M%z4<K%`p z?th-&^dn`oaweO_#Uz_tK*9y>Iu2^zED-L-P5A3fa>Yrd*n@&><#^y7TqQjtbDqp@ zn9<-o-Gd*J$Cb#1Z4EHTh5B(6Ts)-qjs~xi2f*&Z>Jg`VuzI7HMJ-fkpAYNt1RfG` z7{m^RDP0b6WKZsQ$YI2t$_e7dFKqo~q9yC3JC&n5Nb{jJIeboX_=F<y`Bd}q1d5Y% z{2VxI&u?_qjG*?bdM?0b<}Xn3cHpcHPX3<v0whFMqTUW+;IC;I;*ASi*w|v6Vn9nz zLrd$O;(X0^zv`9)#QYq5fxB(+i1tR5$1dHONf>;O(T1@p!vX1R%E<d-;+M`b{0Dm6 zd|klZbU7SqICAHny1SA82B1;Ditxvp`4U1SKyrV4KYrDytPj7vz1ySs@prq!`4U3s zii{EX-D&1i@@<I3uUEYZ@=JUX<vVtNey2s^_a%I{A>W5;e_;RiUcNuS4d{&h^=IaT z{y<Vb@#hmUk?8#o%vbwhy_~Pw!t>Rfd+Ar=+b-sOb=FcO;pdh)xQw&^c{ekRnZY~3 z_$T0MF(~bu@4<w%lygORKPNYE5YD~fi2)tvAiVhTDsDSqLTDo>nU{O5%R=X(v-Nr% z{jDBo;jNX>YiL9y{7>O|uQ#9mQS!6%?bN%_2Fy@*aao3Rl}^{`^dU*Ngt%=Ie!ohs z_j0is3SLu{HNy(#G|QR~ONFdyT=3DazyZyU!=EE8|FTw8g@|k|68F&Qcp>BAF1aje zu~$P~aDwi}=jEBhOl-+w;pV@moke(-<38dIW8Rtxo_PNlPgB%;3vW^KfE)(rF*++? z;W_BC)l$<YzISMyF&2!1Hj$q7yjm$?8L!TZhhJZKK_1g6(EWryCw{_{7qQoywozL3 zwo0OiIdBnf;CC1CvKV*I)fr^9Gz<*fG6;nNvzl|;CM3craXoSe`RI2|a{Uv~$eiWS z%_w}Y!HKOLu0p^MZ(#%{<l)z=N8@`f9*}!ez$8;{woR~@LK^B0aGr)|>a>7q*47E7 z@Z5OzcNiM<d?Fyep1tx8vF@g$rIpLr9zN6IYXODOtAs_eN6W`l?xw-WhL_S+h#p+V z_*)Liy#)<-pe&w>bhktK>2gB^l0M8A<uqj~cm;C?@Q~&r3+L}GfOwxHPc|*(Y`*U} zFy(JP-3=SSD=3!S)+>|AG5^R7o_ADk#=m^W=3-we_3PZB|H|F?S^tT_As@%0S@tNj z38e}UUA&AJM`q%;R`Wc5IY7?kLyOW<0kTqQB(c7qQ0hKP<P}9h%NO8)l>ked?n4$k z@pg``O4X)-EKiYZLLOOCZRjLCQxo~QB*|Jjp_KPCF`u{_Pew?R8yJG$lov6u^20NR z$$^D=(3KR%(Ra9c@JIwAqk=DK;D-;+#^AH~W=0X-%s8KF<d_)7i-yH~C8HIrv3xJJ zf&*Z@FmPr04RVMc4yrW8cMU)rB8EHZE0{lx1@GbPWD-o;LTxm&EyQsYocZy70G@PV z%;lkrpdR)xlv2Q6#1HZ#QBS%)hQjgl1ANZqiFug0kccrhd=XQ-(h(^z6OmYE9!7@~ z+#EMu#x9@lsnwTyaR(cUu(h*+syB9Atk55M^CzqOb?ll!%U|<%jsG9k-UYtSqT2sX zYyv@xdj}}wC}6;#t$?K3Y9mxq(v)YzPKmTYkpfY$RJjF6pdKg$l0debjX6-HYSF5w zRjO2}&_lVT_u>W0y@=Rwe>MRc%B7V{^L~G8=6Uu`iyqJUzx&f>=9yWuW@gRInl)?I z%xn#xq?z2ItNZ4w25X#*d9&4j8QDp=g3yf>9IrZay0nz-=yxg`t99~9o?0TIX!+?x z>bu@BC~|FM6=wL2$<(?=8)J%{!N+V7&uo*Zst6iveJ;_li|Ll^SVBo#B{Az2W9B6~ zj0BA>Aa`@CRd18}fB4sVqHXsa(N=1ob+%#_eh8l=bZz%&{<>apGhK!rT&48c#tvH$ zx)pZD4l5z^_w*P%C>;^IEwij0dLgOui5la&Y1SAOAoY#$hb1a8EAcWbRdo9?ijRxF zXT{v0__(Jnlx)tCSrNl^xUo-B0d)jbAe|d$y^ML5-lI^gd4dt{9_5E`G=vI;Forib z@|2W~5Q7L-JX02oIHuNjP2HAKiZ#Yy82mtGSXG)XwF6kGD_2)jkD%^mHAFl2SNY*| zkH`dD<tmJ1eLYR5L+>+^*sfL{&ED<J${PRa2sKPDvLdrhxj$5sj6FFmUFTXM<4c~= z|0uwg@Y{-(@P+x^*4H*3^WS1s*ehBtavZm70|9>hYs4IN$yYQx>^?zG(Wwt~$jR_$ z)&LXB6qg%y>WPvl-VP^{eg1mzMEYAl;kZL;2{LlWQljzmLFZ8zKl$ahQ^NY+<kL$E z`zH4NCgOu%4}o8O&?qzjy*f<6g05DkVd0mlM_va^5JI}t;#B8G>u)k|s56SvOEcC* z^QVK>*2{V$B*L%xDUa0CagTT|i2ufisP-&X6~D~?iGUbC{HvDo=qzq`gW)fpV)8!} ziIbEgeC~QR)cBF5qAiM*RpEcBu&rnw9KcEkQ1c-JfJKG9^8q*)C4&R_i39kx!ytg; zgkgG&p~C_E3jRVC0{E-|E{!pq?f`lU0J{s|lo-R89l)IhfV+DEI5Wnup94580$3dm z!0qxc#sKgAmsRti2w-jaYju$c0C0uPGch?Hb#U)9E`3?A5?le%?GE702w-)1rE+>+ zD{6$nE#5s)ni;;o=H12Ky~Vpvd-qunzZN%t1%dhH{DDUK^&Vn_|9;bdf8Bo{MA4PQ z4L<xB+`{;Z_7`-^PAc<gbTmdiuk&6r+7B{+(70#5Q^!qoy#wb(;TAj;!nYIz0gN^B zUPtG9rQFHG%qsmzlU)MfbSCZ0M4YVt5n;LnAZMLSN)xH*Dumu7JT?YKOy#tH<9a%N z`9^i6?2VFckmEWilbwS|$Qqw3l-8NtqtGDJ*aMfUSNX=+X;022w^}j_^e`|FJK(&~ zz6<KuCH{Mvcl*qhkf5WyAh!CKi#JnR$K|{(=q-sHR-sz5^LL9!^|A@mORh%4bk-EP znwC5$CqXAzsVxG&`lAiXCHuW<#$wuy^R(Og39|Rr+5o+}N=_!HS&SXJAbZDX3ot^> zN{n80(uv1IHnFT`8m}1_p-C;p`QjP&WpfA1?3=g2BdC)o7yeFTBi;q7c8mHyf7b$4 zNEfi2fOs=!s;j=q>4jFL>GorgtC>E{k>jWh^?g3g)-c-Wn`x+z+N|=EiORDTQ)@Bt zZ4QM7URTtP+R&KR{O?}t;BSsFnQx2<4Su%oQ19k=T3nfjpklP>D6WiUozDKo^+OZP zT?MSoq|o~ei>T|5Rq|Y8McZY5N2)-xYpzpE{YC-ltgV`m3&-Bi<n8rML*ddJcA)b# z($E@~si{|vBEU{Cgh4M+%$dxM#dD@x=W6{uaD9QlGwIHuVbz;2Wm<Cc<%@=V)X~Ri z{XM=yPNXlk`wlM7Df5X>q!@s?QMJdp$udc*{6w4E1Rqv=yzj<%uXA`elP;BEVQ#j_ zntcy)2(NYsH!;RF``$-r{vJ{n_>u6ghjpsFKV^P%<^6)gd(%#M|8lJr^s^4{H6HI- z!h0sEXY~r-9FOl4bar2VE6%8+cfvQ<;rlzQY%2amk8dC0d&cnHw}@sAe?b5bYE{vo zu;5hgrlk0rPH=1u|4MF6+1u2+nlg8qBOR1pl}hFAnXTmN5BRsyjlxUuH)ZeXz;7E4 zEAh~s8Ymir{AP8;_AlkT3HAF6n1ae!bg=Vmt_@*r?!K%1{LhNCi9$R^JtWY#vhJ&G z%6?IzHTzv>Yrkt?xyg?nPi!!%4Gm=M7@d<pJicy^ueuoDFk9Bg2NpZMr4<>QdfrBw zTl_TAv)`f81f_D_3#m<teO?^>@OGGo8r!o-`zW{UTr3}k+NRJ!zesbaHpef68g|s< zXsICEKa<MtN&(F#|LwL(g;;6i+BsL?Vxdime-deP_->ZAJc4;Ve7y&Ga#Hqar#?KX zVqh^JP8W32$boyD(DDlC5qqZoL#=^KvL?I>2AF@~=18jflLI(30$3Y<UjT0|ARV*r zhXB+poyOJoNbi2#yWjHeMc$p@-9B8U+t<5uy!!}lk$FIGWY5doZ6>`sf)ixFigrdf zrE)~7UWS@n>Mn~@I>C`JjnvjN6}KwUvGJoovlTifh56}dRvsY%I)pYm=-f8c_F<SC z$L_SR6|TBQGA8yE*eI93-547@4$#jDSxt!c2T;G%2DvV?2KMuy^z)Ja=AbEnvCq*< zdj}k~ve}L;3VrLxf?_oPTa$(3XXOvQ18mVYhAcLh04qJe&B>T0&gVNZW@$~I!hzMd zeu4?*5~ks%pZth-vA?3opY(prC^gGM985BI-yKh6X8cs5+fGX5u65?2xkH{7CQ+H$ z_GJw9rM{Wx41=Wv`jQdoT5oj9-6LhrnmNcwnZZq=<Ri1y3Aj4PprVO1gTe}_R3y&T zU26s60%d3NyLVX)J3T6+_FgPaaTT=yTOmZcznC`h_RMt~6P>qPus__V>dhTiJ#N*{ z{FU7oj&K~Ph@3YK&7h!~uH$nrtA0>eXsC-#tZ!GS>`U!=u%_&nRew=i)mj8xTrJk; zoZYC#B$KO3(4B3ECkUB<V<4}kO6AH9Lrt}h4sCEJunujg7qdCE4HZD)yvBE~7B40u zEo*Pvo??j`(KTIDoq$$BW?zWyh#3(KQ0@Vvh@xVe<Bp=V6i+>l)`GJ}5HVjd?VgC4 z2Dj*)%1u6${_$$!IwOT9tS6YtY<K$sJi&_hN2D_^r|ZJR)%WT<i+sGUB9%FAThRF` zTYFyv38qyQ7*HjyUI0Y8?&XV~%wLbMFduSWgXh)+8I#FnAJlEU=+mj(UZT-D?Yn)9 zT{Br995tLgXLpP7EHD!1B~3W}c940}*_nR+{fLv53X=OjxvkRV@-^XYaL9bfBDhc< zq=J7;!alQxmQ$5WDXp7RucsZ*>AyiNYw)G!O#F)+>_y|-7c@>rhkGll?8HkHn;W$O zrl9@kB^C+~$=_MnKcL#xs=nxO#%`j68%p`xMW^1zuKN?FRnvKvAuEDwwS0n#vA;xX zi5WQIb$gfcom^Az&fC_@>b}aEbZ-O*&NByNAa}dBKI#)=j;K*UAF+w9bFIZN)~6o^ z!h_7pAL6KFAH$!%9pvC<f>U3e#jS$pFL2z-+wc3ctZbVFI!Pf=ot{{T8xDy8qUr&r zk(B*@i?iR~-2(RetP$)4PMfgGpG4TsCV{daPr<>_4eRptA`5=KnlL6G+wW8S>U6d` zmHCUNS!c!nQleuLLs_QV#!^t>pENZ$?tRtJ^_z62#<4HE`2@}8zFcJXjxJGc)i~3b z=>S+{*YxhRn=dhUpma``zM<DTQuO}zI|HuSNMg7pBI2!X4Gy1NQ<doc6GEEV>Z?K2 z={E>Xf>RJn@R;>I(ttu-tBAwbi{9DtmLr*!I1{1%qDHHg>rR{e#OSB#DaWG~glowH zn{a1MqRr!_78PVXhjw-`UEYcvPC3Pxz}`Xj80J#mB9OA!IOSADc)q0_(HTQ_W4t<^ z(?LQNGSyEMYQ@hC)1x43?`KpzDl&%<)p?;J+@TYtlbOEK4CAW;5kGZ!^cc!gYqaoJ zei$`#fmLnz@u<rOxmIT8Ij!OMJAE&6JdG_OJJSS{yH!(F=frhiCNFM_rckyYggh}i zcxWK(yOM-a_oh478=Ih;hf}U{<H`)}5|vD`^=?#PomonMzDo7GOTrsJOqH;m>tku1 z%KR<0;`uUJaZLw98p<J;tWg6*heM|3&N)bjU^APWGUwX?Iy8*ChBtQ5qcZ1L(CXj9 zfY!IN-E>5c8m-pl*nXjj>lWBkC5i6QWJ}+12WAAsnHrSD@afzwZk*;g)2~KSFJgtJ zgW@gW2}aZSy{9|KUR`{&8pT#;w^)b_E-*ae?MU5gdYNM{iJlVZN;daz2@;<{7z;Y{ z721%<JP?d|JXrDC2#o(ql>p)Fbr$ds1rOzCCH8vQ6t<-gnfx^?!QHH#eNf=Em0V|l zRp9#e0o{Au*}t8hw-O`QwvmjR4{-<a3P#C?R6INQt=UWOe)<Cv%McJvBq%x!{kuZ+ z{6cgqUMiR9(t!=1dQKs_#YLmaUUY@1v7n~fLf}l7h%B8GuQ&p?{eb)goLHaf&i9U2 z`)OH}7HIZtINia_rEUF#o#(Pf;oSu`5bSb40$As)j{~X(?$$a34U|sy71>=y35+*( zGCJIHfQt;>{`^bQgGOkp@*DX?o*jNp<1PXnpVPeC^&i`YhPXzryAoIP%a18_H2vq_ z+p76RpS{#UYq#z_;JuL;l;8R`rz4}&9qzq~DEDL>-l=s;p;XLQ{`@l}m1cpl8Pn$Z z_&kYHJwC9+kIVdQX2O`haE~gr1LaEM7TF_yZLS-n;B;{kghvCTpZsb+^OuX&x~15) zm%E-WG0ye$y9^DpY8<kN5T}{=t@Q+rf-6^V1Y2af!Cl-MYqDnKj*M-cFO^DtxZ5@j zi1k|%Gw#ppv?vn6&FHwp_3#;y|5#eXum7zSqi!NL$j;OvW8y{fn`FVUh-A3>GE0Au zLUW@ohl9za<TOf_FSCZ9c$83|>rp?BfZ<R-Wz`vV1hL{jLS5xi>zE0szv)o_k!}p? zPl8&)$_SzUE@(u}Mby=X%Fn3=Cp@%5K=pnTU*WeXdp?IxfW0EDx(ZB-s(4Nh=NFsL z?`tK!yG9$ChHjoi*Wu7zEp#6jI(s-4bUhJZ+5s+ffL|8i78P(DKHDOgV5p~yziwm+ zC#(C+QHU=`G2JUOMy;NM&+oMb!iaZ=TmF+G=f;0NPqfsuYr;Ode||HoTL<uv2Y5LK z(D=w#qMdR8OB}$HVGMAZ3cpJX@LdP6YXQSV0X!E2oaX>;Dqz@G0DY)fToz3Z;JO09 z$5n>+Vho2kfD<BsRpDPnmyg8&A8`O(1%S*P0QQfwc=_8#hf4}{SVk|;e+>-bRN`Dg zRy(LyUyAE`q4N1qoX^i3!1D!w3xuXC2Dr)ryb~2<O;{@oyG!+j;-K2QRo=bIyHdI3 zw@aS|(7WD!T!BkH#8djHkW%7f{GlQ9pEGw={!7k{roQ|WKE+axrq#pE_imqeU)~{I z&Zir)@HP1)l~QqP`7SISY#v*+!*`H9Qf!5ZeAl6tidnro#_#wI_Xu)VFvZp<hpJ7F zHR{{rHQ2YYoMX~E9@+R|9xUpdSgOM@ZTk#%9>o=dEL{h6#1xBqJq1xu0zwoOhdRE0 zZv}ln|2S>ozgzXaHvVS6#9kucF8}epWZYKG?cJ1q{J|<k-PX3`*rz&<{7l>K;enTm zi?x>pxl@sFS2N>t>-nnu;X)OgFZE8-W%t=DDD_a=CvB4D7R~0-PHJq>K*ZR71ecbU zoZ`wY+x4NgO&yn3m9#$@?oRI=c(@3kBPMde)|XnzavMiz)~-6i`slRgaQ{nanB4fI zU+}$jRrqIh|Lmw6$5Hz1__yV-SGbn4n;^C+{GpPdM16xG=sjsVl1p?iI(6T}3Bzx5 z@R`$2D>hGTKR?L4fJE|^imnqq%;E^cSm~)ws)#OeUgxSNDf-aBu5Z;kK-aQnNi;&1 zu2nq^NJ^`#jc(i)@7g2iJffzgeWDN={J^7b{7t^tYIg4Pam}`9JY`2M|Gd>~UthCV zsIN=0hf6WdrO-*Se$_fVYK2Sj4vr+#yF?-07y2+gdSE~OA>&P9r;<aLr-&BVMgDua z(Q8w<ig2Ts>oM8W>!p5U#dXj(K`ax=>5+h9hhW>L+4$h3K(*SiJlsQw^k@XhK29>- z&!=Rk4o@~ybF7HdC4vvTZ>0VAMK;;bx1AX$H1#O^`tCqs)xdWO9jMr@yZx33#Lfzi z8=n3e7pd$Sja7Q7WwP@3_d)2^=3(D|vTwJ~_#Qn-2ja3n?x4)7;U$dN<G8(xb;m(A zHAZ4j`vHzFPA+=C-N{8kv0UV2M$U0WBA}BQdGyZ*RC%fDK9{<EWud;hy$enCWi`dB z>MyoQRZUnb=Vs^>(ov7d%TN9edH6XQX*Ot6h$Yrk4B|tt8B3p&bbC24G1Z*a>|7;- z9XQblu_!9^Fqx2X5S88cdtZM_W^(i7A@z=kNNxSYWW$FC7JX2ENOZIEBb7f#DtWs$ zE}avSza9gqO$+6W-{C2=?}KrB7I<Rzz9p)j_21(uDyPvpEWq8PbY87B@V@*l-03sy z9;N+U{d8~o|M4EBr!F!I74A`LoWqzZZrzhFRKe!925uvQedMTV@v6A`xt6^SWYHM> z1P%M!!+gW46=Tw=VYS$c8<lY%TiX76_Z=PNO3iwR1}Lg~|Bm*#?f$-_VdJ!GSeqUa z7tFQk8ULb9i^l6Ch4!>b$;Rh>S_RXdJPj8u0&mx%*H)0rOQOgrab!gt*)J+8<v%2( z>aMNZ5p7&bXWmW!Lz{Zg$)7dG#%(tMMrHT6&kupljsNVey@v4Bwl-Kr4QXH%oF<|t z=@$NT#T(oAXZ^(G`LBkLUEujdQSATf{{2_O>m7>t{{2q%5-zmsz$sc)eWX=g#6p9d zdoW{NrxZW>aq*!_t)J^q6G6(bwVU@j4|UL~hx)qT4ag34UrtbOkUhn*6@Y6a;DUX3 zd}r6EphQs*jOT-Y(GUOm{rs<nvYs$LQ)c}!rf}Q8cR&9W>v9oiGyBbpC}3Pc*D<S$ zp{nwgeD)Ueu|)SS+eNNncsLrqBcA^Ei=e9V5Ahk-zj&M^y8mkUJpZ#|71Q0HRr*5= zv@;J>2dEOM)v@2vqHDQ$8f(2*wdT4UvgMz{7v~SWDKZ6LM!`Ck4+0MSR>4mdyiyj! zZzJCD-$_Kr@}$H6Yr*d?_=UyrF+a}NLq6_biZu?;P>tg6jD6c{T1Df#KB~+c@hA-l za^qVymDI+af$>m(H--~yFt}S?iLryAlV(5Td-R8Q9?iQE6xpc_AJ5ZC^{>Wr`Fa1( z<9X$|Zeobu)A%Rj`S5f8H{<zSkQR^U+!$Z`>C`9g?r!FvM$oYF+!6tQmlltJ{f#Ks zsS^}tl(>KVAII|=YW2Sx&%^GmW5%SK#p9-?sD6IjG{rz4Fm8sA>+QuOR>EbX>$xHt zc#WN|G>J!lRsMcHci<1dwV^tUKis-Ao}OZ;s{FS#5F+`A@rho?jW^xj(XqUlPsZo_ z8gIXaZ232e#~bh(kTyO)Wbm2}fqzTzmkWNN7``w*WlUQ<9{*#9@%k0}w(BFJ@rhk6 zg5VXS$&Gs`HHGrrg0^$robYiZl6{#AB*^VEtpC`3V~CjCH&)b+x^JvV_M03hvGu@T zTx6l`B=f=Vc;JidvmKXCC~5Bw7o1HEStqf!>7d?4@&i^S$ou|0rLB_dwnwkF%|g`Q zKNTNZ)S^Gzm~6_tkQ(!}UBdUW9kb`Uid5pZ75sll_uA-0^JU}_WZNr_(Qa^NOKQwJ zO_>cTPMS0Um=?refuL(&e&pDPDKEQrWqWdXb00kEJ(E1Zw+|$2+mmHv+dkEJM&pi7 z+|jn@A@x0N`yAr#w#b*ZvruiUIED8E{quo-z3Gl}v>mv^%^6zV%CzX_Z>o#VWhaCE z>@DfK=h_aXYe5IHKOCEC&h@Y4T>pP=;QP{kJpFhY&_WzFn5z35dTtYyKMj^+b7h}R zXE;uZb-ap_ht<<xj>(PObL_)S-49=WRD^dg>bIBbH&gYSxlb~kP48Kq&ZPILo{Vqw zWPGEs7)bAV$mE{1t757%<Q}<n-80-=DZH@)0yGB{?jM+?a(<I?o~LqNKshg_oPVaA zDv_BepE9dBd2h6kJM;!Pf;!^<QLi!+owofP6GL~=1<B|yW^R)?Ni4DbIrZtxySnd< zlr>RG<0mwTb=DpFV?3S)o-%s;Kk85ZGJ6O8Nqiodb7fifm&y~G@3hbNY<VzdWiW#G zJK4Hf(a+b0V8j$V`MV}q@dDp#f)VGj)C^Nw-?G74wn6%hQ-f?(D)V?M^CtI?Oc?lt z>mNb(<68FCD@=AWT%q<>s_yaj!-BfMv_Bi<E~@Ycj9v;;-!MheU%Vp3o?nXj*C}KB zq7LiEpS=~7%Df+RZtgIXURGy+fl6<1c1-g*?drEVS=&k$VY5h=AgKeh6qXT<LJh1% zoDSu*Bws~H2nfG=7G0h|3AVO*`P4odq8t&+>OOXM&x&I0!H-K0BweEGt9wMpBG`3r zA>gM<*!@)dZkhnPXI3$e>wey+ysr9h!>N0iIwt`6QR0dYnz8rT5@e@v3~dS*Vc0R| zl~{5T#32I;$5@_RGdpadq=+y4Ka90}qO$~~Y>`pha1`yXrw&`TBo688>?=L|${8wS z?&-Z%mn%0?h*b6-mE~UA--s|DH{9JSZx&4|99zh;pqXf;-9_7dGf1vuUTq%{qO?q( z+)lV{=3%|F(9r$S8PRO>xI5%(%VSManNbsG)58%j5p$WMWz5D+pbQpFCnc3N<t|Rz z6+4+Ny0Jm0J4#<yk4wM@wF9W#5uOowQ#LMIm-TejJP0PJ-F8umLml{2*_=8ByRakB zpN_hYV`*;ZG&gR=*goU`dzC`7vIYhK_}|giqG;H_A&|<WVC#GB5`88|Hn2lkA;4LQ zlU6$54;xsjZgu+~9C;Ewb=a7x{M!71k|gIMZx@-$-Yk-^hdD^PHf{v$D%(><YJ-ru zDMP^lgQAB@p6Gs={uCtcS^M8d@;T`qs)58bc;uKBsYhN&J@l$BXRS_kzOMsbg3K(o zGuxBy7GRnk^(DC2hxQ+v8&~5tM6fe5vkmOnM<lNKi5Y)G@^t0}nY}T_)47YQ)49ub zsgB1VJl@GK(pt%?<mi6=J<<Li7s`KH$ewOL)jXL07eTif&u&S~+(6(ODmhNaow-2( z$jVv$1U>`6a{xSv{{agP!&xmDMd+A~&{iH7nxTEy@2LRa$)gAL_eA@9+&1}7+vc)# zIjMns27|>J5I(X-2xn~z%c2~FrSiCK<nW=k>B6<Q{_Lthr&Z|haVT;}TMa0vs^xKi zieM`@!PFr_T@n83bXR!cs63A6bJvunN?%KrZcf+HLWyhazT=1RN~w#2^h{cT*&$&s z+JJ~|z~4&|7hcqeYEG?qu}rV>EmHLpCvVpltlAR)MOCR%PW$``k{%(tw$&u9Ew6r+ z;fT1Qv-o|07Un-Nk5H}lry=XPQ?i}_u5RG(vFlTvtLe$3Hz<DAdYWVvReOY}nNTxv z@?&H$tq*{Qi25^8k4AyOCikidmz-u5PoB2HBc=XR`V8Af(PFCL8Wef#M$6{|b5KXw zDTj?NhpABxY6lftIk@tw>s3w_T!SKyRVO5?52}ND-A*}#E{9i6^>r95%E3DKEcLX? zse)@z<gw~=+k6i1;(vnw|8S%ct!BI@dZ`0G5zz@~M@v!bDwk8`6zhRgw>8zhlDO(r zgl3KYW|_V$RLj9Q;8f$^+@K*n7{Hvy(8}?61S9=(dYat<tR=Q2PG3)kD-b2yUm$8? zH_0dl`@BT`gQ#`khtYY)<v8dZ00<V*Njd!yo^gIQam{(yuWXr|I3%ww+A@hAJo)fN zZpP5l-z<C6y0Q+H@oe^nb<p968vUhYiORV!<Mw3uFeA**=`W(CVk03upDM%kyKktZ zGKZg9pSY%5hD6!MAgEh)@xej12^*fhp5*pwYVc1j{4!wSU20g@snwG@->*+xw+?mO zq$}U=fL@93zQSu9p7nOMi%CI->zD^%X=0J+&M^0lXp)@RNtAt6n>?Wk*SZ#9&zQ7j zyM*&+8pU>R2Gr4N`gAox^Ty8SN(op=dKz>E<DRJ(a~zWGRg>sU!cbxQhpealr`J>y zHysY&DPy-S>BJ!`wFy0r53_?`O5u&#+VJP4uea0L0DHQo>{pVh?6)f#$MQNBXL_Fr zGP3U6rv1DM``jiDQjyBm1)0Z#5#wqmC9YqU+S<>0VMD6)Q3t-hsq|UJS*oVWM|d`G zQ+mus$$4BoQnxNhOj(IkM%&y-$hu(bUxNdl2<lD^5?AkO*0PPwx_>s+$;DlnBQ~Z> z2bx&ybW>ptQbI7ZYvY<Prnf}5_*?X<#@MI0kdFGb*X0kSrn7o6<4j!rV~>-D=D6@{ z;jR!E=2OZOpHi{}BWBjjZmN4J(e+xHb&qOF*1l#eJAazL(i$J>%$YUyDGohKGDZ5C zFt#t9d4pw%U#DSunq<5tk4$Br)}452R%>PpW&bgE$Oyi%c694H!JeSL!dvB?YwEGv zt5?vev|X|(;*1|K3w)!G4zTky@QwB1Pfu2_sch^M(hc~U)>|gVu1*!BY7!UxAe<VF zAAoaXIMw3>U9z!XIK^0#Cm4e|H55+qy${0K|G_xJ@SPd1oS;iK4hpAOY4QZIrR1q% zSxw?}AB1!8gK=&Pf8lW&x^3yhX{*Fl-=|;LEygtOgmhh-=E4)yS2FMBU)iP=HtR^T z8{hvYcaNbHFPAeg^y{E#CLt|+AKgyoMUCTT_5In-K59@hjmkW^yd->q0x&>noas#0 zo&5|`4a`L6wc-ej$w5h<yYeAp&~Mtn<1kAjhm?@&B?o2(Q(IdYdba#Q;$-=)^$=rl z%pZfg6~jl&3L7!3;O8=8&Nr)FjhGdTm_A0#$Zfon*Hrh9_Ft*~syf5u7zR)NQFJG+ z-js>bZhja7z^!K`PS_}Uk%nIR$6ct!gG?xYoRv87Px&+7DPE8|nR`Y42<+P5jJA3| zP3_tCZeGjiai%PfouYYV)Q?7qYmXn2!)pFP#+~o~#)D4NcoxS`XI_UC?)nV2*3V+e z&BAp9S7f5A=yP;Pk6D?j>k}P<x+bdq`wETK6Rg_Y07|vX{+KR(M%BKnsM^zL@%+>8 zs(D;{&lJN{iv(l;o}Wdm<HK45=tr~9gl!tbOpWb>XLkP?3RrnNK;z0wHxSQEaTgnx z*3*7X*<)$EODlBhjJ4*olqP1I_PI@b7Pfh~XPb%ZS2(V@J}7-e(6HN7>60<nOjq7d zA9zsD!WVPRblo?o`={c%i%G7kJtkB3Y*Xng#w4Xn%s)<%cdnPO`TAh&iu}fZ2tw;n zXRupXkapf((;-S|X0_hYs`-`CFqh@RpPBl!<PwA|V*S#|v}W;|6?2o!y72WAq1KhF zwf-rxQ}gXl%dxiO_{JDbpSio;Nfn-Y)FFd#?-NAm69VOgz@R;-W*OtMzt8g;qra-y zM)oiX^$@XjaIS1qP|94g4TF9662+L|2K#pz?#3CCbb{PmIQuQOB8PQ0Yw@E664+Zf z65E28j_cNnTvaf8Ef*=U%F|*m{QWddJ9cS2h*(he18io~DzWVS6rdl<offSTL;MVS z2-)1zI6E<>%j$-qvLZ^tuYoXHn5uB#a=HT5u?GmH7{pbH`t|MCol+erPf8|C#piE8 zuELgtWkR0W*V&d+2(!KB=7@0Mc)H60Tchz|OF<(|X0Y=`XGS<E&d>)#=*!>sy6;z} zQ(pO&N<6csS!{5B$4TU!Re_)BE_rMP92=gpgH()wTDd1N2=p_&V0^N|rYRkg%@dT# z)+r#X+(mz;vaI3kvt69498RM@CL8oeqNcK}0-x3n$5w&VXXzIbh8p>4=lp<GerJ)N za+#vZx^gPl;j=jVmgQ5eeDo*E2Ua=y4dpYMd<I?Bln=p=^Z!j`FY=r5x)aPRr&TM$ z1y)WYjq=-yF52>|w){qW@v-|tI4Th1k6a@K(d84*$91z1NY?75v}ArUFs3(%`hu?X zmyp|xyc2bwN%tnz5$Gpwd<uq-4g&L2HxJ|IOnS{|fMn8$VWspJEd6WwGCf5&lF^bs zfKR(<;is=E+%z*u8@-p9z~s88tTs=jW|1&lhi2!PT-p0VIrgd4J#zVVnD`XrT<<r1 zfYry&cWtyu6)CthydG3C_EnFcH967T4-F7hBnnUJe7;okD6jcqajA1lTOkvK{ksS& z5<A%!@5DV<8x1e3slMoA184S%^V5|J@^LCNW!r^mAa{Rem5JLc4HYv;MR@vj4U5Cd z?<2<0UP#iJ<xA{SC=8o0G>ttEp+)lcMN37X7L6nHR~?T2L6y$8!4fta30%`?$f8nX z`?;^}DE8c!e3X$I<eDqE$Q+-Jo@nGNiPYHpNcUH*nIgPNRA#AO9b`4jM91`We9RL} z@7a7s{I>`}d3ZYM8nqd82RLm;yV}ZN<#l)eOB@NZ`|xzGbZ-yybT04a%8dSiK~_*f zqEqiBq**vQN#jXw7N!km4}3o6>*JnCVPiIC5Y1hE;>NY<(uV|EpSa`gVC&z7#%AwF zRad>kn<=|LRrf^uI$e+w)XkKnGKjzx@x1WZsdT{o==5&>R<l2<NOg-Kn{bN;&Jpv` zPAC5>ayj&$J+P0153(cIrArYRPGH*9zCO!V%oRsIo~m2d{_H@;y0umiw6CpNlf3xN zp`!p$+WcfX*2$N`CMr#jqyhL5gaBvZR(l-f%#E+S53Ek?`7L{Nh&}h&En4FT<skNK zg!e}Jd6)8SDn-gC?u}Enxs=^5<sCldk4Wk2;nLmf(*3xQ?g*Ez$)&s8r<=7yx~pBf zvkK|TT)I&%-4vhhvpb}l>e3w+rNf>vZ8d!QBPQ5=)Y5GV--dw}mve8IZYyKnD6=Y@ z=hFS$rF&csMt=A1knZm)Rc`#IDBYTHrb~COOZOv7w>i9;bdDm6U5c7Qim%$>81+e) zqRXe49;f(Em*V9@ihx?vYGYiAulp1S#wh}qBE!nSxAWdE#nulS6>5EoL2`Db+S{eL zu#m^|U?Yz;F2!~^^o)Ds6wj-kx$!q*FX8j}y-RU}OYvu);@UXH?_G+w3Mo2Vim$m8 zKkz9;4_B%VmttF?9-nQtQf=MM>M_Ho7#*jW>QXEzq}bQx@xDv(1)t(=@VGqobtzJX zQoRf|YW^peqRgjQ9jAEtSgZLPcf~cIuW^+f^);7br5tE=eVk&MOHo@W)hw6dFqfj& zr)W{_Fq6JjtOC2zef4LyLY7#_n!MQG!1D@DDT{;(ABn$hUlE(>5ae(<7l2+uOLQd6 zdBa_lkp67+1kqZRFhZoWc$#F})<hF*`*JtA-bObWxJ}uAfz$!-9Oje|tP=Oa<`k^& z9EtM?ycQhI1%C_Y0XXoO3jI}ldA3cgs$Uj#Qhz?8KUMm(AAbgZFff1Md#d_G1$|gS z*9>$k>yPk9c=jmH`UEPJ-$S6M>5uGs-1v^p!?Wx4maa=QffJb>vhpH=J%xjAFf3=a z$t!4eu2byF7oB^4)Woa8$6SNm!HSsm;bu9I?>5**JdeP{DHf;dlHn6TclEVlZaaQ5 zpm$Y69ToYz5J>ClQIbNSaVvt%k!Yed1EZhR&enx|vb33NVcbCHkFc}TV+T?Dz@+G8 zkXzP)U&k6XFNcqc8p8ee5yCeIHD0Ui1QK0;R6ys%4^a^CpUm>bUF!HDRWd#fn#!Iw zA$9O+Y@fYb#yW0eioG)%*sQ^Ha;63XGvzcQPHyHdhTp1Xv5`iG*p}#A0MhVbgE*vb z(&1M-P3YBxrm}X?f+{<jHRrt(G^0U&9<}<F7E(wZ9Knn?a%|Pm{awYdk#m<ea<qjL z%-51SH)Xq8tUj2y9W%hhHM0$Yb{i7O^1Ep(wuu^%SlPa;)~P$K)^3xofz{eVFtHTV z!O0H1;Wv(^^7Pw<eAq->x4msoZG?SPTVJ~O?%D7HDx^Kx$!Z=p5?@76X8S(<7)?y( zg(<DiV<ef^(xscjL&2LuQ<om0mjIo*P8<8s^Xg@m7gI8)c#*HzOh&+3PoRFb^X+!B z%np|pYc3bDa*fDWKd4Zv>0JZ=C>l~ld05Z!ZEa%-hr}xcAp|;%&&2>15*VJRDhx|A zQfaKaW$7N6B%cCmmXG|WiA)UFU4h{^^$r&DTzyPLt?mS5$!IxI_c_+p9+Yw#L4pYN zLBBaEUgI|>XV~WC7F(X_?GLvrnRdI9l}5iE_a)V@uwST|fnJhJ6rbi3Y`y^{J%jAk z)Yhk2JO0?BXqu*6TaEN^2R#UhQ@KNqXe!+zxjdB_NSCfh4F6l9y;v`eY08y-c`VCS zwO^`kO`>Znm{Qqs$dDtY=B;}mans7qztZ)>QVx&&t@9lYXg#9b=nY4b5t1y$z`Ueh zHI+O(C(&~~0II29e4e}lB@gSvleH7HAyxNo`=eSsRigDu!V8*8pG()J=|o*$qLtI7 zFXUxv?buIGBec(rpMunDLvU4iqIl+I2y*Cm2Xzfd11lUWzcL_K-i%XbGY}KsO>~R9 z4*ZwvpP7}W-DAoB(96MYD^)sH3@nXNm10jaRthy=AML##o<Bw5u{{X-oH?fqtCp%` zH6iOiNhA*H9>GYTR7m&8(j!r3tiOa8XI8ZBIj25Kx&mna&vZ!#V*3yJL|ndnu-gii z54ah912_2cukU;Y?WD>d2D`2I)FuA6E?&Yjwk@y)q+qkXWEG=cq&9C}260P*N9=Yo zzFRbRXfN^Vud@1d=YDJnC*b;^^If)iuVTSqT7?wa9qCXR^K3D)*E=hzy<T_!2?}sH z3>y^hkpU!Z*Ppo+vsl`nSu+TIlBe6w>q*i9Kd9v}jOjAkd(4?)SZHdi!-Yp_mKsd} zr+Zi6qa|nd&q~Z#Pe)1D4D$D^K?;Q)K^JZdca8RElc!hc?>^RYv^qPk)rL2mx+6w< z{|v(lu90kgRDK$KA8I?sl~d}2<mpxVdy=>){jnHZ$-lzJtf}%PwSCK3{-)Bc>BHGQ zN_0QV+$#>lRKlUcQZ3~lLi@;paTm%aNK4|HE}*zQN7|SfMrCcq_E5`cK83!NrAY<S zmcwc`_H4uaj0#p>)qG4F<j*OL`^t0dettH2!l{#Nma+iap#qw+(>OD<KAWgKn<SOf zps+M6GoK+8ZBFGG!Z9sus(Yb5qaw}@vb}wjgbAc=Je^4jL9y3XaYD=v#%-|mzcv&p z3!~W#9Fj-ulc@YQtHnXgn3(S7FxhrnSHdrb?D?W;9{nuziF!zN<?4{khG`r%ujm~k z?E|2>o#AGka-)-aYEU#j9i7^l-JV0i3rXf;gh6MfPP~H1Uo;6#xg*LvU5ly6{`NZC z#W~$}UQX&miJ&b#*a3f;;Ye+J-<5RHaLbXNwxfx(J}XVx3{PJVZu{Rw#C5nTr&$K4 z+h)v3#sbdoqt{iQwH{n&gC=>##$lOt@0?jtLq?gQUMX(LC^O6E^w7Gsc77_)w=B;h zi{#n;$}Aj~S=G*&71hjj7&?mf^VMnSD^_;<d%o~lzj9UrbS90ahyx5KrpS2fmv<s# zQO(?jco6|BrwL)@`9fHE);8^pOh{bw1&tmCUV6ltyo8L|1mjNkxw{d@n3@m|zWn9V zjnN@z&>^n^2BBUvCt9;MOF>NdIO8V&4g2&m2)%5ECi!l?>mAMTQr)<2tXM$i<O4RQ z?27k`P|4cK;-8^p)!tKW{uAWvX0$t~G&9;~f2)pdXNS*(x3Fe96!wXfe4QKw<kK3W z6kRUGd%y81R)wE)Dc*h8Qp~gzo5Dll6vw+1V{VI6RC6dQKLu5cE5uj_kc$ejDjXtl z{-78j=>RS)<oS02l*Is>Ct5Y0DPXu+gu?>TXYqgoNG$ecSQ9P)1NEHiN_~eM>Upv1 z!dix{*;NA^nJkv-zNJzS(m@-A?;kF}O<}iB5G{U28IZOfPNDM-Jg5VSyAAMw6zB#p z|JoG3p`wu0v6gak_#EE@zZ`gc;L(AH2N*cs{xUkN#R-&$IkPG2wQI;Y7z2_=g3fmm zmvFWc=}@=9eLQi;-aSeC%$!=WwycU^d&OA%u>0Zm&tq{gJh)!ra~gx(f1<;J9ffmM z?hFTD5&SFIXMh@irALG}s<d{nV;F`YH}PQFGu&IzxrslP{|84D%O~|+Lv5x0K&zH` z&EYY`OKep?TNS<sdGdEk{lPisHBuK9c&E+(yy~2I;4T(qB@}REfG9aKd`YFGYOC0f zC*ERR%+18WT}LW5v7UJMvY**gcD_>2+226APi|Y+CJuLIp>X0_mY2BFV`yvQ8i})# z)W*IY-maGI+_pJ!4H^c>fVj9KndsSHAGv0Z+P(8cLgxnUY|)(hurr1~O&}Pvu>4B& zNa8Wc2h1h+Rq-eM4{P}uED>kcR5xnhA3Ld0w{KG}NDBYWq=aviQX%`xFa7iR(#*cB zyIQj4PaqHrb^`uKyuUVl22wf(gTTA{T`VgBVgpQ1&~~n_*9^AGjkI-N0x~y#aFHOa z8QP&+{xx(l4&WCK;K2gGbOGEH19Urp-xdHqFMyRXz?lx<=Y_GlhX9U@0U8{@IRy-_ zLzlcXf36G%I)F0@7&Zvtcd;_5)B${^fZ;X){6hL6I@40`-tAp!(-gkVyN8gIzQ=j@ z2y<8Er}^)zaOpqXvl|_sX5#+!dQlbA&g`$3$hYE!5t-M9dOMI-n=+wpj^9MHV+pMn z@?Ioj7l1nT5MsZRAXe?rzoq}c+Ssx&6-ehR$XmN;Q<r5pLvXX+H>w@#ok12t`Zhrb z$&AIJUb_hYj7L~eVojUMO=js@`VQMB>KnD;igzJp2mLSnr6@(BT>TttZveyFzerfN zd;ym6lHSL@&7mQ)AExKxZB@U`G|XWgmg$tf0N9Pdazd_|Fa5Y^ABxnoT3VqDtGGil zOlJ;88Lso<6$0V@$S^3~6ke>sku5(`2(sf_3=b#E`SBKe0CctaGY`6<)yHC?VYCZs zn1f=0JocnCXwn;MEjYe%wxQmEArpnQ^e}SS?QKJgn21uwc68P?G#gaz(is|<LFQOy zN3gLBwd>7pdS_?|7!{c#8#Bh<&muE-X)-)tbVI0Qs9ux|e`j9|N*kfvqU9PR$W*!3 zC{bH$F^)8xOxL}H24MRHAs`8TU`CM$Nn_+M^V-Dkn!6@{pm*nacY${g#x3x~Nhcs| z4r6NIJ`&O4XVylixJ6GHANPqe;bKaFLh20ev0ELi6BXEpZ>b>Y?5afNY>KW|6{SSt z(KLB#wOrm4^~MEn3byM8Pz`EV0{~gR=bDn~%r{_eVn18k=>eJ~Ms9Eh@`<htsYv~K zrEcFVf0K}Wq8`GS{rq{U%%5Gs_H+enpo_1F3s$d!9ZSKq!PS6bYm$`<*NRnc1xaOi zSlksRGpSI99OnC|=-O{gXR2Lp?bmDQ`EeJmTU<F_jLVT_9FXa>VdZF6IZmM*Gd{2! zWunLbP>k;tig81}4GF~QGlJ|%G}t?(ZATA#3h<IC(HkSWn_%k~5YaNv>KZM(!t|u^ z`aZWL|MBe+9@}U7CZEB&qisl}bLZC%{53ph94zpPg5ERMuS&VPk+wt}E>FNy#a3uZ z?n|h1eUv$WCU;$-^I|5lpTx;Qd{V1?fi^vTi#~D<C`G;jH*yNE#m>Ni^B^CwbNyUD zMO<`)p(N8DQZauqLer>Ag)qe|-bxS~jQpL6AsmnAC}%Ms{Rlm?rkTIzNF~r^FPHNr zyK@lE2SWG8l>kksQ$8ie%yTzwa!Fq`5xS&Lldo<ctv7e6yDg4v#N6}~>Nys7w_WI? zBJ}#WQ1?2L*)rfvy}=T(PicTz;$dV!u@FCK<_paoT{U+D($iJ5fKL^5y;8t=$Gl;; zbiOkJeG;<Fu(nTJGgm>#ghMo*hC`Gv%93^Xn772L=$3lTEhzny2&`g6ciyQOx>M|< z?9?JBgww^|9jc8gLuvlL7C>lyXX~MhNf#*%WnX3yyqDdFC;U$eVVupn-r}+99gtYC zrq%q63YlGCL{!f>m!8q6R!wv@*m_!*9B}E$tL>kV+^KsI-VNXSyqP@d{7e+B`;B#X z*#CGB#Jl00E;aW+^olmKDGOtRQmIO*E7vYm4Wnl~Kl3?PV#TSk)WG`5A2ocMi(ksa z7SO6D>hG$Fu7bF1v;62?52r567`8)t6j7(2XgRB%zsAsLvA@UA>oHsaE$YAkExxPl zt+JmNEj}6XAJOBD@C6E~8<IUe1oCgw<Bjl_E_Fl?4KPQK4tzWZvJfYx$H#mNDo!-e z=+RGpm0LeITQwWy!~{Gs&VLOs5#vQ*b|A(J1nJf=s>$PoMRQxk=QJ&BM!_7PaE6Au z|JsLRUt{`t*FPSkkY+%zuUJ~4kL+|$Mx6fN!TUxy*YL7v(jy&(=J<E--}pG~k1!`0 z+RnUj$zm}j+_Ct#4YdQR%IgsU`hD>^&P4ax_lM(YYmKStaQwUofY+<}^8KZ=f#)`V zi+mFIpQp#(pC3fD+mX(8iAtjTDqv$GRpl?Tr1eGTKEG>pkI&`hnTLnM@t-9XLP@7> zS**%0^zKi+I|tX&x$|}$j$>i^7|W`6S%9%&a_GDPOxU9rHzj9q4R{M-?t*@8dAO@P z%O4?7LRp2qG_cNzdCa!jHVY8Vcqt%Te%(SY-0bWC4blpfZD5>mS!w05A0Pq34+R&j zE2}X#!+Qhk8fLh#h8FM6_HL_p=a`GAa}g7*Ek`$ny&5UVZp8=pbZ9hWTxjcV43u)0 z(4Bagh&<}fm#v<2<gm5axe~Dm(LEWyc&Oqh9!&&S&@F@`z)`~vh@96WGH!u0NkF;r zKU)Bh=K0lO0(O!=Q(Ip&#o^|z$v1fSEL_$j{=DF0kqmg{E87@?n$_!mPBuB@Pu<k~ zyp8%s@@1vH*)Jc47VS!g`z-|SjKTWEymjGRE2zNU(Bk5?V8sCB2Huty-F=iQaU&Gg z1y7jFZQu3c3NBde+FN^;E1nx+6C>-?YEct?d}0hsi`0tkBe=Ro%#wMY_sH#%neg{M zD=$4i8<ot7%@RCKF9uhtqC)y<Zc5@8JF11VZptgr<d2^tV?-Nn0(Vo>gNKN83k(5* z0h<o|oz4lQ+$F7gHdM$6nOW1q-!pYx5Vy*ZVMml9lglFVAa-$02Odco)h>}!5KGFa zbpeIzY<mdMytK?xEb!vSEhcVA{LoA!n@Y_-#&vP^Q~l;!oS$DQ$`T(J?)j#P9~*&- z#E*WZ0cL}Tk@(Sv-%sHZKmG;?;)l-6sYIbt3>QC+-C6u-^WsN`k<Y{rONjVkS^g98 zL&b>14;3S*TN{fXj~PEPRNGVr#E(Vv852KB1dI5w-dv2$aM9W#m*5HCJy_z0)I%B( zhpI?S^oHN2+*^s)q}{93+Ti(?Je?C)zC=EWZnZ()w$eY*fwxG~O4Y4pTSFZeN$W;o ziMEIIC`;*+ZIe4#aF?`AO!0)<*oSpNW(8-dFFKT+M~<P4V~_B0?3INxH|`0uHLH8` z5)8&(zvSt{`Yb$lwDJF#1@rKOE<{bl0laQ7*$N>!;OWnp9FXX`MR;RzA--32;LeEe zNAu>tH}7>jylpU3$y-joD8AEy`$!kNb(xm8P&T18d_|Zhp<A(*u2KlhPyzves`G~; z5vy0Jzk7KDABSER8d3IIgHsaz8UmPpRP91<RS(y%xaX(qHY0!a!%)?fZU8^kHVygf zy%&I#$Z<Q#;z?5SSNpi(Gx4p(0Cpn+?}PhUxX>PM`EEVu0?V%WncmZV0hXwE>J4|u z(>d{C3Q^L2ME)9_o#_8B+H)Y>9gJLV(AMhcZg=EHa;|^J<9{I+4TO4aO!%sc@qKVR zI@aadWg!JmUAaz9EC=rKY1b|mNV{mH=%;oO0rX4P$!;OA7Q_g|an)}t;gzemLGM_N zHR&|2h;{q3l1Dbh@<=TA<Wo#^vwBT_nT-%q5IDX<M7Wv_>F_@OXsUZs`)s&(?chG- zzaRJRM%<nBXF+Crd<g(5@ys8!(6z}2gNg9Z*2Lv+JxfcvtHpKB1A%1~A?`Ww#WlLf z3e^TvxR+8!9r_%E;pTe9teuP@kotWEADU<fQEEs#J4Yl#Y&nqs-$F8BB2frG`Forf z&d*XdF@}|_F*%9ciWV~c<X0}{hbzuk7#w|}T;diff}i|RGs$1ve0>JtVAgv-miimy z4Z817be)$}on^~|#8XWVpZ6?LZF`tLh1u>+{=Yw<bLi45L;;$!`hk-X(;zM=#k*bc z<?;SxkzZq3KdWmsAL8_*&F!+8Q)Qn}l%T%I9~AkUi~KXZp8~YV8>y%sc!rNem#!wa zn$QlP<1#i>Ic{P^lKe9u1pjA}w3;#s6EQHNJ3K@6>mW`=n=crC@-Hm%FY<n?*Af9! zuVr!pTwesZp~%0n$lqV&551o<4azGV#r;7SLhGk`=u&7IK24z2{@zbuDhHoVbUjt> zBp{up7Q76Z^$Pv@dWHVQ-8EKku?T8^Mv^~gSEI#j4RZH%2y?b!4q>O%D(eoPom3{( z_=&ivMak+Vz0|sp<-H^mnr4;CE;7#Al}vgSp&l5eZZ;Q*$6O>HT;<S<tDln2w>Tso zAGgHEg$E44B|6pLP34VHj(Um&%aq`bPWJ3fv{AC0Tguy%C7gW(hQF)Un|}z}(rU@z zMe|s_`ik2OK_O4f&;}g>P=U1m=Dv!FJ)6Ls=+fh*P1#wzzVv12-F~23UvU(miE{|! zKWXdJwx9sPdVNwSR%E5^Po-F#2(`UxeNK(;KaYcW%EvIgaiC!++;bQak-kS^;M<V` z<C_Pl=q@<ObuSU!prPod-!OXG^GowVE%OZ54W4g1n6=l^zgq<pUDJ?w+&o(J6#R4o zqxGX3mfSyct&^#sPLfV3?pK8iGni*wz)1rf>z-h-ciTarAhXmuh}dI2;c4PW<4R60 zp}?fR)_4c_9ru{r;}myk@Xr?}#?b^IO*E_~fF!rraFp<!BxW)KZ8ff{IKxuRtC?Zn zcbi$gI-b~*+O^psaM~&ITwn}CW`(0I52Va`JnAzoxazz8N=hLrNUB0<)KeIzUQ<0< z6#;;#B7j8(Q0df+iV&F;u^v~2?$9^kB9y{6;Hnb6KHL%NS`&OlK;<a-z{B_BiY&@V z;mtlg#EtonB18>qxIsKYW}cNm1a9#umRY#J`#m=|P@>U3uFvA!YC_?)3Wrb|&4u$T zo)LtS_?8-FozC52#G2g0V@C!w&v8HvKmpT9XG2u|<ZmtV&+&dZUWdGjZ!U@tyg%I6 zIzJpP$ju4DkMgC_eDVJ7Vt;L|y2W!DA*^M;{$Jp7TbREGHh&b#Y?H6!@%KO6=jli1 z37@dhndmwb8MkPR|I_<Ccf98_(fxZ;7IKBTuv1?#Z0tEz&v%O9s`7f|&(FK|%$b)r zCvhwt&CeH2eWUe4cH&rW&I)HT)}@R166PFE7x70?B0K6}Vi9XzdD39Z`EvRQiJQsr zg#83G@eal2MkU~Qtg*t)gxb3YMe?$qA%7_xzpoLz>$}Km8rzae7?`Z3`J1k3q1Pc5 zd6r-XR9<3X40Zq;jel{G9=>gKYX*r1weE)ys}~CPSe$H=W-+jyz-Yvi*JA5(c>K}D zh~Lm*#Wk$=m(VNl%rk*U!$90Y5?8Y(M+r(|4Sl#adc**92}Mi979wuI?OQdi8n6{0 zG!{^eM^)(;4)pJH6q~eI#=q*Lk<%fUh$@42{7&JjxWXq8t}3aV3Rk@qUeCA2zYr>1 zoL=F9g?qk2xT}HV1F>AC1bsfi3`^iRv%-yrV(2Xv;aN4~U%X+1Ph@-mU^T6Bb>ly} z@W`=qKmHeZzT4#_IhaD#<eqsRyPH46IaF@F%hjx0|NmeH3Hqnj;Xzn#{TG2=OOjpK zfFEi$os`=}YL2@X9fiB9pmjBktkQEqmYa#x$eIJx*E*=Ogg(tK3a|CyYOuJAC_-Hf zA@a`0fp@EiafG;K1NFtvvcazz=2`p6l2GBYgh%RPw<hV1#&%h+Gy|UdkTkT%uf21u zdU%%TA$!}aJ5&V6lC@~9a5qlPftF6o#!d!!^^>k-HvX8IiXpsVP3^d@)luK>Ng}0F zUsoE9JM}Y7$2|gZPZEhKyC;d%*}}accgOPiBk+?+50PyDx;a!8bo6o~PzW{N7=RJg z-P~$zVq>lA8mk17K#59_Sv~>80eA4?Oz!Ed#Yt3V^&}_F=jQw%b9=oDuEq&6*#L*8 z>1?*o8)lo+*(yheC>rI#)H{274?cf--0GcO-a^QZBt(#%XY|0sfTLIuQG-mUP^dN{ zN}}?HIehkX_VPC(xL{qHti9A@>V@O0%yv)2LYmaJdA^%JvHoD<1&>161u#}M*RuB` z{{7+qF#dfuu_+l`WhVxB#k|G17wlt<`+dyYVcaLezs0z}KwkNFQ1q%<TGftFq5FQ3 zSN&T7bYBr5|6y2ofxjDtT{Af90X|#F9Kk)M3%$5gwHMA-!wnO{-MO_o)s_NNmcC7Q zpk&nU|77N;07lD9R_zsJtce}j|2Mp3ZRdQFmr_D?oVdrp`@$6vP2}pMfz%(=iekmq zfztA*11&Y~X+3B`%|eT|`~1g$ohHzdp~*?pepj$9txYg0U9<z{^lEPA5x@EaCY|EC zCGEF{BPqndErs@o^(E49an#o-u(+b^?H%v`zbT5W8o0KQK6*1XsiogYF5xQQA~D22 zq?+IQdbDo?FLZCEPOll6mocc5|6{d`zyF_MX)UfnwG7duW4Ss$iq%{B2&d8;^FP7o zgs?02f}*+cTRYWYW;S}LI_4wnjyQnR96(<I;E<02aCHnY(E-f5D#kFNEPfdS?CStd zEdadCc*#HO_p9FX?nkvDg9fSJf2#=jAL`GiYhBE@Jj74EyN?e)&3})!?=|_)`0&qp zSKmcCr%5NJGEYd~)Rb8hS%z^@BQ%n|7K&1y-5gmHzNNmVt<Bds4MM}Y4H)A>%x7-X zK|O+BFkibS$Wm*uMspGsm}Z0z@4<5RDGA+TqNWr5A>c7TN!L_m{whs*7Te7Shk9q1 z1BH%<GOR>|16las-jsr%Il<vxEbDq+C1so%&d#-V#z}tt0uG+b40Mj>AJBk$*P&ib z2pTr76m*M>q1aUiteJ&_!)J17R`)r!Q>uDoCxmxWbeS|KOOftsm&1)<UWbnDE-AVA zb8-0%_K1z)cT~?IP94;eqisrsmfRiRPPRW5WcR|}M-_u~v%_~rsZ#Pa1IM0G?O1if zce_eT>Q-6$MM|$KOwPPLnfh>B)EicnB_a_;s+X>JqNvKTfh3x|K-I?`CpXBI{l#3a zO3@Js5;O-nw>6`W3HmT<WU$G^-zQ8&RI8MjouN6!IxM_^Vpxld5+)EW6k2}TxapPK zeGf^;-Y+9i+F3lp?A0*glMudSPhSr`wdx{SGy=r-X<*EXH~rFMxJvRLbg(66e&#Pe zu^Fku=&-=46tjyh%b;$ceR7`j*RVj_hY-(eZ-L)&`^f|;n)YKAaq$D<b*~mZ(adTD z4yAhq*0h3y_dl|8XHdKQxo@c+Cf9xk0nOrecXrI3UDE!$@IXpG@arAdN$vL*uaRC& zbjg0*aa7Y!m?HlNf>BOTVF+7W%G?|G8($Afhn{b9_`^yO>24GFBCcMc1`!9-s{@{^ zk0#7>^%9H5-h-N6OH)(RMA!1&qjJRd9(NJMGHhHa&8kZP#+A0!=+#S%`S-AOk&dZS zV=s^D_y<F0CS#!P={BxcboABxKZ!1Sr<J60RfPr(HG08;vbRY+HieHQRS)_m^NHBW z4)clVSm57JXij(d{WS*+!8}U%{2Z8bw68Nmrz8)CKKX+Qjp51tY2IeF0ZU=+sXHQ7 zLY2{}6Rpq>k_~g0#69}z3ckX$(A|kIZV!$khf}QZ!+V+-AJsfMzk<|7_&=z>?ZUR& zzt`U$PE?-o!Ts&`l;Pj#Z_7Utwe$b2za5-#?bNK>T`R`E>MXL-rGpZ<7K(y!1g-2O z_KPcUq|A?Y*B{Ei`A4YfRukVt;O8>JFA~BoV~rLQN5Jd6sl+0Jd7AgiD{zpS20^Z$ zB<?_6m@q9J^+Sx6ju)eHR&?74(8Akz6hH#jPfH=Okx1($1cK2|=}JJR<#UBx;<TQ% zIQ-Q`eytp%aIG5kLp0Vhmtn_t!}1T~VbHLcKIAnmab!<=0JT9d{vjuR?IACUTg~hA zHUqVIEQHNY%KqbWmE>BJY^B~4Jw@Nya<k=!-LFb4b_-2{RS9Gsm0jy4mA%7blwa!{ z{p9a0^3N~wFZ6!uu}EIQMs$;ZMocHCxROo67pc{;6on?2P2@urkw$*0flG#yM*ci$ z<X<6DrU}B6${syOw@qGxZNw!e3-o#H3caFQe~&ZG{Dz|Nlj;?Idb8-To^XAg)so6y z(?bcgRny`FZ|hZ&n1!mf#+BS|N=b9)ndVy^V4;2A;@%turDRyqv>#U$9>moz{LUjo zLzA-CEW#h;(2B~XufnrPwGAlDey;J#`S+9<P6bDkNihH=u(I&Zk2=jfyZW2Lmv`3y zV%8L*SUDe^R7f*Vsn+5KXF%V9b%3@1M?RoY#MiL1Y0A76{`@1-*I_@G%AShGz8y?Y z@m@^l*{I|H775C1y`POPo_D4Ewy;k;+a+#i1C!D^YKrfv6^ium&+MjxG7<SGQAJeJ zPyVQJf^wsYsp5ABwYId?%qkMI>Y-z(f=w}!(A-oPPcdf^<mU;N{Nes&sRvADZ|5ZF zeN~9u_<5Is&*?e;I1&zA!Je#7=T{7|Ag_hQZaY@F{liADL>Gx!9u#P{Y1jhfp@G|C zzCLlq8k+7ET1)HI6Pa9`zVWM|Lf^QYSXkF(I4oIjl*5vh>v!lG$?$~=0Zp8u;M}Mm zGxXVg39q0s`8~B)TsXf>Kjn19_OO9IBGaOB3Mh4*Sc242gQ7N#RL!?qVzk0hGAUee z8tA5zz-4i})NMlmAl4`VjehG@hG>BaP|*!%LL@FHIB^CkAo~sRJhPoqyfRk2xP_FA z6YM+E<}LO5bf?9q0CzjCM9C`vL<w@OaV=pIuS89<)HsaBrm`|Mi8wZERmZv~_bm09 zIFq4t##t~6MTup;)Ui3(iH3#3l~<bYW^>WVnCldPAoiS-@u*r#7o^%mdk;ORVhx^E zz6D?Vui2>NnHqI)so^ltU*N#$?qlfgUs~<E`wBpI=<XU}UJfDEd}<vW?pE1}t}SG3 z4Y)zUw79i!?(Eu_LW#=H8-e=;kZGu9CfIKkjZ%4Ok~%i<N<@iz(351M*;Ll%y?|;@ zkW(~~e67iqN!v|tdZEMV=P)sp%0A7EY`VF5)-iV0(Txy*>FAEA8QNg-{kn!0vY{Le z%__I1jkOK4#oVwL;)AlzP(-u*Qpq*61er^Lb$LkDWc)&^xQ5wI4X0igepiFP(`nyX zSh>z`Lhcwh^_!IT40(GYS3S%(iF$-M(5GmUU#|2`axsBBG)X%_;)9%44LGQZ9C<Rz z0j)ShJBbn2Ov~;(_o+Md&#gDgxPxSU(D{!cUM|s$Kk*Im7dY2AJmS+~!w7jC5uQX- z)%9?qTQ5Dq#ll^8g%7OegZG$yyF`t}Jb={7u1t9SE&u@-@i|0%s}h}WUqX>fH4>3a zVH`u@QbFtSE6=}**;=CeM{J`+ED_xb;(k{#CEUi$w~K(P@@L!5%MNzDQ&fS1js`q4 zcgAx-5l~frPrjhT@bX-T06iR!PVB@j<H}>FJEEy8e~QW6G|0p;eRPfKFj)03p?7?F zCAEhyf09_^!r6%>cq2XHbee;-f4GmJ+^CPjm&K4@BQ(FkEe^UMChMJ}*me5a+c6!@ zey4#R5zh++O+fe26guwM^y>T=O5?_NiuX7&TiuTMdD2S_finuM%D-+oOfGtlCO?M? z^4?DL9%X*p4&Mvyd-yu&$Ty?>hSPV-mf`emJ75Re9#jNSmEVW2IOBo8yP5Z=rQpbo zFS~#`I^|ZYy6r8Wu=aE?mphnWo^LQRSUiOJE5Bs_m|hO<L<jdkYYbN=xbqBGhv@F$ zKI`DlFW`Gm{rdL(C<@xG73QwV?+bn%T|5IfKihv?t&g;Qtjhn~yVrSlsdJ-ImcQ9! zyw$s%-reHir&BWJ@UH*9!N<?=;kpFM<d5ilk+TT5r*@eUrl8At4!bF-J&*=f!5}w{ z#mQbZ#ucw4Af2639b~7DPS^Ea^3Bwk^<y7K_Ch}Vm}r2tBkKpmNKBD+xDwJTER8re z_F<GZhzwj7q)5|xuxRRBl?>`OVZHVprSDuldPiF|*T1<h#b=bt^o74IVa<5G$(7lQ zXJFr)xMcqzvjNGneU`27j#^t*TarI_C~i-p=6nsn{U_bGKb2@oG;B}ht|$#BTd9wz z(VZDNjux6UI0X81=8&2cbz!}T8c2^0p|qGnghb1M9|YNpx%zN>X?v9pZ?k8V?;<nK zQct^(Q9gBOJGXE=Hn3i8@uIZ>2Uu|n>*?I@dze=8XTHCk2B|NResV}hN+0kiSFe0! zkxfCDp;^d0A$DtQl+DaQdsJ27G4KsDno)zZigKz-7s7({46a5ZY*3Amlj<^qWFNMZ z9n<e=%Qa`a)1yxJuk)3XH&iIe|B^R!h7{34A=&cfB$4zyn@|0CiKfD4+3dGNE%~C- z)pF&u@?yH&HT;56xBM)hoeEN~Wwquza;%cSUVid11ljV#fmLv`1EKO>_{o;{a~G96 z2g;Tp(cxCupw6UGkn%r~&MK-h5}8yPsg(A9__QpPNM=|qLfXoQDl|v2{KAYHH`idc zNG}Cj-<q78T{@Xfn@SpUa)wGWE*dolou!nyvR9Q#H!_TF96J<^P(f7(<Y5Gt%0>ss z5>p=I@E|kbA4E*u2w;=hFO9TqHP{;EHFd6oTy~3TxchtChwRjOAi<ncb|^D`;SJlj z)9kIRw#U_=e>AInrjNG&2C-yv6@azrWbUQlr>@eB@Re3Sel}4lN}Vh+LIABl`FvR^ z`Rw2`Ji!{weQ1$~`7=2Ene^@2+PvFjSvn6I9Vcv?Fq0Nd+&+vX7!zjvi8at1z>4rm z@+rZtiP*asDmtIlT)~OTMAz{Iab-$rFlIpadsU|oXT&AC`?hPa&8(^Sb>t2%DJKFy z0r%FzLX4ek3b#Wp7%jDVZ<)%m`&;I)idL~YIhWkI0>yaTlYV4%9_emDc3rBh#(G9# zZ_cZX@x9TQ!4~1pmY)Ge>%YT#;2&rWEu3prvI%~mMSZ!RKC`sU_v?W>tt9YLSBmgs z?~wG$)ozsYWcj3-dU49kbD3X7=xq?pi2FMzf!vg&GRRFo<14(3ft=UP_L1DwT#-9w zC=(b(yWM+z<i+MrS<TS#N$7tv6##8nP1`49`i~U-yE}}~f*v})Dt{qg%;(J`4d2Lt zzu9~d9{dj@asC2rdorhe+q>WQ?j7De#Jlyl!}ec-%mbdqxl{8x*t*^Z01BXV&FFt~ zt7VlNVYbzB*1Ow>bcc_&1&%GJcDab1NR`1ypPHWmQQHeMoSflZK5S_@X5)8VU`MlU z*TqiWC&QDBJMPzkIeFra5n7Yvm;<qHpGpYocC#AL7>hK0_gXk=J@SFkY=FYXQ#Q)O zPr5Rv8)FFfTaXZy&pI0ityc-@q$gWKlsd+sTtxVK3twE*r*D#mxcbSjyduBg%uibD zEyQd1Sr~rnF?=XpRZ)0#k$<%JQ~p|cZK8h`6*}eARcYK{Xu7@`Q5rZNRC9$yGL*NO zI4xRa?E3m2gv2y!L+EUE+9$FA)HcQL-ujaFibnz`)f$PV6uUU7zn531au+-0n$M6c z6HV<l%c<6UsA<l8%JWRcb}`o4e#FDHY7JS7O1B;`?g<pXQ{Z*IctrIM!lgRvHP>kZ zgklNbO1II%uXPr6P;iiM{ldfgq*}Xr5q_!Ggik#S-fOxwz|gH-9qHDDjlSsG7!w?! zE}`Rx%ih|~WaAI9WyKC?`h`NalX^|>Fl`Vb)N5x#<ZG7#*8Z{l{q%Ypp9obvpA|MD zeT&NVyrx~t&XBdr|EOQ{i7Q;<f__aI!iV~Bi}Y((a&R)pYJ~YX^)As$KlxQq^Oswl zjf*bcszk1_)l)vrs9Jx!RVdMQG`i(j(06Bcrw}B$Nb}F|&6=6o7F|%&K@QlDP=BU3 zr{2_pd;`J3m8;a&rhga|Smh7rFyR<nAKyI<^F?V5(nRGcJwc{np}zb1#=l5@4LD9C zuE$Z>5`~qB9P1kTEDRlh2AuR4%Opvu-U6^l{E9SxreVDjYgBSbvJYfTqFxJ-Y1k-# zmBlsm%U|vMq5K+|zz?ckwfK7)whbG|9PXJTnayxRTPq;6)p_W1c(ly+o<Co)6Ky|7 zkksyUSp-{{;WA2U*IwB}Vci>SK|LKY0*E=mveyUQPuG8QMi3RkV9<%!+I1(5gfx@r zT7X9a%%{}KjEH)bJa=hpSpJ5Zz5F#gLP6e~FRqVhrn0-~&u!8=d9&B%gz{s52M~np z;nN7ottH%Cl4QFw)a9I(MJYS=hlLyHkf=0ENnuTqyR<%%AT_6(l-Xjm^h}GudX2&* z3e1-GP&y@O)hFdrRDeIcNpoJdyj4ZvrAXCCB_b+Gkpv3UT*xQQ<dVSmTIBB)Dq!a; zl3#cTW#q)W`ka&S4&D+=cp6H;_6d+*xa3V`JL)I5iKwkfq`V;KtPHKtfOu}hjR9;7 z=o=YRxhXk6db;0jWn!%|@hMR`nwp$OO@%}Kn$OMTpwB)&!>`&1bsw?lS5+kXNy9S~ z-aw18MJU47r*fbd7qIGr$P@HP%ZAmSzxKdrs#-#w#NK#BZEHZ|4ZW_Dl&Une-LNK@ z4v%C78H)B%v!hrov((x08+B!vZ!%>bt{zlhIMZ+F>WbK7^M~46lm33u9Ft2q3IQ9{ zF7gzzG)eq7@R5iEH98Kom>|zdK$GZtR^Qo6Tb<}Tja%G?+J8lWai?E*EboAbCG9^6 zpQSchAO36ozGt~N(hK0%iLPsH%oTX{Kh^JNHib$0l5|LCpx$vhq{~4=kJ<Da74gu8 za=3`9Q>JK*c$4AouM1$}d5X!6S|{ef%I24Z=4-%$J`b8f;HL|tJw4`yfQyl^QDDHY zFsa}Yk$|N|eUGhm_u}a2hQ!scmk|n^t`E0bE*Liixt<=VA<>KJF3n=F6e}4;z)@H+ zdp_-%DBaZ?VbX^q#!28U)-i+9w_;A-(^xIWs8PgzGWZ+m)sAx~D!-xH*6KSX9?slY zPrw8PlxR3+ct<AF*sO2km}I7K{Rp3-@T9>tw#e_ea^r0It6ew@G2+UNb7HRC-60HM zQwsg$SBl6#+WR93i|30IU2hSHs)MJEZ2m?{#4t9f4$6&iSFV&pz3Z1HD!-%I3xMXH zo@3@GDv$4B?Y%I_9J3I25$+<~CBeF*)?0>|W0vr}p6@>U&bIbJi=&nW>*^iAx_TFp zIc7b88-mP*eIVI@e<Ln4Qb4JW>Boi99uwjY;xY>b+j@>+2CDBjyzA;i?+(gcTfdF& zuqx60IqMspDzmjVVu>D23({ly^$uQhaF?G?xZpEX-&z^idIb1S1P4$Yy#ZVnlx{$1 zTI(7FZNk7&(f)(%5HybU>Q7|%a`~+`j@wdhWrKylbnITZt>+l-iWVhh`f?dlheq#H ziON%L^xjT@jozgl_jITZ*!k7KKd|%bousP{au*IwqSp#j`yDCFnPk^;yi-J<LHiZu zrj=Ct^@8y^8mO7T^f*10NPy^UWkwD5QuzJXw$rLu8pA>vQ{toQHDZtx#FbSfGcnVo z7SHqA#9=8}5}C)40%GLfGgYtdi>Syo4`rqq59_`cb#2UvOdMDZe8lMdd-C5e0;<Yq zO~bnbo(I4_9M82o<M~w)P*whiR-h8oL+_2cw@43dBTMx-8W`EBfQxl)9be%s5G8+@ zrB(e))DBvSPJ)Y!%-^K7sg0N7v$Tos>Bc?9>U497arAdd997i6bN|^0%E^ZYTqWM# zT|-RNzh=S*5uz%j(|r3C*yfe1wM^xL=%O`xZ?Nnc_ha{k$I;&tY9yg<8A}+8RN>{B zVBHT-l|u6G0;v9wVj*r;QZfM5(KDVqL7N>1SLL7KvuMB7*sd`Am&#hCZ=c}S>#_d5 z)qOY0>L_!<$UGVyipCb%DYR^?e+*B_BYz}87+&x)mCTKQ<YWr$6y`@V2=cpWOI(i~ zL;3pr$H_I7S)JcqyT~Xtzu?_(nY$`K7FY2bwDpC0;B+5<mUriR_v7ZS%0KVnuJYe^ zcy|!j=|lbbVAQCb4K|a_D_^M?p)z72AtAGaZWH7%+6#aDs%n6^`8{rN39{u^{GwRg z)ZV{j1P!53e#X`yX`Q$!YXFk`jU?jRYHI!9dKJa$P0I{QaQWOOKdtQ=o}})9{OGfj z@FK~kR|WIh&4u!7Fw4!Bp9-izmN=05s4PT%NG0&1j6}@vKQ<foEebD4pX-TIvk8<5 zrJqmBKuIF$bA$X;W}_ncg}>$Y5>3$)!bhQyELJy7Vn6STw7g#YM7b;;-l#FDl_{LJ zrh|YWQ--0W=}RTOF?VAi`LcLd6TL?^PqydxTKIq(0RE*32Dzx{p3X7}2k||fkE#7S z?Viq$sHyCp&L3N&`g=Ox(nowx=NP5V&8g>J&!=9B;NLJL64TctdPWO^vC&G-{+`YY ztZXB(XQ1Q~o@Ed_>LWh~5E3A2Pj+^rK_*B#)gnsLdgKlIIfArPk{)sk;2@Yo%Z&E^ z*6<6~;8L*u6WypJ6B7`}%&Y0cZ|b{x>tf7X9L^)mWYEQSI!JQpLF90+a*#;(4O;9j z2CTHBCphisnFQ(nEO%>`o4=d^fg#_sfP6NP9(g@pAs-F$uSLjPc1C_WLB>xf@jzsw z-d5ck_8~e|v!fgSLPTe0BwK#V&p?<Rf3rCls|tUo^r~x3pE0O!^hHLgal`ehPU?4R zRT~B-^#4$(v*ja&ut?~)o`1TvAnJA#Ae`tQ4I!sgH9Xn!@BP$@L051?I$ZO-u+q`h z3nTl^D_1+dQGDA~_5{<HHaUIikF3%XC@0JOX|pt@%*mszO`f66c9==dGk8DWROL_S zGt$Ej!~2VWh4=eA<Gpw|-m?mL)&3huU~PWS`!;Wy{FUgwjj!+=$eGt|R7Lab3^j_z z^ve#um2c+RzX|>b!9QGpk8k_Rf1KXO{1VR}XYlPNia^B8jlbY{(7Fo_-xiiv*b;RB zpLPJv1%Sr{a5j6P4qz7ta99CgsQ~(73@<!qS$yiGIExzu@P`;+wF7wdD>1-@0@xA* z{LBITxd8AL0X)oZnJdFp4q(-ZF@{<J{7~EQOl<dgR|)j}NAKQE4*I^!fB)XQwLbhf z@BX(B|0Qnz&;Fy^yK}wUj$5Q0$If8VN$AIm_y4#O<H}cpG72dJNsM*>g?>DJ#x@fa z&Hh}SANbi}iWdWTdpd_286IJ`w9TgjI5{WHAZZpae;Z9uK_jjs2-vW1W-1R;FKR4F z0~DV^cwi5e2OWW!PUl*uw!p97uu}-^HoZu}3#Sk)h5F)im99$It(ZaeIo!M6eI9*G z1Flu45XMq&okEzPQwTHfQKwDk4*J9ioI)5E%P$`Kh%o$+s;P4bGs)A=5wsYBHD(tx zq5ZLR&Y1|Z-`lFq+?MeCe$}je#*d1{geuY6iwQQXz*A~RL}EhHTB7_9B+<-T!KZ$K zX{B1=7EUXrYNkusp!;<AC1XY9XOlw2ZZRpIsow++qyvg$(n7yzNnS=F_CJA_gfM6% zhQs<|P~QHVT?rK=IVId`EmXdNBv_<YsDdgRg)Wj1be_sGP?A*c=oTeU5~zB}VqL%R zT1LfW8hfU=XA1`~!jFlH2m-xCGiA;YnD}GRC}DEFl=s9M<0q7>cDvkGF(38PCm<}S z%EA{YtA#?q&m<&Vx?cE;XQciB-R;sOf<aT1Cg=)5D!1yGpOxpiaLt>F{oS?3opuGA zEud;;gUz;KyfyPl5i7g<d+-6oq(Q?ceS>9CP&2TsE*M}Z6)D?A4^)Dy-v=3BSNJ$p zO-S{VUun(Ho_H00spzWZ5;r=E!(UtEpHSqlFY*W8PX(IgHIuEg5_q64W5u?j4%r@A zYF`35(24(!y_?ul+jgHw-(l&q<*ziWJ95<uMbKQUpY}pp@T+4il$$MY60OH9_I8E7 znNewxs@JEIe}3ervWpA@iX6GbEsNst_Z9iq7x_07`8RrhEYy;2TTy%k-B&-AvIM`@ z{@QQ4@`nUU9f>LTT6(41a9V5^v#=60Q*Q1nO>{j&r0zrzB7ctxZ#?R56>@6z*rC^@ z+Oj*=Hk&#Mq0{OKJ*8Q>&mokLGg^YIzexqFRTp-vMvEGzUK>f(1j>fH*4%k!?=Bne zX8VTdxI)-t->7W$O<@+|>KCs4dEsgm_N53?%S&lbszpi_-><S*X|oI%ShsL4;X6F> zDZj%c&FvWDdNhX6#5+vJ?}v1|%3?b>E0TR3vVP_VfVEfUkE02gpS%h73s2jCmHqa) z-hTU4*>CGYSnRjGMC9*I`%@f%frFWU|H372U(u^sM0Au~(~4WPzjVOg#GOt0b8#Yx zNTi?qQL7XiaHqknO)_g-s_<#FyR(t7Oc!t*+$EOaoOX9ALDBwFb@)+8Bm3>Q$-zkp zGoc)^&TCW2Qj6kM;fFUNC2YnP#N_3;YX8H%ZocBbp-#r=nD|-b7ihPsSdr2G(7-Qa zemC(STvrv=P%<qHJ{sF`UvK0Y^#-x<x25Jx{EHmZ_j14W1897;YOf0SRxPp<FHvl6 z)CLB%xb6}Qg>B|nOPAp5<3;{XBDM34-kqdQQJc79v9s#Ux=VL;TTBW4LJYcAKoj_o z(RT$d3n^y>Zo;&*q#YK!)YP`>X8lk(vgGb@yje)Zj<i!Ou@)qLE0?akjQ|0N^T@!p z3Ri5p8aFb>Hm+ddjw`8O)!X+{+sjeBN+&$aa8a$gD`R9D-s}RH7R`e_w>(NRS}sZY zaAmJO+lSZs@bwm6;FBVR;5S(KE#BU{%NND1$m@F;X@UcPfGZO98~g&J0VHWwG-wuC zFnc`u2He%$sSk8v#C#uKZSXV(nqJStfo}1ktZ4j-jX;%H2IL!YF(7vtyPRakwM5a3 zv1PdW$=?_GCBHo)$-x_tFLvNsXENOWaibq@GR)awxE)52?`Mm>{rKXVh2T$g{ThTc zw?E<pEi_lBbKeL|>z^cVsrrF%CjNd4Kz#x|F8$nX0h9oT-+E&O{v!2%`I33SIvex< zue~pUkFrSm?m0RHaz~ILD5C@+iZKZZBra+~GB5)PMuK?nBqS4v<}#Tect!(A43Vgy zxax{4A}X%9D&87Yj`i4$uA;8GM%QDqc;Kpt7xVqAkC|sCGaQQV_kO?6=SQe{>gl7p zy1Kf$`pN+vq+3$REbc)Xw8;Du2ktl)OFG2*@(fC<qs1ZDmt$gp43rVDNDzkBe#`Hn zXF!ai09Xl5wzOJaZeM^5K&6~Rcoe_4EN&%~7*-xbq_o=`g?JH?R+!rtINITpR=JR5 zY@<)K!dyg^i_CLDq4yYS#aBSu5n!o^FC|{snUI39R1B)I8lh%|`@sR6@(|d>=g-7C z^OM0Kp(wY&t_rES%|Ow5aB;2;u?}n6;MGi6e4cO{&w_C$>@Ii<a&bK<*?oy}hST}F ze};22iBThRq{rLyhFrqtU%n<BG=I5{_#vg1JW}OEAfmk~W1;zdLbeoD8f~^MheWIq zZwW~@M+a}aUPfO)(K`5G^-7s~F9%`7W_mI9eDYi|;aUNYG~*n^FJheBnJpMM4>;{w z5uZQv90!q)LN0=Fb-W+rYJoENZ=v_7rbxkc5yY)%v-;><B&~l3&}vhf*eH>+1fS97 z)toipis(50fk?7yFuS3qC<%vS!>RU>Ucw?eKh=rapjnQ;;$ACU_lSbD+(9@o(Xplg z?)Fv$XDv5*^H2@6h*rGQfNLyT;}|R|rzQV%#}qm~?_dP#OU8=|r`ruxx&xjy_yfHR z?CrZ$_!7YSCq-`{7`Q_E9ZEqK{0>XV@9_L~guNu-cPNnkUqL&(T{x?KF%BC>|40WL zOHV$UUIi3HpyYCmJv<5`m@YkqP9YvF#%ev#>gThVTs;LTkk@hyBocCbh6!SP1~vqr zgP8;Ddmx!DrvhQry36|)9665u9x}866@@PLZUM0Tjx=9eU@ajH4iReRwE*aCLnphP zol^pn-Q6TK!tZqll2;k>1f+~B6!K}<Nus|Ep}cdzAtXCpW?m$NKuhGq0eh+lCKBr) zN7xFh{LMR;3);kIqRac_EB<-#S{#_1_o?S-rGGD#!re?1!L`t~nKm{fK+A`;5gMLN zwL$y)e~^B8UbrCI`>jtjnk@;7Hws;#tv}HsZbqg1Rf(*$E@P;azAKFL5y9zg|Bj50 zp!6>8pWX&zsFc3j@#^;v1gCcrtf!FlhWAe|#TY83Zy;X%>A{Ms!RmMK3e*nSV~-#+ zIrnZRK6MPK#8;|UOy-?<GJeZlbyyvj=<PyX0Q+Spu$@DuL1&Ph9rG$$LL-t9^&<9f zOvjXuYG>*kv0||;r--Ub+5b>Xi{At7{%!+(M&FCF=m-~RJD69(0mcc72O<eSHl^=- zkp~O-_bSj}XueZ8{Ll_#`~HCD9{7$SdXPWx3h^G&jN*2}+x3u86N^Z_p!mxFlCb#N z&+zSt&#7qngePH3SlkR9(04Wd)B5j8wfPc?cO}eHC@LBkKyO@yH}5jy)}{ER_!C^e za-WbvUOagrvUdrh`XwNxuTG>S_zKlXO6&<#QeOc`^_P6d7C96+72D}C6nO9?luFKz zjf5WQn~8NE0sSt69xa3ZQ3fR^+G|AjSS-&_bjxsi{w9(fbD~I+!mX!pLy%-pxa~6B z&_K8~P#=9WX`KZo*~%4Mt>D`VHYoVIf?p^&4s=kwiGY1~E9CKjeFf@EnSyQwZ&GlB zf(uoop$h(hnL_f4ex|;Equ?Ftd$cH#;PX&NhuP6_QMAKyQQ|ri=2ts_DKh(fy>cA3 z_wO{BKOPI`8~m4gwS8~)WVU^8X~FgDCqJD6QSn|y>~DKMI<x!ibDz_4g(f&4vm1Mc zTM-1!CSBg<HXUIxh_}N&1M)OzPAu5LWf3~MKRh4ULA69_1gg7hzv%gvs!u1l4|<+J z*!`pD$g9K916A&9|FXCPWm$lAR@fh4d9)klE!fp9cs*c7M-lUN3@XN4s;`&klMXaI zuR*@@<_PMoU>05OaCBmar0CXOdKpFS;3u|&w$6*;m!UA0v-?(n&9Z)Z+TP+XlktCo zEn|r9J&kI#{k^B>p(tnf-GxA^UT}>CHZ8S(g=;L}%bJ>q@9ddv_|MyfKwu<pbi^_C zWw=_nZ3EE2l#ySDPR=b9p~XC3VG3}S<k~sE6#q4sJK9ere;_S3D}XA0uoZC~9Z!&n zg@`0&cx^+c3~)UebO}kPQo0?h#qTpS5|S$N6Ow956Ot|{$Hlcs=Ydvya<o4t(rCXz z#BIM;#%-r)j;&V^iS!iXu=_=;@q^6N!RYkq>aK6Ub#$s|*6kf_kJ9J%5)|n&{tb>g z$JK7@fTE}U3i=4A(31oX<u@SD;`X*yye++gaG<@Lz*gb7LN`J`b>{d<#L<JFL?Atm z_A9ypFpLq611(F|=k*lL(fNP?x{%k>Ti?mB&Y)+YZxaLNwpi!<1-6i2T}3>qHPDzS zy~y?Xrx*V}s?X0B3*JYn&*~-rJL~f#w7KB=)JMA)cg%&RA1PK)k^nl}f7)>(jVEdS zanO3q;TY{~MDa1&@i_gxH;oEBj=(xh&wRB8vl}a~#pvc?y#{lmUW0j`_xCSgXtS2& z%4jFFJnFXqutsQ?XSzQZvQJyEW)l`f$l9QmE~&S1_5#`@ty^z@Pp<>&dt;kl9|f$2 zhlD++!}$9h|9T<4^Aa5y?m)%m8wpokENpRffK#C~e{eh&Sa99r*tA=&zs%cmAx>kV zYheu=h4Nu?UXCjQzj1E*$XeXK+iCj7k=BD3^kr6T*~I8625Dn3MH|7Qy|Y2ExWot7 z641?pxM89=@B4&B|DvYc-a}i&)OkWsX;_cgxR3RS`AeJBg2cwvSVy?p(b2dLa2=pw zL1NPeyl)FENL&zUL1NQ31cZXPs2hp&;P;z=ZvyrL_5xza)8qx*35W%Wjk`g>URNYK z^R^}|&Y;0!-sd<!>oJs5_rn-(&zHmP%djTT=Q#{c@0>OS@JxRk5q!8Ne$(k<5#e+! z+$QCXG&~nNJ1!H82r<0=y_7#-#Nc{kLF?142*z60d*9s(0lX7(UeIy6Tt;}ISVj1- zvtuZEh{TSptydzj=iESUdJ*AJm7xfU6>6Wiw4<g-WU+AYTW|Rm8r$KZg!WM-rjJWZ zpBA?_<3lCs=TP}T{;^^i<At$1ZU;TMv$VMVCwu0HRPT$UVaxBh3%geD8;_oI)$=6I z(ta<P<!t{EBHGbF&6f_wvZ?VSor^{5O6VXtc_!AoEq%_85fHbxJvP-m`kvgZ`h5gK z?3S%+9~p@++JxbLmoV+&E-}72p4tPpj5*T0q7|S83=1h?K<79*PJ!7o=^Ns|qn-Bi z{>_<&GmOvzj-=FRfbKmVM3taKPLzl?Bhyq#-pdJ#FoH)<ws3cGCps+b^h`r@+2h#s zxuqoU`I1h|L=+@kzo|HHGeEaJdXs7|ZN9{WC9hFbtewGPGWc8^?O!@x!J6s!t!O_* z9Gzw{$6lZ$t5q08d7E0`!@}l?-aU@a*v*1g)62!C_vs2AdDF~RJ7GbusTfOEM?qNC zc#5v(^Fb5(`R)W)`${^F$7q|x;tQ`6)mPF!1|m^xNlh(jzc9L_eQtl5Q0bvwkVDH| zJD!yF+P=ft{xamE4{O3O-osqFs_$HN%O^^J8!)#-JK8VX?LbLB>N|_N`OXtB$b{^i z^Q^H4z<3fzEz+VrqyAryf|33|!LR?zg-`ALTyi=H{f{HFNX!fTnpx=oe25x+81Hy$ zpzG=MI#T}m<l2Hp$AKJR%ww$=<Mg@*y;Wesj`g&DMXZArUZs}qV7*`HPhIH!KZ&o% z!M`B1Wfk@dhSB$*ppeS`4Cs5wKdqLvzArcpB^9d{$w2LJ!4P`eGY=}})yu#Ic^{p; z8)g&=-!cO2rnvpz-o?m)CN@tBeB1B)1^?}xDr2`ii}5~${8Hnr+Kb)Tx3SfpLed48 zNRY-Rl5n!sAavw&^OubJ#6&+nMd?jR=ewIEoi`Zi4B|QNDGIbs+$TdDeLq1<r0qg~ z2PE;3Dh3R~ST14F6`%;?xrD_x;d61v7~#6>#BXPZ+G9uiJgGke8&NnY+Qa+a!FOSg z5?wpIc0m{RXnzR2q)puA(<`Sa-|`nnO;K+1BT}d#<RebH97Y_Al+cKYE6-ugyWkt3 z;F}mS+7pXA>QWrIkC<BESkUzCC7{XKX~#tLaiCJ(b9QkGi6{5&c!r$b=hJwj#OCN& zOdX1&^Wjzj#O4uvd1^IYaa|sbC{DvrqAUYH&dtXzH5#J0F{r4dVM<+jLelx=_|XEH z6_z{_BzqlZdn*7vMq|ER81ul8bxu!0QgdTM(i5xk(>v!X_&gRril6ObXpyjSF&S!0 z#TWwVcH_m_4)xyNMn6erXnajph+#mCvERar@l}nr!N<^X_W{V)4)3Ml7O@qaqI<C9 zeN3V9MfK%`TVM3e@Q<~EJi|QlTTema724T}Ni%$QVnTr!MGyV%l0TJib^vm9cANtr z9vW)_cG4V8MFJV8YK<Xo6m2=$xE9_s!~apJ6XO|PpFF({8F;^VI^YM8e8c)}p`Xg4 zcl=rC*@8AZ4sdxJ{;<G-CJ(@~0{x~0=pGaYT0*9HE2RjXzZp_0<LmQnVGIV&w_PP# z!PHXRUhJTe7@g;L>O~+4_oP}$Li6c<=(ukeVrr@=wG=|57WzQ8j>KP0B4NsjNeH$f zoho=1oy}f_!!&VFe?EPlCqAQtg)lBkZKbf_EL93U^xMy61NRBPT7-~c)~VlE{vo7P z>?=G}OxN*eKp!T}#iA8rC_TohA<DNKIC68+?;iS&YNO>La}XB|jT+?{aE$Bi$ZrBm ziZ-wv`8<wr?d!#S6KxZ25!;b3wK+S{%4iWCVju&t75Ur@hq{jkrN{@9>7gm@UO<ZM zE*DAU)AaOCg}bzraQ6{2h)9}?b4ooxYeE=aDTPD<v35kjHFCBQxrj*=74;19ZM}br zjOMlI@!rrz8<Q6kFN<+52Am(LC^1X(Ev<ZMML6V4Q)JknyqNmqrHfK8!K-)NW3*C4 zEC%`3>l30@L`%N0pd>JahufMC2f|vBCj~uZHeUoD@3{~uIiMqOWpKixI;2;O^SN-F zF#7VABc&Zd`uPX%olj7^_IdIH0mOYgIBT)x9cRa(l>AKE5`nXaT3*x$Xju_^u8b}3 z&xJ-p+b1sSxqd_2e;38kXh+~K9%39lBs_kG3jb9slBQxwx$x2Yk({w87D|w?2!}5P zW}v3Z1tf%n9YfPfJzy8D9*IWk(!YBIW#5^zVH~kVtR8vkNg`;-%K1cY&MpG*gKR-e z*{|xN;@Xqha=*(AF&aqFAB_F)58iVh7ko@ugsq~WTQl2(drxN5!sdeZ8DhNAvtzAH zzo26(DV&0K3~6Ydt%$}%>bP+l3@0r51G+4!$uRJlDrantFoa6!!$Gb7@qxI<d=q9! zlszQW*HeahLYf-HeI#H6ttR;hWMa)#DUQ*@Q6$hThv08|B3=uj;+r9(7_N3q&&8a7 zVSdrl=|EzKWyi&1Y3W@z)mTSiIc9vZg^(6hQ3J;t&X6vK9e)O0($Nc4aYva0N#5Lw z-?YYoT3#`BXo5k|P4bjJu|k3xTXDzvm~3B>nu6cx4-;vTgjYmdBpYAS@Z$FASlCcZ zH^3#HliHY&Hj6H4oh~w+hlI|~$6GEfd|8T!32E3s0K^%DpjKYA6gm}%azaqALt7xr zfdD~Rd?FCVH2W~Ukr32sk!TT)Uu~bh0OinoepiU(i<Wk@A_UO`x-Y@huDBRzx`X1< z7g8#eEn*{A92F>j<5&aw)<N5i){8tSTq|&NAJ7KCVp8eUBF2If^w94N!|z=68>%^< ze$w|u!*`N<db~qX+jIts!y6~sA8C*)o?nH}+C0YbCR*P6?SHx)b25hgPmOpDSdfrk zd5rSuv3(WjC1pF?_^@baDWj?MXlI+fi_nhI&fLvZ=A1tgKsWh&G#7l`F$l)oj$`P* z!_oX4Fc?!eYLd@-l)}8{Z4zO>@D9QkN2g??_vHsES!rUBJ-6LE9G`b!1NL(}y6HcW zGOYOLb{rK!X`;Fky`#aG9m^>tQZ&*sd<BINin^~^G_Y3U<-xX>&u&7Z{q5x;D214= z{l)h34=4rN^nuvRTQ2}jk?iFs9{OKuFK1nHknH7wjU=3MZVJNsi|pkRaO;=a%Xk0f zK<(u>&?<uL<y{RV)W6tXeui3G&^Y5~*vl88IB3t(Uaq4Qp$Jw&-u?FSPTFs=N1eAO zrjnrT=SCNIoQ#dxC26~h({_{NDPhq_bd7K>VZ^X`H=3v;Z+F6ye?Y4#zbaO@(rVTA zY}_9~nUZS`h7&NGr*&G#lEW1i43uGUCu~^Ef^J26j7@8Yhp0FY784fEA^o!5q$1$O zEl&I1*$ywR*W%2BKD}JHpB#BV<tJRfwIKStghj$X*7iNzPq%`Kl1_(mKVfpu@ydT_ z`*atXn$`huJ;e?{m%=e@!-QO3Ps=A1rtJgxpC-LdL^wOGCp%1EIP+eirDZVp2??<r zI-|38b0T)CdoH3~>S=E}+PrZ6Y;|n<!h#)qCK!d-zl+=*?cWRESGUSp*#l4ogDr4; zSwO<9K=yFp?w0N^dF!yFGy9NY(|>3V5!pvMw!+}2G|eTQwLgKsKa}Ks;yFzCi-i9R zBN^>?1a0ttK^MdMh4bwnpb8!*cb*mF2WeqA_@*S%iC!Qg3Ia9O&<!a%$%#oR-L^ye z?H$4dLM6lJxi#Uc!^m!dpJ{Z#Q_&y{XH8+iL%&$xczg=<PtSlAu(`+mB)<JSu&$x~ zKhpli>ORF=;YWKG%)AJGwCCVQyA~;72}0iwfZE@u&I2Sj((C>m^E=69@-?Q)zM;7T z2i!^!)`__GzBuih;<TM=uHacwH$Px}M>0#M0z9yg*vAM%<crfAu-qxyJ7oiLjrg+> zCo5RMJ{tIl5zM8DSZRsz&e;DJBqZA5f@ncf_OZagFL5bk+n#~nFxd1l3!Qcc33&=U z0Vm^lE4AQq4Q1L97YB@^BNLOEFj`R2@H06(PAjKnmNcTolxyI|Ew$n@mBwQb33=OA z!eTm!!~x^NMynyIREcyPT~X>-@&`J$1{PZy7g?^P?8PF>A`FBAD=aB<x(YAXu(Wa@ z?gyLa{RatB2W|D=>1cnwB=2jiy+++ap8wxJpTU{;ygZ+Q+#w4{lXXl=^-EWx5TS5K zGPHH$s+z#D4CA)?&p7CHwEsuU?ODMi^2dlXEIJw;>Ku!$`Q6kYz`a$JA7J@XhdkT@ zXIWsA^=g>bbjHOW1Uc$Vi-e>(OL64HI(**%0h|0NUN+!m8>W4_@p~J7_rMvqjnc+D z<lx|(U2zieByc_jHLPC8aZ4$euxC|-Fs61h!y)(1ynnS!?t21}q@NpNCFhysc@@Jk zV19WXz8dD4)M*vPxRaeSgKOV&8T_|2(exQ~SKS$;gB}7#Of)@Q5IC>Ga~}*!|9KVi zoCGoQLue9WI<Ep5o+C0m&p*|KLlt0Qwk)SWDpL^_lTGioEC$`=g@<DO*8A1H)E=yA zOD`!_i%_gnwPJ-ne%@hFta`j@IJaTP^<8wq(xxv?OjwkMF`_iyzq=bLcjg?GLUp*6 z7D3&G!oYZMoC{1x&MN>khP?gHdvITxfBsbH<!oB2x7C^oW|0+*do_XBaY#TP3q?md znKsNs)1(Vs3544|HuXYzpagy22aPYVKCitRLzst<8Fg(Aq$k(+9VHkrqzzx8$ch9w z@mvCJbiNK+2L8vv0N%T)JQ$rNA}vneJBR$9)L#4FyeHkGKJ|TzT&16xG#P3&o^)6S zbku^rxs+!jX!ibVonXz#p%h)PW<BTvYdS^tIa>gTHA}!S8DqVSL8nZp7-y{)F-FN4 zi&TtR#A1w62GD;KcYFt0uXQ8GcPKtaKURWmlTw)ngiT8IevLwTM~b9yF_Sk3%-DgG z=05T6pn&S?pe^1w1l#cp()2z^VUf_WDx<e4qdVz8tjKweVm=^ZLVtU|rC5*q?J;rw z?RLlG`p9xQMG)HG9WVO3&x*yYpjDh7Llt&2Y5Ji4?o7f^e}}b+AeH*N*;Ga7?{K7` zzq^}2ecolLL-c^1_6+Lori|4+SxXgKF}gnL@~ErRXVsreX?Mg;z-Qg9p?i_t+H7=i zEvV7ZZY^KTM!qXz2D-K8eRON(kOyRMFWq8SN7J85DTo7${C!%vqy5rSXUA+upii^U z^PoeUU+Ogr`6XT86&>2Aa2ipPay=aNYovlPf6?6tiN>_Rv@Re^SL2|e)ue#`HykV^ zI<8ZNa#q3-=fu2hmk<K46at~!5F~*P5hBIhIc_uY&XKm+fz=UY-C*%4O(P~O+6lZk zd<n+cA_|0B#Mw(7jTxE^V8I#0hw|0vXSziQ)dz_V%h36V&L|=2tS;RVq4!#i-)|zx zUWDsak$ds(#k*I%d-1*#??emgQy#EHrzoi&uc@0(TcQ({%7Y3n#J?Bjcq96xNJUrH zb=tF7)gd+C<`Ud#2jzZZ-vh|63wAW5gXUjxYqGdMjQXXVIr!!&G;|Zxmw1YtZCk*& ze|Z*Stpm-!Qm0f8dG@tD0*sh|eYBwUGI4*{2RQVA?sGz)rq1mV{n3*_{n1Mmlu^I_ z=tj{WAv5R+oJC7LX{YQ@XmBfLxhwA^y>syTqgH=^RC5PX?##I^iK;^=-Jy3;UPF4z zNj`E4Kr`vDB<L^nN6za~>7QBjM=L`6qd!FMkH{YHw2Fa(gGQM0s=DaDGZ6)a)$(}a zcP{AVRAff#Z8`M~(O{bQx;28RukaNb=pg}4e3?K<e>9BBqxz#3`rbL@Z&d*3kN&Jb z_0h`s@cmJF-@TNl=#OUnNw8))UKDFa2qiw`Cy{;51aOF0vkP@EV~mh7Rs@*&f~dG5 z*UK1R15k|T5raAhLw{t#CvP#uM>pcR9njDpRe^Hva*-7EM|Xl5vOhYTB6M7u3!(IE z@m`8I*&mG(8NBDcS!L9}KRQyxr2c3w5?B3^e;jzZUZ3J@f9<qlQ!kCYaxtG4O$)8^ zx)ZJ%19e;i>zEeQ{=K+;b4l6<C7sr6x*WLuhvLptu!QBC8J*e5xPZf%hqWW;!76cd z;0~$0T?tpwEgsm_Jp*onf`pZuu`)hCVda*BBflxm+daR`z6hIurgvsz4r&q<^mmv- zUqP;j_7}+G5bYfUK9Hv&8a|hV#a}`?iaSQpVj$><!Y$Tph{W@`2Gfk}Q4^@%df}ZE zW6?E8j27cxxtLOI`w8iL@T-_ysW=(}ry|3eA@^{!_en4s<P=l07D9z%_&wJK?5$ub z|EQ@%A1Fz^1o0_yqM(;DZ7YRZD=|F}55)1L-`ffhsU+{mgsW)XKhhg2DH0o-iJYz8 zt*a@KQGLTm&CC8vhN&ht3X&<<Y6R=ULUe>-Ph+<{&NkhXkg#M0F#h&MQyxpj2INlb z1Yr<5u@dh}I}#EX2w+=|i72-b58&V-11|-#2t8EXfu$;Vr`1D%DO4gGc0>#Q(K+bF zbW-Mj&`>A=m=eV}(p;u;4pK3TLeK|9woyt*0UyYfF@>CG1=Bkxjg#e~k*6+KASp~x zOM)J`P1e6oiaro!kSI@PvFS4~(E`yyVkA1RfD=M={P8ao@pY_{ythcjzUO72kgQU? zjH2}oeRsp1q4*6?!GMCLBxilsODBiaR|y-VmJaCi1pFSI3xtoT{Sk0I(3l+0fCze8 z1<%z9=0##YlG25UVRaQ!>AP3tN%NGJf~D5fzISDPQvS&3Fd+emC-W`9tL!Hv-)7@o z^KGk_p@9>@ABgFx@bs3X{pf7_(c}?bVG^WBoNTqn`#kC#qK?Fzo!MhiGq&Qi4;<7f zjutc29Y|AbV^NIR@j56{3~HkMh%c168!uQa^(a)R_xM}E;La1r((h6D4LOxsMo->f z@#&jX^PT75xJ2Q<p~KA{yP2wm|LIJc`_gwetfO%7?a{v9M}@S=sc?C?*=oXca37-3 zqz%o)&?8SR!5#r|Y6+dmNC)Xo^j;<8Vf3Ra2Q0+LY#ZkP$=1Rtx|o@wS}Ud3e40h- z7HM$@!Y8CHN^PZA@-rgP8Q!ymTIJ)<0y1!X8BI!2mqOwEQ~ea)(OFndX|AS^pmiM} zJ>LJ~a3<PN`h|D*AJm3YClnM6`+V{JN-(gveIDkzYg03@rIhqm$B?QskPOnz5GQ<* zqZSb`I3g^x5GD_)o#@lPrVxV=+5u+$Q^myoI@Jzf=yZGH#7Vfgw-;%V<B584?0H>u z3(3c3!4m&*xT{4re*R>L)cfxcq#~>1aKTmbfpU=^i1_Ao5ad4=i`Y3GY`|hA*zi<e zkO!zKYbk<WM?nSGWrp5OHz6c0w?<{d<~OK+D6Ac6G!5pU|IVWS+`u8RXz!=S<aN>$ zB^4MKP^GHm=OB6Nxu-!`NK1s{4wT4?tMD0>Hp&?6V{+hS1kM!Rfon0aJg)8e=;x?v zetUl7z;7J*jRU`N;5QEZ#)02BVBkPYeRIv+dUutrVZOVmy0+m$^{uj^zP`a@tEjx7 zrKZVktFYBK)F0Q_&`@h@Y^bUCxSMPZ)wZgd>S}kByWV5-ENq0g8&6|H?Lt>geU*EG zt*)inW1Hi)U0CCpS5t30J7b)!x#l8wS|D6qLsd&{OS7xWJ-5m2w$(Hv7muyErLnQ0 z$pf0I8=7p0+iYvDoae3+8B{j7tE*iq;(o+X<!Ev@yPM{_MT!;mwi&Z#**r}Z_07nk z4jGl!R#2f9c-VY*rKh3kgcDQ*)-S)B`uP>LHB~4mcmdK{>TBw2JT<^w<W^WJgb`Wg zZuZpFS9od~>TUBXn(<8}A-Ex!L42!ds%v)DROuqg5X1m?ePz?aMiCi-D(c)|elyz_ zizVF=L&?vT4K4MaG={ud+|4${D%4ngOKq*fx~R&CC{II!t+t|RuA4PiR@Ay}4J{rL zfjLN`s+obvhAJ>pB~<~D2}GbWHWG*Tw{Ss$5sj*gYKj>-8A7gRs)0w8Z<!}E(^b-f zx^>N|Xm&e`XXLqDbL(4Nl?xVRX1R(RD$jR8W@{=J+OzC-5kAjV*66CMIN$ARo`-@| zxjYqf*#x9Tf1wZ3pFrUSD-m|TGC?9UvYXu%wG&E8N}C$0k*|IvJ~K;xm|EK~r=r&I zSqJmOGEdfcWB!evCNw`Vu+r_6X=S%~YHE?7yRM;WVJW1xWR|_G!daH(vhRx-{#=Yd za7UBxy#k5Zd9s8Vg%b8ExKqR5shF(_;wnJ#T&~~2ubN6%v!|)1e(nh;xJtmtdc3t% zdV~}Tg&^93icsHDH%BTTAtcmhZOyd}9<-|FfIea@5U#SOah@CMBz(aB`8T;MJZ@H5 z0p(R)QB#XXUC<2WAsV1Ul(x))#7uWDv{hAjDkx{I6-u=t>G#DzMrfpdG&H$I1yYew zzuB(S-4*A%ppjZ?JucF;t~z&JbxXaoMAmSgKUe$oDf4HRHQ0;iWRfr+czi^kKBFwd zm6=&)FDNc-5K7TLv#ch7oDKB@O=dS_B614<-~P~_nNjGkqtFr@aQuT@0<`am{rDf5 z@%@_W{>DELvW=PSHT`gmHAWWlv6u1%#s7H}mU(hqlhF-1r$iLi(&=THE|eq71v##C z&ugfyayL1PYaqf(h{KDP!0s*m1EIK-Xvm5{Mp=Oi^q_y}Pfx%6gLtNz<1Z-T-qZiJ zOV~KsG~DvL^%l%261SJ^@omHJ#B28Wb^_V}tD+dY4sbW%IzTVx(k@w!%?UUM)C)Lx z5M#5u_V~UA9CRHf_+qfy?*_yNoCLTuma&I!+~XS&$JqCP3jqHF7guyVW4i&X0C&Pa z8H@RbneYkz2$%ua*E~#s%?3=w$=Nh7!48494nE~|LqI0MLmB%k;L@Rt{SF>e@=<mI zehpZRIg>?)Vap_@u5yPn_88!#5saOKX~{PMdoc&+Ih?UVtS(7Bg0XHay%-GV>Q2D1 zc<;p$jITLk`F{qTD0jyFdwiD=#1gj+DE|g5icLlNd$1^$Zg9)T%J&5*f9rE72g<$S z70?IR_39p<73JUg_8#AA!2Eyh@nxg@t3Lvt05d+_<Fldst9RjsSitf<NC)LlOz`>W zpllo92*7;648T&rBEWLMGXOUL)&h0|wgUD5t^iELhLa}&ZGb(1jexNzPY+-QAWQc7 z=#0L^Lwvq9fW3o#zQh4Y?@*tw9Iy+s1RDS|j_~;y%DWRV7qHjn^DO`@KN9qz+#3ML zq1=r}`Fxjv{>Br1zEsfPH4X7lf2GqwFJKp7E8qseRe(K!8vqk$_<Ungzm0&GP(7D| zF4S+&Or(SQwVmnn4MzR0KFjA@4Y;8K5cO;GAbr$t26kY20XJY_<{H#<H{hdyt2=$Z zS*YipTX7lzVD}oIZyD-)^<yYM>bV=R2(TCM48TUr+phv-PxyRq0%iby3Rn(UPILjL zK>l_DW&?IV<?~emc0B`r1FnY2)C*Yp9MT;OdH^p0?Ai=G!7bnu%C&l{&({seUIaw> zdS3PUCZRkT+fhHm@ct&~0qlOy=UWe0`mxV<3(D91AD^!j<)R14iN|!&OlDe;Xc`_n zAi4`4>r`M-RF-iiWEyqE;uELDr%X$TxiGqwoqX8ujv1eN6h)-?Mfgj(8nR6Y^-%H8 zq2Dn+ACeOf;TrMxBkq<#b%}>?i|`j65^fc6R^TAk;-PTs=@)R*1P1jD*p0tQz(H)p zL%7%QmutXLS-kkm1}-0v6~relF&D+Bv{{PcZ5PC+6vZbN#7B$tXg^Ic!jV|h19q}h z;2gl&34yq6=JN){N84kFKlGR`DlHb=8*sA;VTNlS+VG9?J_m8n1+<PqCCsM0!OAvE zK|BIv1Mk4!Dug)>`B)PWW^#PuGNNw@g=w=+jn6k%4T`q`gWM;_M^BAmxrp;5!gs~` zd@+DBP65S1ncA%8a|Y?q(_+j`{+H>{BgF{58*!@Qe7+Y|+)nf4_>^T7cZsziery{r ziqA0rI4FKBC_t*92>BMom~Rh3IguXmoDS>!5c=>`{HBM>WkcE-(2bwqxAkmra+;Cv zmukKjJ-5XHk}XS+1he6f9s^JB!{9*iR0boN8;ie(uG-^k0<;dMT9V~jfjnsq-Y!6^ z75L)##O1{AYb=xFZJpN1@ne_qg7}OjQAP2&Z3Bwp%gYDF=c2GEGy;GLvZzHU7v({Z zftDi;w3N3ESQ532cUrHpEH@KP8Th*nG@&Un>vd2tc!<|k==-+y=QTLH#9E|yZ84cb zI9n8BzBL48j}hrTg!CNH&Hn<VdRKKP(i3$zH9o`gU}$>NVl4JhoIQr<vsxK@6FU5l zdrR+*(6|t*`})&I^*#&fjfL)i0>5Qjvy;3}jkjGc<q(Z+a?C>iZxoA7LeOQ1M}5Xe zs5>gxH6)XrRBy{jCYO**wndpALavZge^sG1NO^k+@yk<uzKiiL^M&M!HtzE0<qXM} z=p`Fv0c2(&1km~c6e{vwW)^KpwZC%9TPS{@0jjo-{2pDr$M+i2uzrGg1@Yx=)+Lr@ zNQWfy94L`e&=aaRD9=fVy9oBeP~>ghs`D&1wD}CnR5S7f8M@J@A|5>kdAv41&^9u3 z-h&Zl9R7BKwpu{zrPN7CJ@XLq9S3`aWYT(p%r`urr^W9ySw2CrGa#ejgP-5j<ioWM zo*$rPr>QM!314RIv|w!u(L-(HCD2n3XiW*w(+WH2lA%gg!qqw2D(d`3DQSKhro})Y z3JpY&4($TJyJ1_M09(S^ja-GEFN}*SU2eI?+R2wiwGFVOnbU1%PkMfJIy2v#o^Spj zomtK_4??#G?dDF;KPO%A4#^q$oCe2*(m}CNiD>xdWl^2{8tZauGE}~7{GEmJJw$rv zFN8te<1ZEEtHqASbl5Z2&A_9K%grTW^icunqiOLQqAX_&gi1oWLsXKGbE;Q*3^LGp zlAkujHYz2A)3i0g7N-?b)+;4#q}zUkO~$tD@x1~ddF7PtCBr<!-(V)k2wA2e!8QWw z1uC0#zYTwR=>K&eOywSdv=<@mv#2h{=yId}`sra&?y2#;7E2rtTW$x*TbOdMJTXx2 zUQ2{>uWyG9Om*@D3@SWC+oPb(@pr7FqV}F7Y3rmuVj1<fORUpLCp(}+CzH;Ds_}P# z&X`#tDsKvi3)3#YgFe%JRz@3NVqIqG6nzAhqYJ#60%%<;Y0Ag%;%L0X&OaBlS^ogM zmfaF5yY^*KGuj52CqrVFLSl=oB(YOs%rB(pSEeIA)#{@t&qYXk9e!Kylxb7DSckNu z-`wN73bJTj34FwMHZ8u(V!76WHV@e|ws}&~;o5wXp`2ZoFm3)i&~qBm;|P>B7cvoz zu}d`s#+qv=>*VE@Q>dYqik3RVnx20q%DPar&gn7cJ5cy<QCKSL8Z^9RNc&r=+w)}F zRJSXT_NN$!d=ER<dYvrma_UEv+&QWLM?a$a|7r0(X3LO-$a0s9G`&c3+b3w-R5w$R zyRI9D)+<F&EZ>`;Sf+yB1tzK+v{s_C7;WyIjy=9PL}vwIqzq0%*^APBzM(L1t!D!- z^g@8n5IbdR{A#mhMRZs?Nd$#lcOlK*&-eIlN29b}5Ft%bcGc#(qAXX(hE1~|LYq7C zgn(RkMbb4hmhJJqOKq!NmY4d%d9kQ7jK@h<8SL%TVQ<p|QrRMWn|C+HrUD~ztAO(u zaAY5>2d)*k#S~81lA?cyU2%4NN};gfr#`o(R2qp@8s&Iy3xZSpPZ7W8XT~Qw$Dyul z_xNNP@J_gV;3gSx`M}KrZfTy+w-LXktf;X@M!@DNiaFJ<V(l>(`+pPTZHRUbXtkc; z^JU_<^+w3?RI;6genz{LZD1V2c6aXa9Zg{_mTiDI{u=U+MqIcah}X6=!E3^Q3Vfmg zPj!$DJl!Fc0{lr-XTOteM%h1zP2-Pp8VZmal%@!X45RZn+KSeBemjWr9f!Y@K<6}S z1H&YplrN3(dw=JXV+P27I6L;DU<EAtqK6P~_d1Mqkp^?nc&tH4Qz>3opp2g)&7-L- z-56{K+Lb-5{oiG>ya6>IroWBU1`eMXC})=`Lcg;L<rR8(E@;3*ZJ-vuj8@R?B!r;* z5<xfdrx;~=2;ZJ19{mn+gpLt%1NpYpfka|HEN6QRg(u$4g4~Jl!}cDY`T#HT{`y{@ zZy?f-tpClghuB%gF{cK11m%dH4Z(kOz0bD}JeK3DaC%GVn4<X73&Jwj9%D&@c^ii9 z_85{4@}F4mm*Zxl*Ek+-k-Wfw)CuyGOd?J>{(2B^9OAu3<rHJTz0p<~mV)Uqmczq~ zgfoVCnGNT~x(z<xclf;*UT)qOFR5=KKh`-9_<Xc)lhW!mW2~g?5cI)v9DYlP(E-UM z*>6QzGa&0}B<rK_5)xmGEom$`lI$3$4Zj`Zh@p1*C{*w&^i8lIVIM(2@RZ|^d{%FQ zrnAw<TE7Lp%-=Sl4+8!H%l#%{g!_}7{@j+AUM`1Ws0<zpr;K5y`$H7!zG@Q6y=Icn zH=S&fyO2kqora9lbzfz*lvz<-{fFw2?BbU3hIVR=&`!6*hA2dttxj3~QYt@a%YDe_ z`z!VLd73uVi?Si?ejkFM4K{?uvN$TNC<)q%kmfivQR`%%?=ouNBW0S#aTn@Jjc>`@ z^Y{LvF4Xy0q(hIv_P;hOP{w?N`6cvy2EvkE-UMjf426M*%J(Q}*$rAuRKBNx58-{F ze8NXEHGYHDGMCENpZCHpBz;7Wfv)c|19WYO#QOx$HWTrz7f9O5@p~4!hh>i|*{cBl zSCG9&kw>)C=W~M()+Am2K>Y@6FHyfG@vxH`62sQ7D8K0AU?X0Ou?C$#U_C;Xhjc|b zhPor*-(5wzQt(;i6Vewc885fI7hdl;VyNC4kx?Uj#5<_XyoZi1ke+BWrv%%!Me(gB zO9!=?2xCb~^LeBh3x9G)|1?kW_m=^CS)_UFKGLM&eaczzOM=hg=nTl0Za-0$Q^Kbi zNxp76J|JH`k?hcJ)JGoa#xKAtJX9Y$;hbClqR&?cyF6_FU@U8+d3{*!!8uq=2_mih zTA%Mis`LAGeT4Kk#`Ma~OTxAycvMJ+sDACx$JzL8^+=kjj^-hsd4Ko$c2ilZ`sWkE z&#CcUR?B4ST>H0IVbdfc&E0TMTI(=RfZy~`K0U~X0rxt1V@=oj>_uLCx#eRF7Q^Ht z#w20%!u!Vs>aCYY(8(>(3wMAN>pPM<qOBJ67F~dO4-jvC33ws<4${ft+fr+k<*xX! zw85hxZBhWE&axY5YmFpRmw>iKRNr?8Xp4q%_hzfl_W`w`+ca$fec}l2FQ&$KM_F>i z_hpf6&b!A2_|_eXwkpU@CDC?ifVMTDZ5-yc-iB;hFAAb99N(tJw+^(lk*@16JCW)% z&Olr1zzB7EEXITvko>Ndw2{1DLUU1=15T%kFpLS&GesZIr!hJt7r$-Uh<CS%!=?X> z2`;|3O}y32zBKV4Osva{c{qegx`g90o?A`)4HJ9X#Bs1Kkk+IM@iFS`t1O>eBnI*E zB`*`EP0LtJYCdk_eOC6aiN9}U_nP@@R<^~=-?FlA%^dFJjh47atnf?oFRko8D}UZf zkyFND0N~*!c0Ex8!rDyi?<TzCTk`c53*0O}Svh8qzTtd%6k($eMZj@9lYS3B9KRiw zccR#*CjL?sd!6&=quAq7{52UgiKnwPN#%KDfL&$gA0@G;th_ggJ;2SMC9ya7;BS)H zf1|7`li6$0<~x(w=L7jqN$k5pCoN89AH`)L%>D7aJBjT~;I}5TYZG~AGW$y+|4TCa zM<VY@VqYindy?78Bo0ewV-nw%M1+k=;zzS1r@$4?FJ)V(X#X~GuZgWOpLi+zubE>S zZ>0sF5cI@03tu~et+4R#hO>3t^8RqvHGtnRf~^{GN7NM~*t<jcA4jkohw}CjY~^tN z<#4ufIKO5Ddt*578%_xfN#aXPYz+S+XID^8y9V$d2GECDYfLEfe@*7i$!tA;(X=6% zZ5m*{Ihj2&khdkX=VSQ8$?WSG{!}u%JC<Lc%pQy5KP0iw68XAhc1;q0HJRO?#8)S? zo+SQ4GW#-#cO+A~$Bck(E94H=%zI3%-NYB0*=?rbTg>c56Ja))`B!H4xmhwk?g7)k z%@0E$to$ia<<UvJfK8GgrymPgZGk=YxykZ!JUmhS^?3F-&hLw7w@2|i<JnWud}Ta) zVjzDwo_#-%!@_zlW-`Lx8Ov{tXC3kMwJV-tfimk1G#|h*Ba!SWmTzO(t!Dm3EPL0= z--~5e58zPI-wfiP#Ik?J5}u-rOA}F=ZZd;ei>%*T)(mD}MO!uwX3GZh6@%HPLHzE) z?2Q=y;UVm?*pqJ=%pQ*Cf5z8@V-V(^B>wbZ_F@wM`VjVM65leIbtdy44`F{!=GzCe zzbErWgW1kx{<p#G+C%uV!R)a^_`e3Te;&d=J%n<fl*FB^9>jK-IA)4B%O;u8Yht&U z`R!(~vVgJcEgXouE&R_G_Kt<n{}mX-Uu5nu-)4F$huxI0!TQvAw)QaoQVx6aFb>p5 zM?5_MsC&}*v*X#V<M<sp?7?yT<{b9MIR5B(_L=y-{5ZZYhuwJ`;kO>gUmefh6~BGQ z@fXIkmFaxLc=kv-zb}WqEPj8H&R-kP79Y=%^6kg-%{lB5@%tag^S|V<KJj~T2LDG6 zyE%jZ4PWs)o4uSt1odU`$8s1}IpFurnf!$u_CO~6emRprlEeNje)}@{&K$Nfi|`L* z@h3o)`2FuJ{!I@14!=Rwc>YZ`yLUYOzBOA4(##H$^?#c!%M#etR=y&Ey<(-`e~hx+ zlEAJSz}F|RZwBzM<Jrs6{6~n-K)wi=f&9aG_V6I2@ya0nPCR=cp07z@ug3FFg*?S4 z@nhN13mAJrtHJ7{5q1jyWEeZ=8D#jmmESgu{lxhnhOtMY4#Dru1Nrj9sJVVVjNLbs zzW#M6zxFWp-cUsRVJQD}7`r8fUw;^TDusVFjQuNxuRM(TQh3iWwsIJMaTwb&4AUd1 z+8a%GQ<1K>d|+B^WA8-s&qlI#=%$fuP26p!KiJrf3Ff<OY;h9D@2yFEla1YxZ28W{ zK2GM3+1N+Jc>9s8`!If+jqMo0zZuDv9nP<|vHK3^*V)+L4(D&#*cXTMn~r2%NANxy zd*BHEfQ`L+1i$u3_SF&meH;7f2uky&k^B`KdweAS(#GB%$v+xNw3R0DL)qylP#s{K zi7z#=4J5kIbceEcp#M$mI}_xYJ#PWxpBDbCnf+wpS6bP_R{pq^ePZRGTG>s|&g@N2 zA4Q3+BYB!=l1Z?6k;@6WIhX@B@jqMHe@uL>m8~`N8?0=r8FNL<YvzBnvUUqb$Lt;p z-)LnoTKFyt`_#hsi0&yNiB7ZILuB1%K2{_+W=t+7kSWwhCW^htLh0RNq4bajQu?bU zO2)R9BOYiNG>qSFVNY55zui2<6m$Gq^BSw^LrXqvAC!R}-_1#;=%Y}wJVa4Qq?0K4 zbCZQX9L2Ur$<&TI67VrIe<GHxx9|sJ*#lPoTr7LW%I}S3Yhww6m{yw%Jz=qxFG--U zS0u2PtR%!gMDcIq*=+;(pCH5oJ~rXys(8LWp4}SH?}}%Cjh8VG{T<{89sTEKw%E)+ zHnVrke3zNhj*WpP&*BqNkD|2MV*n>icqNWKW;*=+IOavC6vy7S^0(qxFXvyzvCld0 zi(^}&mJGn>m*VIH(Rh@MAFh<=k}f{g@>D8&VE}(Pm0capuTNz+M)Uhp*+&Wd=2X_6 z$X`xnS0wX0Q`z=p!Ym$U`C%0M=P>i?RJQ4G-jT|#8O2{uW%rNbn^RfODE@9L`*IX- zOQp<@NJ0U!kV!NDi<!Ekf0?K|`ohfKqRJO_a;nMh#_#t`mKWkoYs}`&ah9(K^7msc zR}bR&-5$f=iM1?`<*&tA(6RhI&ax?%e;jLhBbL4{i{lT+vF<n|fC{}-B#C-fX>KuD z{u68JFq=P$wX7J(uZXqWGmzu=-Glg#F_!yd_}8(P%`xb0EbqtgC9#%oW9aMpSpIq} z>!p=nhBSSq<)))yoYy$M8qIc^qW(P^7Q-C;?y#&ey*Qffu<|EHvsFCmj?rwxK#tEV z2fbzn3Vri~quE2r{NB-QQ!>9{G<$b2zjidcVhG=X{}1JFk7jQkN|@({TmCVcb&fFq zaWv~Yg5&p1NAmAd*$<=m52M*_sl0tO+nUPTj%Ht^@+(FYgKbIN!x}&-jKSwjY`2-e zYGT(|_^TG`uYp0Bq*FN<UNPzuGkeN(rwPAFEQX8}VwJwg#6LuPH}l<Ac88gNZY8n* z%1S*O48}(+gn57?2WSHsBNHw5brXMhAiL7S?;FVeU}?B>AiKuOUmnQTS^4IHYzgO2 z4P*~QB>_))Wx)FZc<xacILtb2nT_?BEbokDFLA!c#;%XzD{O3a6#sD~yE*0*e0@8X zKM!3N$3GlN_QPu<*}d_6k&WG+z`q{Jwj~he-DLjKNcJH*>yd2zNPdfrG8p6pi7U+f zCQ-!SkHgy<6TdHxy=>-p#<4fd#SoVJEc_`n8B5OQIQF=ecgC@moPQI`7NZHou{&e< zKhcO{_^Wa3lNi1-jxB{X6Gv9q3vuj)SiU%p?TF<+iT0nAG#uugu<ULDXYm_E+-ss@ z&a~f~#<rWH`o^-W%~SCE6$^iBEc03UD`VNUod0z!%;qy6Ok*oy-lehEV|hm!`yiIL zrLmvl`JS;<*eBB1-HH4kX>4f{VcLdTc8z5(4K;r`mOVF&Bkrri`1@no4)jfF?485- zb7|~}!#TpVkEHY;8^s?O%UZ{9(6D|Czd4P)F@|pf4P*EnY3#1C{4x9%B6KFhk(um9 z6Tij8Zm{s{tfcht8@0KKI)LAv-#G9a2Y%zgZyfj^<G{`vWmdg6NSLVJHz@c&M#yg& z{i+<$W0OZF%53!emXP<?L*CyDdH*cr{U`M<XJ{B3fZ-KA(kqD{jpCKXNx<HOLy;b= z$`()SF|y+yuHI7=gey}#tJL=q>OES)Z}#{a=x4dI8(>$6r%^1@U<{^_crq|2M~_}h z*RJRzpEW%?KMarLlOYrIn<J_q<X@#nCC7%!SJqQ0e`tK~netuJxfAnP^eEuZ52ro} z`gfiozhhUXc-Q6FsouYzEx&&k@?Bzo`~UxS4$K?pm$aawOi=A+ih{EhtX6Qof=d*< zLBX{OZd7oKg4-4RSixNiMqMDYJygM?6`Y{p6a{B1SgqiE1(zszgMw=n+^FCd1-C2s zv4Xo4jA~N(D|obm6BL}H;A{n}6`Zf&5(RHiaIJzH72KlWb_G9HaF>En%_@Hdk5+Jk zf>RWntzfl+^A%j8;0+3{RdAz%TNK=`;KvH?QZUM+@>lR^1t%ytMZwt$Rx3DP!6gdb zpx{~sH!8SA!R-outl%yMqgqt{3LdTC1O=xkI9tJL1?MZcM8O*rT&v(l1-B@;UBQnP z+@)aDe3id~M=Ll%!6^#PR<K&Z`3f#k@CF6fD!5U>EedW|@M8scDHwI3%3r~w6`Y{p z6a{B1SgqiE1(zszgMw=n+^FCd1-C2sv4Xo4j9Q@bSCBl-^rX-vo#B~USa^bMY>|6T zO+~#eH+_71#&HwU<PSrr@INi^FXaX&CjwfS+24N}V_KMn>3%a9Z)LiF4aRe(`>9}j z6x01%Fn$2j{bVpc+TWiB<I#I6eHDx!#B5>lF)TyzF*tlI%MXi>^V=oC;bGPEYD+2b z#ljMp?LXo0iLB?daQGxv72UrAEG(I6`zMe(1NRSM+U^O)4`!>CJrs-|!ou|<7A&Y( zl^!nNp;+h^4xiFb?kFD%thKQr@+k4zN()eP4GbwO`_dP{$X~=2Gps_*f2DrKT2{jU zMb*opDq<<1iN&+A|CapJ_S9^Z&NdbPH5Hzk6Qy&@LW!tV_?s0z`XY&_SNL9qpRe$= z&x9UYQ%mU_bBTPXJs$MbEBvXg;?-|Yu2%SS6rT1*&=UvtQaZ0)A>XG+zzP-quq!2E zox;;xK81f);ZIZe<qDtDCK0qgpPrW$K5?;pr}gmkq=5gFPSX<kPV2_$xm@A(I1ke< z;#mc}3H6(%>Q^PhVqhSLtH--kIvcWO$c-wU!@(!Yx3pa%j#l`I3O`@riNEyJDt!0V z@?G<Rj=QIHx=)Y+X)Pu_nEn^|c^&ed)>zV03&WejuT}U$iDp|AzE9z`yjdVf6#ndG zGT=-V{#=E>PvL3Z89k3FeC;*zoz_3mlL(bW>D<{R-?JrP^A-Lrg}+1L;j<RuFS%AC zXw3^f6EQHMblCOsy;9+CRQSpj5>N6$&yNbP$LXQ<m>*J)$r0-DaNsH5{2OJu`u~v% z-xz|=Qut860Bt5B`O*48k<U&S`2KqAGL_EGko2z>;rr{uNc7(=!Y8uwFnTqd+&+Tv zIL)=6-hM%((_insq3~j{3}QkI#Pb#KW{hU_I98$AcLI+}4(X44BAx#FIT}@N!e~(Q zO{X(h;D`3p&&L3N6zVJEdl^!X6K@1QSw}X&XCmO!Fc7D7dc}fntXorZRSo=+(6?Pm z-)cSFs=~L5P0?6Sr}9`M!VhNaj+F_dC_E+u!_s4km-#k|Mc&YlDtsgG;poS7rl7N2 zh1YV$6C}R-4vEozfh!dr6EPxXr54!{iHs$wyo}&f;FH0Jo?9iB_Knh03q0|s2em_w z*b@x6MBy`j5@;-Qk#M!b=PNpOUhk^(cTSQ4X<arw=?D<6yq5zn>V@t=!$bQJ=oyT_ z;ldvs0bd9_)#K_unSdH*u*V~W|7QgJ=PI3UB@gFFa@dSR!t<>LctOA71JsmwUIJe8 zM4<#2F<8<0v81z7eQX0BS@(PHh=6|!c*=LDY8SNUpPu9);nO)I0-nw-377BY2>2Zl z@W-Oz5dZTP-?Ak+tWw|yvsJ3SKs?0r0`TGT^#U*C@IF}})m&KyR71G%g}@)B(li2> z2#<c53?V$UC!3xNBc#(20l!hC54{m0pVQIsiO!6Wc3KTQwXfc5B>l8rlAdRQC;oI} zB^*5$i2u<K4hvtdb0XmX7y<u0@VBEs%2oZK*4zJ$5dQGP!snYQ@Swj*(n)K0>A76t zdvYXRxf|KdzzaE~LnQFfQbT%b;MhD8tC2S-yIt$a+ZDc4`F}J%1qzM$Q?B?^Ad_L| z0w1p4mqx&2Ibqm#*Bc=`J6!TDBZO}mz=z9sA@EeLogw=Fb&U@xZw?q0P7lllK3w|u zN5H=&@Q~-5BxC4Y3VObd5T2GFgv)mt@S>e6Jy|G|Vb=mra=ZFIiJ*Pv^sEP->Nm7M zO0dcBjY=LeMQFxs0<T^SAWPuUUUDVjr>bvP0#E#mR{R{R@V5X@`7+i2Uaau%D|};* zOh>H0$G2mS4A0LK1s?WZp$t#^X6U&9c#*zOzUzK%3GhVE(hz;M4ft^NH3-Tl9G*_K z35PELp7QN}KoX+cOWY_5kNUkp;%P4cJ?AMrTP5E$-~Ir6IQs7w;bmCk|8GaY=cR_1 zlXl>V54MnYx(E1h<w_nc=`UAwexfqZ20ok|E&`tDZ~aaNoTkFBRrv0EBm>0WKzw^k zrN1FW|9ma*DzX6#z(9=X-=^B5?spCo_;_afM5eFh_7vd5mCLQtc{8NF-x4AG-+`xc z<$o`Uu`8OsQ|aV}@H2l5(HYMYRllkA;W+|7n9U23!&_ANZbkoimHt0~569<GXi(wW z^+^J6V3_)OSp@uDDxLCeG9aBVK+j7GpSVT7(^&}gd<VSXgQ{Om+k`ZU?+US3UjsfI z|G!t^cdB&qMe0~wevC|~v|HAr*cXLwKH#a|-we^i!;TG~em(G-HkGshR{>AyXup$| zpT~ixa`paI=1cpX=s9+r#P=wD^@Kz-=W&w%tzL<sJ@xe51-$6*R6Wj?Xl6<ePv>ah zDc?qA7oDrZ&y5iN^$7ScBjEY*;rVa^@ZsvED+2z{z*D(;Kal*vt9afQ_`xhg$#bs4 z7i6e<jF#aw-_8P_<aWc8G9aD*LC@1V{GBp9iX$Em8Zz;xN6Cq{!<Pdej{e&t;J;Go zbgA~M`8+gB($h0t=Id68o(DYTyWxI`SJ`73x596bc&h{qYsv+Gn6h)uQ1r*bU=CNm zGk_;PbQj44Xs;zbx2f>0iVu?{n!N)&@qhJ4GQ7r*$dT#Wev|=~n6i_BS4B}NNU$&h zzB2;;UV#Ulf0yy;j1YRdBZU7b0)E&8#b;%I!U7P_0)baA2JkfS#J8?kSzbD?gPzQZ z;mdWFz(YU(UWTW=h4ie85dK-<DSg|+GCb|&rROV^PM5N0=$rt0hC#5y@$*>VMY~Y- zQXzx0E)gE%xYZIt=PA+i67ZC7=(zji2<gY<$@I(Bxbh^K3@+db->z$cr}Q`6Ez31f z0(O(ahsw#5I=m8OamE+&cthhu(y^W(=~<xqwFE`aP~a)w#I-U*J>I`Zgohm%(m#Fy zd^mnue<#xkmHz_Z!-YRr;31#Nu%i8c^gIpx?PxEy4<$Vc&H5swGYAbTT>9gIhiTXE zISqK~N4i7ohx;R>^NvUd{mUJao)Id=vri14?^56eKUD=P@nR1vd})YY{ab`|J{EZJ zr&nf3XVlO$7J?M6yv@KT$sj@huN2|?>w$ZKw*`fiFIys{|E~ymIzTHNJ=1^>N6+~Y z!rvPK|2^=;Purc6|GIzqbAI^rUjsfIJzonv<UGVKI<i2f-!(?2Pv`c~)28sD{<m9! zC;i{^gbeqJ1Z;~6AL@U2UxhE-E(6lpNc0>!IXs<HBjB$9p5%5xi2eDo2oHUw>~K2k zg`VU>C5IvH`cB|&B1G_i52^4wl>(r>==6*!3Qy;00*`U%8X4{cWe;ow-X>8&|Mvhd z^jnC&+M&|fd6h|or@glHOopKzt{$%hKAc?r4R}hwTgi!2Cc|Q;giog^0)9;d{Ie18 zp8zk)6=H8N*I9XVzPqcrv7&yaI?dmBzIk1GB@Vbw&zg{hpjnx?grcgU&V}>d<pmTj zitG<Fo~64oGqMpfGk31L-ra<w=}Y~WK+M-kQGgi}%IJoRrot?HW~RJ1pv;5dI!q3Q znXUrmHn-G?ob0ERc_zp^FVuAbLFZSidjjk_kqIKB!rB_#`H?kIQ{=*(3v>g9t5KYb z?>gVT(2ir|#SI5}W$wx*w+CnC6RQ8o@|oE)D=sX>r6KhVbwwz;&S|0{r$|N0MVQR& zxo(e(ZUB&1Fz9@P53tv%=804#qsC`cR93dswbX)1VM@?!pD})Jb8R-&qf-?oh&RP` znYr276DDS7XG|PFF(WT0Cv!q(4vq*{nN7@~%tFhDvh0<_iJ4TW3_oit_8yvqXhu+d zBZDeU!FW+zSzVdAx|XU?Bl1F#04YWZ)v={iso2@-8WLP_K@u9ooT`1YB1rfqRzOTE z=DNe@9>jIXi{WyEeP#pkoLYR!<bq<CtGQ*4%jv4EnKKV9ps^8*DDxEOx?DB5y{5)P zd@1uxbctI)GBdM`H!wIWaTa}MC9eK(;fk0VPqVX<?#rmO+i^m@;eHBd>2%=c&vKRJ z;@J4C3|vrhVMSAws|NS8RNxL7x;6x?#BCOhO%09dSve%qd36h272-OHIf$+}JwY_^ zJX|s%uaOC81Kj1%?}`T<iFita0_peQ&WgIyiPL5{?e+6rrMU6ryg3U!?t-SKiiJA3 zXu2p_hRadWJkL`xr`GMn;qny}##vWfmu<I~d9oWTYMPwtJpK%I{RyrV@izeDwGf$^ z>V61nw~%{M76mRl7z;jExjnc%1G$yCYjLGcX0r#$=ejRwftv6%IC0uOWry2v=84}k z$_ibXxvr9iDi<{3T(^t9;7SmrR_Myga22_$E9mwM@v%yL)t^e6YU<p$9fM-%t6W5K z69XmHHA(kk6wk?!S6A5Wd8kj+1};EyLLO$g3bP@TfcDdBOQEbllH``F9+%rt;l#Zf zcDoDpUUsT$=JZ)EV+~IbE&ewi|GGTZ0zHvUtC(NmYG|74s%Wgh1yQc%h0RsXt|r`R zL=u6f=V@xFMdO=`Ta!q`SJa`2&3DbfO-%FM4!gr$+vuj-a%w!XN6;x#zmk=OyN3J? zGH`hdu0q+XMxKy&5Xq9dSCNJO<lq#z+3oRPV58H{JV<FP)vlAJ?kO{`qPDi7l6qNE zN-m*s{K{&kQ4i-yJ=`F#1CnYjvO4y6wBrvF@v4i{#AhF*WGm+4cC)$Caw9Xz&-`Yc zQ_exkN%iox3yWkIuhXA!kkStrOo0n~{DnaG7To!&p0~2e)9h)fu1>GS^<7{@O*1-3 zm#5BES=&(WZf58!8eDV5MOUsW^qbADik1bqcdf3m7FV5BrB586g|04KAXhcou9PPi z4AQ2BteWmeb5*s})h$F6!<%U3K|x)vDKiU7pvq?yxm*Z5b;fBfyF(!zMKdYO<_7dM z^;NZQR&;uLle;<{SBEWF;DXj@Zm7qNZ=Qv&`Ox`91`|0jSJ9a>3QC-Xh&`*MP$#bV za>|q`W%gOFSp}1e?Z}c$kYDZcF7#X_=!DUR8W!5;Q)bhJJ#I8%Gqak}Ug<I^T&dNJ zTb>#yoCqc_4a;e+pjx0_b)l<rP8GWBOrupWqs*0+by|H5-OV;r3}mLtK}tiDovb^2 z&anHhuPgNrT<oQF$SGj(_{|HW8EOx<V*(mu7V&OYnHpP#+>{nnfQhB@u0m(seEY;E z7$OzTZo!OMFn#uN?~%dgR5$q8i8F|kg=_Cz4b{>PDa^FPh|ABPTI`%$c*Ys7@#)#= zS!BQe`W0K#+}NPXVyv*BTi$5M=y9QLq}~-b)Y*ygzb1J_nNr#Wf2Xw6SJJ&{sueXk zabFQ`C+l~oQ82$@^n<Tu*%2;df~84z8*W@{IG@H4Wu81&LH$B8EJ0LNQQ0nk=IB)1 znd4oP0dTEfJw_f)WclvLwN$uSFT%A{cG136YG^qI8bFbq;=!7gnYkLEFvRVTpl#M{ zgq7V~O#_A&2pBTfum=eg6(_>To<7Moomn^?nby-4j7>#ZvI5i>!!>UD3$o0>t(6zF zpkc$bmucvm2=OtyN*UW#XfYnxy|C5oqFxV<{Sh#@WL^1(VF%Xz6GcI+^0&R*Y9)6< z-fA?Hh!y_J>+FS@K|+8+8dalR-mC?ral~VXe9^s$rDYT4s4v`|WvG39S=jvfcIbOX zJ|0D>D=R%Koknhu0(66QbS<bmxET02tE``Vh9PeMZt2_ai_L^y$c0dfJ1pTw2qnt} zUrANvc$B2D(3PE@kxs)MBK246#7I80@Mi|Qwdm8Ym~&Bv>?dX;lTi7D0YkS{?$bOC zBb<U;`@GNbXV<m7_A$63snrKjLY(Vw3*-q?fG&H*xZ0JOL-tn_-LvZ!H%^NFFJKwu zYLnhsH_Z+qFRG}6ArFDf2wXF2FKRqxmej{GyF5GH4oj|)+={9yQpMyGH25Epuh9U| z&c~T(azu0>`>Z@@f0!(8SG~vMs)uo1xv&y9a-yGs3v?k$kl~tG3f+b#<}9s~wg_BF zxJPytIX;>L#WChsQAYF#lA#7s?Vfo}4Hu$0RJlZX`}0vM*@-o{!Wcdfm$;~udR%Gz z<oPXq`5vrGP$UwE1HV2N@u0f)eJ!Z5XiA^$pUO~T1m{?VYYq(ddKX4-mGkoRVdNA{ zcDgdtg<~;Lb0IxSSxvZJq(*v(#-lTVEk+}8H>{B+x<5EXIf{&7REs;6L%xM~gfrN% zm357wVU_03LH*3Z2n53pQA4_Rw9j1`cl+0EWc`nI8(QRpF9R06_`&Ij!w%<^SVlIF zew8~X$rD>4TNybD+zTq*;>zn#nUf7}Jp47nRVu8i%!!4htjb7t;iBz2Bw(+LsKk`& z%fgI=f3Si<jhLknGZ!>i(uUP|sm_!kmp9ezQ5QC2JW&Uq67FtBXf(zQtt@KY^>aP* zT<)f(hNjTL0ZA#GFTqlJaH>+dE<j9HhS71=f?1{USuPLUR``a_Eb~AO<1ktzE5s^7 z;xnM5rF!#QSfyn!pCcLZ@U3biY44`Ab1$YeTH?>uo+08LrjYxmDi6TA3S-m-k9%ln zwD+50{Td;XM$`9FL!*fXHMfOkBr4`KG(qZuv^R}IGIQiWL|QV?koz+nF{G<T)FCNP zoqb@fM=I$6D>Acc8_?sTtw^r~xe<bUP9bk0;s)v7$Dsaa*D-YuqR0K!E?P(Wvu9vL z3EnQKm{T)9lZ=nbT1-vI>%MF1s~a#Z45~n*Ih6Se%pizCo?m%2%!LsZUUx;FtX8sC z-Q?bv;+9^e3Ch_wzXD2}tT8w$r27wbjyvk%Zfd9%Oy9>aXuob(Mw^)+{pc9d1kIF6 zh4`xtUes_qc*c)4UqDS!c#PCo@ZfX+dcwGXgmtzueER#weul;ro(`k)%s3?4-<ebL zMygxw?R9nN80&<|*0?ahi%gmYJ>Y2<s>GmLRc!wmCbAMI{~&7b#(j`F5x&k~C8b&g zjFAO3AvLbm_(V*1w$!#X!x8Es%d~|IVthm;9cV8dP@l@u*WIz2qJ@j<|4JYI>rMWM ziVs;ll1q#f%M;*{)5}`wFpk6!u)m-vPo=D8qYDjw-LMs^`!%CW2w~AbMR0S^p{@(* zSGsE5^WB*EUf3vy*At{&Sqeqx5EBeS-OVz#EbS!x1<r<`k%}m0KQTGzR!&uIX}@9e zJ{ClwW}vb9`&k-(e$m6}Kj94*U5jdDT0m6bP}c)_Jq-g@Ok_g5^a_oDUEOdV=&!%9 zPAM`u4M0T-ajX)GhWis;r@Jf8hcaxY^$a8t&Jsz1p>hnf4#ZF-9_ahiX%W+b*gA&9 zWJckoHFE;_d|KpUsJdoXO_jlsEP8JLbP85(&{S@iNtM0X)%)393ohlk${OJzIv<nz z@abUD5f;;^V(;7W(Vvl8r|+W%f^@opZL`9)%wHm&+9C<#)RgMTAdC|rDm1hWqmn|_ z?+Ce;?cDEbi>S+J>5yDd=!&q$4E)~z<!cle()}|g7Sw#j&*=2RRTl<Ozrw7Ia+n4L zEBw^yujr75MNs0R`8+u(Lmoz&LV)_fOjxy;xD9l9Ar-Mte;kw&nP<YuiR7T>JkDw0 zBc|Hs&%o$lZhZ?4O=wh7kEvcP;nIuToMM!pLlZ>s_KJyFy^gfM@hL=8Xp*e)RSPSs zFxG$vrK+Wp)^571tKn@9aotWw#|<AsosfNPE+I^&OSpAO=wy2tkEIVnXSptNH#O)H z8y3QqK%p%VGaAlfV^OkP#SN9`>-Az;(mn`qu%AXIXEgN)VOcd(eA*9QWx51cHLqr% z4q;yZwjfV0$jgS;w+goEc;WEZtCbJ_f>G*9_PaAa5QnEQGf}CTwSl2@zllnzE4BG? zpp8`xaHNFF5j7@n0_!v|wM*+U4(?iw8D%xuP!Lu2(#qOUOQ))$zsVQk&mjM3bIm?o z8U`CM;F`izTLjk>+9X!slJ06=SYHX!>VmB>Jt5qXb72z{7A@B3N4yY&15*>^I{VeU z27Z;6TePrVEa1i9MK7vQbD_W5%5^E2P;vgX6LR^JT(}W_BHcJseSnkbIj*t^5ms0j z{lj}(?Qx26ZZKq`gr;z*PYJjf<kUOt6g{|#umU5SmMCDBKdgroy+CLSFto*7Op?2r zE2`bJ<_MGjppQsAI8Gkp%FSRU60yJ1+Typ4CTLZ1P9+wDHOV>ty>|pbM&{4x3Vybl zC-=m`5gU?b=>nBP<6qFQ|4LSS)xfJ?GX9cc8h-q#B~=)7S$W}?6;7$B#QK4Snb~SX zfO-e3;4BC?0}PXn79ZET!}h{Ko<sPDtCR)|TILrvrlc~$;zzg|<j~4q2PGoL$_J?` zb;4Ln8MKs`=vJ0+(A;QtG3HWW3WUyW>nRogY@HoiYEB>(qdP1!2eXT`5QKK9HDOjB zex-?WeeMD$oCFvolwv1DUC3y(3M)d4yLZSJ7S``Y<3^*s6k%EA!D$O>fx7a}VrLpH z@i4kq0z5E`!Lw;fD%=QM&a2W`dLP?x7!1O!(4L5lj{|B>2cmzp&*Nur`U`Nv=<w9G z2xLmH70Y&#f(c!Yhv@rPz90>&ye^t}J(VHt+@R&H%GB2SRf)f$(!_=&|Jp^}Kq4rg zAZKvU;sC9eLI;OIWb5IfUr$PM^r}i?W&cJf8?0b%pdI#Kml*Im9Mn~b(uo-2uruVf zU+y~0*wr=6cZUi#3j1?98Bqyhg4tbP6=u=VK~ez`_hX=%gcmEE_Yuab%TohwmxI|- zaz=*P3J|JCpx7p0r5e^TVG_P+Zc80@DTL`wuncz|4X$YUi3_`BF=rIY(0%xO4b^ra zDnc|NDTU!z6a>k*6yivugD}gwgJMIRu(a+MF6@@H1~t!!4o=8mux{}Ck91>|>;0s! zL8Kfs<`RBMA&g=B2<5`f{COK=iI=i4!E2ea%g{mf>#rN^_FDAM)Pwe0m`R<skd7dm zrQZl8*wV=)OUF5jc8KU5IoPg3`*FnjNai}N{z7<STr)9CQAfgv6~8$0JwUZV&Hn85 z8j1l~v^*MH;AuvonFd|LULs$%D3;MO=vOMam`hGhz}%oE{+Z*+G{NQwx!VU@O~hKT z@Ro<D0@=4rq&2B>qs!0sMWVg`lCg8YRvdBp`N11G3m*4VQXz~Mtw60Ms=T13rU@bu zn2;i==(o*XOrc5%H(Cz{jT3BQVpICwO-$heHgKqPmS_)qsaNbzHf)QK9sK?s7Z8y1 zASbpqeRR{`?;Z7q^a@G1)V%+MLZR#Ju$;A~s)268LRpFsMz-^aOG3CI))DP#s?uNG zs>}@R-rD<ibt=mKnm+y3)*9+t_G=Ma!~t!{K;K=5T}jOzjG}1sA5o}lU@vy$!S(%z zGY%pQeL$d+xnJkd4cQlj?+J8271Zg9mT+*avCuuMfwmHPS-pSZc9=O)#1CFgu>VdD zdyv}G-s7KT$;#~`QkL@67RZLC@V1K5Y=*U)y732MW=>EmeEoxPIT|%IkU4W`z#U|B zMN|<a@6s(2ZrDccHE3)X#%Ar`WDKNe151brOK)CShw~-?JxvnM)4%E)a6k=a7t`mo z)YMiTS5w8r8y)4q(yJEMBc_C&Ci#VSePBaj(2EP-us*GVBB)=DwH}sEZZDROpXqZO z@B{OJ_+RYRO~<Sw_G_?o_dK;pcwQ9}(Qin-63ZaP8g$tDSfVHslNN<cL;t39D0d_& zD6OcgsYLt+4`mFD$YxG+Gs9l&I<o)O`9!}I3`YWVG?+T}$$!6&zGo*jBm-;6JDsPs z3wOHd_~q({hA*iI>KHu#{R27~9Dm1R@TcSJ`+79YA1XsqGIYXIw0!m7KbMbpN?XU* z_xfn4@5xiqL>l@xK?nZm&N>}m-~XfGPI;`9$b-V`a2lQl9NmGZ-}OB~en^q<5d9o~ zntsC3Q-weJrsM1Tgf!Im?uGI%g(Bmr#~;O?OfPu!J$@SENGtJBd}5`}U&mhn92GGN zf8uEjhKl_PDf9Bb;72q5jPdn7Mj9sS3{@n39-oGng~ZqQ9cdU6-A}*9UlkHx-^-)n zDwnFjko?sv>kNsn@59hg9ZRNz3tszQ@Z)+FU)P`io~2%yuT39Zsv_AG)Zk|PppZJg zzK==6^}6C!1gdM}b2na$@%6n;8m>6Z7&=h@8v9^Ke0{%@hIi=$J5_|x{5Jz1k3UVO zz6YxHFj*okXDWKX`lIvj=qyPcU*8wCO2yap94Y?0_(0{?@%8z>t5y7MvKn-BfBqWr zAN(NM*YWlJQ}aeh`t|ivy4;!`Ex%VHt}%aoPt~d;Wc+y%^4IBn6%yZnpOsDVPiKs9 z^yqjR{s^2ge|_ItN~(;n=@gL-@%6hIktr=*etnP2SQX!<11hNV)#w3;Px)tPgn~N0 zKJQY0rm*xFp`=ve_5Ti)fAmO+p>$^9Pv@`YirS~9U51qZm(m4Qc#@~ecm2Q4SHII) RfZ^iLxmxirBtpn5`#)N@cp(4) literal 0 HcmV?d00001 diff --git a/src/main/cpp/systemds.cpp b/src/main/cpp/systemds.cpp index bed1d42f578..86ac053ad10 100644 --- a/src/main/cpp/systemds.cpp +++ b/src/main/cpp/systemds.cpp @@ -17,6 +17,12 @@ * under the License. */ +#ifdef _WIN32 +#include <winsock.h> +#else +#include <arpa/inet.h> +#endif + #include "common.h" #include "libmatrixdnn.h" #include "libmatrixmult.h" @@ -248,4 +254,4 @@ JNIEXPORT jlong JNICALL Java_org_apache_sysds_utils_NativeHelper_conv2dBackwardF RELEASE_INPUT_ARRAY(env, dout, doutPtr, numThreads); RELEASE_ARRAY(env, ret, retPtr, numThreads); return static_cast<jlong>(nnz); -} +} \ No newline at end of file diff --git a/src/main/java/org/apache/sysds/common/Types.java b/src/main/java/org/apache/sysds/common/Types.java index b6ef98abc72..3dfad3413e5 100644 --- a/src/main/java/org/apache/sysds/common/Types.java +++ b/src/main/java/org/apache/sysds/common/Types.java @@ -44,7 +44,7 @@ public enum ExecType { CP, CP_FILE, SPARK, GPU, FED, INVALID } * Data types (tensor, matrix, scalar, frame, object, unknown). */ public enum DataType { - TENSOR, MATRIX, SCALAR, FRAME, LIST, UNKNOWN; + TENSOR, MATRIX, SCALAR, FRAME, LIST, ENCRYPTED_CIPHER, ENCRYPTED_PLAIN, UNKNOWN; public boolean isMatrix() { return this == MATRIX; diff --git a/src/main/java/org/apache/sysds/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysds/parser/ParameterizedBuiltinFunctionExpression.java index a6bcc2dac44..56275fd3b93 100644 --- a/src/main/java/org/apache/sysds/parser/ParameterizedBuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysds/parser/ParameterizedBuiltinFunctionExpression.java @@ -320,7 +320,8 @@ private void validateParamserv(DataIdentifier output, boolean conditional) { Statement.PS_VAL_FEATURES, Statement.PS_VAL_LABELS, Statement.PS_UPDATE_FUN, Statement.PS_AGGREGATION_FUN, Statement.PS_VAL_FUN, Statement.PS_MODE, Statement.PS_UPDATE_TYPE, Statement.PS_FREQUENCY, Statement.PS_EPOCHS, Statement.PS_BATCH_SIZE, Statement.PS_PARALLELISM, Statement.PS_SCHEME, Statement.PS_FED_RUNTIME_BALANCING, - Statement.PS_FED_WEIGHTING, Statement.PS_HYPER_PARAMS, Statement.PS_CHECKPOINTING, Statement.PS_SEED, Statement.PS_NBATCHES, Statement.PS_MODELAVG); + Statement.PS_FED_WEIGHTING, Statement.PS_HYPER_PARAMS, Statement.PS_CHECKPOINTING, Statement.PS_SEED, Statement.PS_NBATCHES, + Statement.PS_MODELAVG, Statement.PS_HE); checkInvalidParameters(getOpCode(), getVarParams(), valid); // check existence and correctness of parameters diff --git a/src/main/java/org/apache/sysds/parser/Statement.java b/src/main/java/org/apache/sysds/parser/Statement.java index 995a1e23309..d22a5401806 100644 --- a/src/main/java/org/apache/sysds/parser/Statement.java +++ b/src/main/java/org/apache/sysds/parser/Statement.java @@ -76,6 +76,7 @@ public abstract class Statement implements ParseInfo public static final String PS_SEED = "seed"; public static final String PS_MODELAVG = "modelAvg"; public static final String PS_NBATCHES = "nbatches"; + public static final String PS_HE = "he"; public enum PSModeType { FEDERATED, LOCAL, REMOTE_SPARK } @@ -124,7 +125,6 @@ public enum PSCheckpointing { public static final String PS_FED_AGGREGATION_FNAME = "1701-NCC-aggregation_fname"; public static final String PS_FED_MODEL_VARID = "1701-NCC-model_varid"; - public abstract boolean controlStatement(); public abstract VariableSet variablesRead(); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java index e398abd32b0..e211bfeb471 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java @@ -38,6 +38,7 @@ import org.apache.sysds.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysds.runtime.controlprogram.caching.TensorObject; import org.apache.sysds.runtime.controlprogram.federated.MatrixLineagePair; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.SEALClient; import org.apache.sysds.runtime.data.TensorBlock; import org.apache.sysds.runtime.instructions.Instruction; import org.apache.sysds.runtime.instructions.cp.CPOperand; @@ -82,9 +83,11 @@ public class ExecutionContext { //lineage map, cache, prepared dedup blocks protected Lineage _lineage; + protected SEALClient _seal_client; + //parfor temporary functions (created by eval) protected Set<String> _fnNames; - + /** * List of {@link GPUContext}s owned by this {@link ExecutionContext} */ @@ -152,6 +155,14 @@ public long getTID() { return _tid; } + public void setSealClient(SEALClient seal_client) { + _seal_client = seal_client; + } + + public SEALClient getSealClient() { + return _seal_client; + } + /** * Get the i-th GPUContext * @param index index of the GPUContext @@ -891,11 +902,11 @@ public LineageItem getOrCreateLineageItem(CPOperand input) { private static String getNonExistingVarError(String varname) { return "Variable '" + varname + "' does not exist in the symbol table."; } - + public void addTmpParforFunction(String fname) { _fnNames.add(fname); } - + public Set<String> getTmpParforFunctions() { return _fnNames; } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java index 74e113ba020..1fb1e8b1ecd 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java @@ -35,6 +35,7 @@ import org.apache.sysds.conf.ConfigurationManager; import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType; +import org.apache.sysds.runtime.controlprogram.paramserv.NetworkTrafficCounter; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.meta.MetaData; @@ -193,6 +194,7 @@ private static ChannelInitializer<SocketChannel> createChannel(InetSocketAddress @Override protected void initChannel(SocketChannel ch) throws Exception { final ChannelPipeline cp = ch.pipeline(); + cp.addLast("NetworkTrafficCounter", new NetworkTrafficCounter(FederatedStatistics::logServerTraffic)); if(ssl) cp.addLast(createSSLHandler(ch, address)); if(timeout > -1) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedLocalData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedLocalData.java index de56a1a52e0..77ffb7f847f 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedLocalData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedLocalData.java @@ -25,6 +25,7 @@ import org.apache.log4j.Logger; import org.apache.sysds.conf.ConfigurationManager; import org.apache.sysds.runtime.controlprogram.caching.CacheableData; +import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing; import org.apache.sysds.runtime.controlprogram.parfor.util.IDHandler; public class FederatedLocalData extends FederatedData { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java index d95b02afd21..5907776898d 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java @@ -73,6 +73,8 @@ public class FederatedStatistics { private static final LongAdder transferredMatrixBytes = new LongAdder(); private static final LongAdder transferredFrameBytes = new LongAdder(); private static final LongAdder asyncPrefetchCount = new LongAdder(); + private static final LongAdder bytesSent = new LongAdder(); + private static final LongAdder bytesReceived = new LongAdder(); // stats on the federated worker itself private static final LongAdder fedLookupTableGetCount = new LongAdder(); @@ -80,11 +82,25 @@ public class FederatedStatistics { private static final LongAdder fedLookupTableEntryCount = new LongAdder(); private static final LongAdder fedReuseReadHitCount = new LongAdder(); private static final LongAdder fedReuseReadBytesCount = new LongAdder(); + private static final LongAdder fedBytesSent = new LongAdder(); + private static final LongAdder fedBytesReceived = new LongAdder(); + private static final LongAdder fedPutLineageCount = new LongAdder(); private static final LongAdder fedPutLineageItems = new LongAdder(); private static final LongAdder fedSerializationReuseCount = new LongAdder(); private static final LongAdder fedSerializationReuseBytes = new LongAdder(); + public static void logServerTraffic(long read, long written) { + bytesReceived.add(read); + bytesSent.add(written); + } + + public static void logWorkerTraffic(long read, long written) { + fedBytesReceived.add(read); + fedBytesSent.add(written); + } + + public static synchronized void incFederated(RequestType rqt, List<Object> data){ switch (rqt) { case READ_VAR: @@ -164,6 +180,10 @@ public static void reset() { fedPutLineageItems.reset(); fedSerializationReuseCount.reset(); fedSerializationReuseBytes.reset(); + bytesSent.reset(); + bytesReceived.reset(); + fedBytesSent.reset(); + fedBytesReceived.reset(); } public static String displayFedIOExecStatistics() { @@ -194,6 +214,19 @@ public static String displayFedIOExecStatistics() { return ""; } + public static String displayNetworkTrafficStatistics() { + return "Server I/O bytes (read/written):\t" + + bytesReceived.longValue() + + "/" + + bytesSent.longValue() + + "\n" + + "Worker I/O bytes (read/written):\t" + + fedBytesReceived.longValue() + + "/" + + fedBytesSent.longValue() + + "\n"; + } + public static void registerFedWorker(String host, int port) { _fedWorkerAddresses.add(new ImmutablePair<>(host, Integer.valueOf(port))); @@ -232,6 +265,7 @@ public static String displayStatistics(FedStatsCollection fedStats, int numHeavy sb.append(displayLinCacheStats(fedStats.linCacheStats)); sb.append(displayMultiTenantStats(fedStats.mtStats)); sb.append(displayHeavyHitters(fedStats.heavyHitters, numHeavyHitters)); + sb.append(displayNetworkTrafficStatistics()); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index a41f656524d..cf7f80478ab 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -33,6 +33,7 @@ import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.controlprogram.caching.CacheBlock; +import org.apache.sysds.runtime.controlprogram.paramserv.NetworkTrafficCounter; import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import org.apache.sysds.runtime.lineage.LineageCache; import org.apache.sysds.runtime.lineage.LineageCacheConfig; @@ -53,6 +54,9 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.SelfSignedCertificate; +import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing; +import io.netty.handler.codec.serialization.ObjectDecoder; +import io.netty.handler.codec.serialization.ClassResolvers; public class FederatedWorker { protected static Logger log = Logger.getLogger(FederatedWorker.class); @@ -62,6 +66,7 @@ public class FederatedWorker { private final FederatedReadCache _frc; private final FederatedWorkloadAnalyzer _fan; private final boolean _debug; + private Timing networkTimer = new Timing(); public FederatedWorker(int port, boolean debug) { _flt = new FederatedLookupTable(); @@ -183,8 +188,17 @@ private ChannelInitializer<SocketChannel> createChannel(boolean ssl) { @Override public void initChannel(SocketChannel ch) { final ChannelPipeline cp = ch.pipeline(); + if(ConfigurationManager.getDMLConfig() + .getBooleanValue(DMLConfig.USE_SSL_FEDERATED_COMMUNICATION)) { + cp.addLast(cont2.newHandler(ch.alloc())); + } if(ssl) cp.addLast(cont2.newHandler(ch.alloc())); + cp.addLast("NetworkTrafficCounter", new NetworkTrafficCounter(FederatedStatistics::logWorkerTraffic)); + cp.addLast("ObjectDecoder", + new ObjectDecoder(Integer.MAX_VALUE, + ClassResolvers.weakCachingResolver(ClassLoader.getSystemClassLoader()))); + cp.addLast("ObjectEncoder", new ObjectEncoder()); cp.addLast(FederationUtils.decoder(), new FederatedResponseEncoder()); cp.addLast(new FederatedWorkerHandler(_flt, _frc, _fan)); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java index 4c90c74b1b9..5fab13afcf4 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java @@ -49,6 +49,7 @@ import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType; import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse.ResponseType; +import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing; import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import org.apache.sysds.runtime.instructions.Instruction; import org.apache.sysds.runtime.instructions.Instruction.IType; @@ -73,6 +74,7 @@ import org.apache.sysds.runtime.privacy.DMLPrivacyException; import org.apache.sysds.runtime.privacy.PrivacyMonitor; import org.apache.sysds.utils.Statistics; +import org.apache.sysds.utils.stats.ParamServStatistics; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; @@ -87,13 +89,15 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter { private static final Log LOG = LogFactory.getLog(FederatedWorkerHandler.class.getName()); /** The Federated Lookup Table of the current Federated Worker. */ - private final FederatedLookupTable _flt; + private FederatedLookupTable _flt; /** Read cache shared by all worker handlers */ - private final FederatedReadCache _frc; + private FederatedReadCache _frc; + private Timing _timing = null; + /** Federated workload analyzer */ - private final FederatedWorkloadAnalyzer _fan; + private FederatedWorkloadAnalyzer _fan; /** * Create a Federated Worker Handler. @@ -111,6 +115,16 @@ public FederatedWorkerHandler(FederatedLookupTable flt, FederatedReadCache frc, _fan = fan; } + public FederatedWorkerHandler(FederatedLookupTable flt, FederatedReadCache frc){ + _flt = flt; + _frc = frc; + } + + public FederatedWorkerHandler(FederatedLookupTable flt, FederatedReadCache frc, Timing timing) { + this(flt, frc); + _timing = timing; + } + @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { ctx.writeAndFlush(createResponse(msg, ctx.channel().remoteAddress())) @@ -122,6 +136,14 @@ protected FederatedResponse createResponse(Object msg) { } private FederatedResponse createResponse(Object msg, SocketAddress remoteAddress) { + try { + if (_timing != null) { + ParamServStatistics.accFedNetworkTime((long) _timing.stop()); + } + } catch (RuntimeException ignored) { + // ignore timing if it wasn't started yet + } + String host; if(remoteAddress instanceof InetSocketAddress) { host = ((InetSocketAddress) remoteAddress).getHostString(); @@ -135,7 +157,11 @@ else if(remoteAddress instanceof SocketAddress) { host = FederatedLookupTable.NOHOST; } - return createResponse(msg, host); + FederatedResponse res = createResponse(msg, host); + if (_timing != null) { + _timing.start(); + } + return res; } private FederatedResponse createResponse(Object msg, String remoteHost) { @@ -162,7 +188,7 @@ private FederatedResponse createResponse(Object msg, String remoteHost) { private FederatedResponse createResponse(FederatedRequest[] requests, String remoteHost) throws DMLPrivacyException, FederatedWorkerHandlerException, Exception { - + FederatedResponse response = null; // last response boolean containsCLEAR = false; for(int i = 0; i < requests.length; i++) { @@ -294,7 +320,7 @@ private FederatedResponse readData(String filename, DataType dataType, throw ex; } } - + if(shouldTryAsyncCompress()) // TODO: replace the reused object CompressedMatrixBlockFactory.compressAsync(ec, sId); @@ -405,7 +431,7 @@ else if(request.getNumParams() == 2){ throw new FederatedWorkerHandlerException( "Unsupported object type, has to be of type CacheBlock or ScalarObject"); - + // set variable and construct empty response ec.setVariable(varName, data); @@ -429,13 +455,12 @@ else if(request.getNumParams()==1) // don't trace if the data contains only meta private FederatedResponse getVariable(FederatedRequest request, ExecutionContextMap ecm) { try{ - checkNumParams(request.getNumParams(), 0); ExecutionContext ec = ecm.get(request.getTID()); if(!ec.containsVariable(String.valueOf(request.getID()))) throw new FederatedWorkerHandlerException( "Variable " + request.getID() + " does not exist at federated worker."); - + // get variable and construct response Data dataObject = ec.getVariable(String.valueOf(request.getID())); dataObject = PrivacyMonitor.handlePrivacy(dataObject); @@ -467,7 +492,7 @@ private FederatedResponse execInstruction(FederatedRequest request, ExecutionCon adaptToWorkload(ec, _fan, tid, ins); return new FederatedResponse(ResponseType.SUCCESS_EMPTY); } - + private static ExecutionContext getContextForInstruction(long id, Instruction ins, ExecutionContextMap ecm){ final ExecutionContext ec = ecm.get(id); //handle missing spark execution context diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java index 004e35b5718..54d778486a7 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java @@ -34,24 +34,15 @@ import org.apache.sysds.runtime.controlprogram.ProgramBlock; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; -import org.apache.sysds.runtime.controlprogram.federated.FederatedData; -import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest; +import org.apache.sysds.runtime.controlprogram.federated.*; import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType; -import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse; -import org.apache.sysds.runtime.controlprogram.federated.FederatedUDF; -import org.apache.sysds.runtime.controlprogram.federated.FederationUtils; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.PublicKey; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.SEALClient; import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing; import org.apache.sysds.runtime.functionobjects.Multiply; import org.apache.sysds.runtime.instructions.Instruction; import org.apache.sysds.runtime.instructions.InstructionUtils; -import org.apache.sysds.runtime.instructions.cp.BooleanObject; -import org.apache.sysds.runtime.instructions.cp.CPOperand; -import org.apache.sysds.runtime.instructions.cp.Data; -import org.apache.sysds.runtime.instructions.cp.DoubleObject; -import org.apache.sysds.runtime.instructions.cp.FunctionCallCPInstruction; -import org.apache.sysds.runtime.instructions.cp.IntObject; -import org.apache.sysds.runtime.instructions.cp.ListObject; -import org.apache.sysds.runtime.instructions.cp.StringObject; +import org.apache.sysds.runtime.instructions.cp.*; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.operators.RightScalarOperator; import org.apache.sysds.runtime.lineage.LineageItem; @@ -59,11 +50,13 @@ import org.apache.sysds.utils.stats.ParamServStatistics; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.concurrent.Callable; import java.util.concurrent.Future; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.apache.sysds.runtime.util.ProgramConverter.*; @@ -83,20 +76,23 @@ public class FederatedPSControlThread extends PSWorker implements Callable<Void> private final boolean _weighting; private double _weightingFactor = 1; private boolean _cycleStartAt0 = false; + private boolean _use_homomorphic_encryption = false; + private PublicKey _partial_public_key; public FederatedPSControlThread(int workerID, String updFunc, Statement.PSFrequency freq, PSRuntimeBalancing runtimeBalancing, boolean weighting, int epochs, long batchSize, - int numBatchesPerGlobalEpoch, ExecutionContext ec, ParamServer ps, int nbatches, boolean modelAvg) + int numBatchesPerGlobalEpoch, ExecutionContext ec, ParamServer ps, int nbatches, boolean modelAvg, boolean use_homomorphic_encryption) { super(workerID, updFunc, freq, epochs, batchSize, ec, ps, nbatches, modelAvg); _numBatchesPerEpoch = numBatchesPerGlobalEpoch; _runtimeBalancing = runtimeBalancing; - _weighting = weighting; + _weighting = weighting && (!use_homomorphic_encryption); // FIXME: this disables weighting in favor of homomorphic encryption _numBatchesPerNbatch = nbatches; // generate the ID for the model _modelVarID = FederationUtils.getNextFedDataID(); - _modelAvg = modelAvg; + _modelAvg = _use_homomorphic_encryption || modelAvg; // we always have to use modelAvg when using homomorphic encryption + _use_homomorphic_encryption = use_homomorphic_encryption; } /** @@ -106,6 +102,9 @@ public FederatedPSControlThread(int workerID, String updFunc, Statement.PSFreque */ public void setup(double weightingFactor) { incWorkerNumber(); + if (_use_homomorphic_encryption) { + ((HEParamServer)_ps).registerThread(_workerID, this); + } // prepare features and labels _featuresData = _features.getFedMapping().getFederatedData()[0]; @@ -160,21 +159,43 @@ public void setup(double weightingFactor) { PROG_END); // write program and meta data to worker - Future<FederatedResponse> udfResponse = _featuresData.executeFederatedOperation( - new FederatedRequest(RequestType.EXEC_UDF, _featuresData.getVarID(), - new SetupFederatedWorker(_batchSize, dataSize, _possibleBatchesPerLocalEpoch, - programSerialized, _inst.getNamespace(), _inst.getFunctionName(), - _ps.getAggInst().getFunctionName(), _ec.getListObject("hyperparams"), - _modelVarID, _nbatches, _modelAvg))); + Future<FederatedResponse> udfResponse; + + final SetupFederatedWorker udf; + if (_use_homomorphic_encryption) { + byte[] a = ((HEParamServer)_ps).generateA(); + // generate pk[i] on each client and return it + udf = new SetupHEFederatedWorker(a); + } else { + udf = new SetupFederatedWorker(); + } + udf.setParams(_batchSize, dataSize, _possibleBatchesPerLocalEpoch, + programSerialized, _inst.getNamespace(), _inst.getFunctionName(), + _ps.getAggInst().getFunctionName(), _ec.getListObject("hyperparams"), + _modelVarID, _nbatches, _use_homomorphic_encryption || _modelAvg); + + udfResponse = _featuresData.executeFederatedOperation( + new FederatedRequest(RequestType.EXEC_UDF, _featuresData.getVarID(), udf)); + + FederatedResponse response; try { - FederatedResponse response = udfResponse.get(); + response = udfResponse.get(); if(!response.isSuccessful()) throw new DMLRuntimeException("FederatedLocalPSThread: Setup UDF failed"); + } catch(Exception e) { throw new DMLRuntimeException("FederatedLocalPSThread: failed to execute Setup UDF" + e.getMessage()); } + if (_use_homomorphic_encryption) { + try { + _partial_public_key = (PublicKey) response.getData()[0]; + } + catch (Exception e) { + throw new DMLRuntimeException("FederatedLocalPSThread: HE Setup UDF didn't return an object"); + } + } } /** @@ -196,29 +217,33 @@ public void teardown() { throw new DMLRuntimeException("FederatedLocalPSThread: failed to execute Teardown UDF" + e.getMessage()); } } - + /** * Setup UDF executed on the federated worker */ private static class SetupFederatedWorker extends FederatedUDF { private static final long serialVersionUID = -3148991224792675607L; - private final long _batchSize; - private final long _dataSize; - private final int _possibleBatchesPerLocalEpoch; - private final String _programString; - private final String _namespace; - private final String _gradientsFunctionName; - private final String _aggregationFunctionName; - private final ListObject _hyperParams; - private final long _modelVarID; - private final boolean _modelAvg; - private final int _nbatches; - - protected SetupFederatedWorker(long batchSize, long dataSize, int possibleBatchesPerLocalEpoch, - String programString, String namespace, String gradientsFunctionName, String aggregationFunctionName, - ListObject hyperParams, long modelVarID, int nbatches, boolean modelAvg) + private long _batchSize; + private long _dataSize; + private int _possibleBatchesPerLocalEpoch; + private String _programString; + private String _namespace; + private String _gradientsFunctionName; + private String _aggregationFunctionName; + private ListObject _hyperParams; + private long _modelVarID; + private boolean _modelAvg; + private int _nbatches; + private boolean _params_set = false; + + protected SetupFederatedWorker() { super(new long[]{}); + } + + public void setParams(long batchSize, long dataSize, int possibleBatchesPerLocalEpoch, + String programString, String namespace, String gradientsFunctionName, String aggregationFunctionName, + ListObject hyperParams, long modelVarID, int nbatches, boolean modelAvg) { _batchSize = batchSize; _dataSize = dataSize; _possibleBatchesPerLocalEpoch = possibleBatchesPerLocalEpoch; @@ -230,10 +255,15 @@ protected SetupFederatedWorker(long batchSize, long dataSize, int possibleBatche _modelVarID = modelVarID; _modelAvg = modelAvg; _nbatches = nbatches; + _params_set = true; } @Override public FederatedResponse execute(ExecutionContext ec, Data... data) { + if (!_params_set) { + return new FederatedResponse(FederatedResponse.ResponseType.ERROR, "params were not set"); + } + // parse and set program ec.setProgram(ProgramConverter.parseProgram(_programString, 0)); @@ -258,9 +288,59 @@ public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) { } } - /** + private static class SetupHEFederatedWorker extends SetupFederatedWorker { + private static final long serialVersionUID = 9128347291804980123L; + + byte[] _partial_pubkey_a; + + protected SetupHEFederatedWorker(byte[] partial_pubkey_a) { + // delegate everything to parent class. set modelAvg to true, as it is the only supported case + super(); + _partial_pubkey_a = partial_pubkey_a; + } + + @Override + public FederatedResponse execute(ExecutionContext ec, Data... data) { + // TODO: set other CKKS parameters + // TODO generate partial public key + NativeHEHelper.initialize(); + + SEALClient sc = new SEALClient(_partial_pubkey_a); + ec.setSealClient(sc); + PublicKey partial_pubkey = sc.generatePartialPublicKey(); + + FederatedResponse res = super.execute(ec, data); + if (!res.isSuccessful()) { + return res; + } + + return new FederatedResponse(FederatedResponse.ResponseType.SUCCESS, partial_pubkey); + } + } + /** * Teardown UDF executed on the federated worker */ + private static class SetPublicKeyFederatedWorker extends FederatedUDF { + private static final long serialVersionUID = -1536502123123318969L; + private final PublicKey _public_key; + + protected SetPublicKeyFederatedWorker(PublicKey public_key) { + super(new long[]{}); + _public_key = public_key; + } + + @Override + public FederatedResponse execute(ExecutionContext ec, Data... data) { + ec.getSealClient().setPublicKey(_public_key); + return new FederatedResponse(FederatedResponse.ResponseType.SUCCESS); + } + + @Override + public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) { + return null; + } + } + private static class TeardownFederatedWorker extends FederatedUDF { private static final long serialVersionUID = -153650281873318969L; @@ -298,6 +378,7 @@ public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) { @Override public Void call() throws Exception { try { + Timing tTotal = new Timing(true); switch (_freq) { case BATCH: computeWithBatchUpdates(); @@ -324,6 +405,7 @@ protected ListObject pullModel() { } protected void weightAndPushGradients(ListObject gradients) { + assert (!(_weighting && _use_homomorphic_encryption)) : "weights and homomorphic encryption are not supported together"; // scale gradients - must only include MatrixObjects if(_weighting && _weightingFactor != 1) { Timing tWeighting = DMLScript.STATISTICS ? new Timing(true) : null; @@ -354,11 +436,17 @@ protected void computeWithBatchUpdates() { int localStartBatchNum = getNextLocalBatchNum(currentLocalBatchNumber++, _possibleBatchesPerLocalEpoch); ListObject model = pullModel(); ListObject gradients = computeGradientsForNBatches(model, 1, localStartBatchNum); - if (_modelAvg) + + Timing tAgg = DMLScript.STATISTICS ? new Timing(true) : null; + if (_modelAvg && !_use_homomorphic_encryption) + // we can't call the agg fn if we use HE, because it is implemented homomorphically in SEALServer::aggregateCiphertexts model = _ps.updateLocalModel(_ec, gradients, model); else ParamservUtils.cleanupListObject(model); - weightAndPushGradients(_modelAvg ? model : gradients); + weightAndPushGradients((_modelAvg && !_use_homomorphic_encryption) ? model : gradients); + if (tAgg != null) { + ParamServStatistics.accFedAggregation((long)tAgg.stop()); + } ParamservUtils.cleanupListObject(gradients); } } @@ -377,7 +465,13 @@ protected void computeWithNBatchUpdates() { currentLocalBatchNumber = currentLocalBatchNumber + _numBatchesPerNbatch; ListObject model = pullModel(); ListObject gradients = computeGradientsForNBatches(model, _numBatchesPerNbatch, localStartBatchNum, true); + + Timing tAgg = DMLScript.STATISTICS ? new Timing(true) : null; weightAndPushGradients(gradients); + if (tAgg != null) { + ParamServStatistics.accFedAggregation((long)tAgg.stop()); + } + ParamservUtils.cleanupListObject(model); ParamservUtils.cleanupListObject(gradients); } @@ -394,7 +488,13 @@ protected void computeWithEpochUpdates() { // Pull the global parameters from ps ListObject model = pullModel(); ListObject gradients = computeGradientsForNBatches(model, _numBatchesPerEpoch, localStartBatchNum, true); + + Timing tAgg = DMLScript.STATISTICS ? new Timing(true) : null; weightAndPushGradients(gradients); + if (tAgg != null) { + ParamServStatistics.accFedAggregation((long)tAgg.stop()); + } + ParamservUtils.cleanupListObject(model); ParamservUtils.cleanupListObject(gradients); } @@ -431,11 +531,16 @@ protected ListObject computeGradientsForNBatches(ListObject model, } // create and execute the udf on the remote worker + Object udf; + if (_use_homomorphic_encryption) { + udf = new HEComputeGradientsForNBatches(new long[]{_featuresData.getVarID(), _labelsData.getVarID()}, + new long[]{_modelVarID}, numBatchesToCompute, localUpdate, localStartBatchNum); + } else { + udf = new federatedComputeGradientsForNBatches(new long[]{_featuresData.getVarID(), _labelsData.getVarID(), + _modelVarID}, numBatchesToCompute, localUpdate, localStartBatchNum); + } Future<FederatedResponse> udfResponse = _featuresData.executeFederatedOperation( - new FederatedRequest(RequestType.EXEC_UDF, _featuresData.getVarID(), - new federatedComputeGradientsForNBatches(new long[]{_featuresData.getVarID(), _labelsData.getVarID(), - _modelVarID}, numBatchesToCompute, localUpdate, localStartBatchNum) - )); + new FederatedRequest(RequestType.EXEC_UDF, _featuresData.getVarID(), udf)); try { Object[] responseData = udfResponse.get().getData(); @@ -444,6 +549,7 @@ protected ListObject computeGradientsForNBatches(ListObject model, long workerComputing = ((DoubleObject) responseData[1]).getLongValue(); ParamServStatistics.accFedWorkerComputing(workerComputing); ParamServStatistics.accFedCommunicationTime(total - workerComputing); + ParamServStatistics.accFedNetworkTime(total); } return (ListObject) responseData[0]; } @@ -492,12 +598,12 @@ public FederatedResponse execute(ExecutionContext ec, Data... data) { ArrayList<DataIdentifier> inputs = func.getInputParams(); ArrayList<DataIdentifier> outputs = func.getOutputParams(); CPOperand[] boundInputs = inputs.stream() - .map(input -> new CPOperand(input.getName(), input.getValueType(), input.getDataType())) - .toArray(CPOperand[]::new); + .map(input -> new CPOperand(input.getName(), input.getValueType(), input.getDataType())) + .toArray(CPOperand[]::new); ArrayList<String> outputNames = outputs.stream().map(DataIdentifier::getName) - .collect(Collectors.toCollection(ArrayList::new)); + .collect(Collectors.toCollection(ArrayList::new)); Instruction gradientsInstruction = new FunctionCallCPInstruction(namespace, gradientsFunc, - opt, boundInputs, func.getInputParamNames(), outputNames, "gradient function"); + opt, boundInputs, func.getInputParamNames(), outputNames, "gradient function"); DataIdentifier gradientsOutput = outputs.get(0); // recreate aggregation instruction and output if needed @@ -508,12 +614,12 @@ public FederatedResponse execute(ExecutionContext ec, Data... data) { inputs = func.getInputParams(); outputs = func.getOutputParams(); boundInputs = inputs.stream() - .map(input -> new CPOperand(input.getName(), input.getValueType(), input.getDataType())) - .toArray(CPOperand[]::new); + .map(input -> new CPOperand(input.getName(), input.getValueType(), input.getDataType())) + .toArray(CPOperand[]::new); outputNames = outputs.stream().map(DataIdentifier::getName) - .collect(Collectors.toCollection(ArrayList::new)); + .collect(Collectors.toCollection(ArrayList::new)); aggregationInstruction = new FunctionCallCPInstruction(namespace, aggFunc, - opt, boundInputs, func.getInputParamNames(), outputNames, "aggregation function"); + opt, boundInputs, func.getInputParamNames(), outputNames, "aggregation function"); aggregationOutput = outputs.get(0); } ListObject accGradients = null; @@ -540,7 +646,7 @@ public FederatedResponse execute(ExecutionContext ec, Data... data) { // accrue the computed gradients - In the single batch case this is just a list copy // is this equivalent for momentum based and AMS prob? accGradients = modelAvg ? null : - ParamservUtils.accrueGradients(accGradients, gradients, false); + ParamservUtils.accrueGradients(accGradients, gradients, false); // update the local model with gradients if needed // FIXME ensure that with modelAvg we always update the model @@ -564,11 +670,12 @@ public FederatedResponse execute(ExecutionContext ec, Data... data) { // model clean up ParamservUtils.cleanupListObject(ec, ec.getVariable(Statement.PS_FED_MODEL_VARID).toString()); // TODO double check cleanup gradients and models - + // stop timing DoubleObject gradientsTime = new DoubleObject(tGradients.stop()); + ParamServStatistics.accGradientComputeTime(gradientsTime.getLongValue()); return new FederatedResponse(FederatedResponse.ResponseType.SUCCESS, - new Object[]{modelAvg ? model : accGradients, gradientsTime}); + new Object[]{modelAvg ? model : accGradients, gradientsTime}); } @Override @@ -577,6 +684,102 @@ public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) { } } + + /** + * This wraps federatedComputeGradientsForNBatches and adds encryption + */ + private static class HEComputeGradientsForNBatches extends federatedComputeGradientsForNBatches { + private static final long serialVersionUID = -3535901512348794852L; + private final long[] _deferredIds; + + protected HEComputeGradientsForNBatches(long[] deferredIds, long[] inIDs, int numBatchesToCompute, boolean localUpdate, int localStartBatchNum) { + super(inIDs, numBatchesToCompute, localUpdate, localStartBatchNum); + _deferredIds = deferredIds; + } + + @Override + public FederatedResponse execute(ExecutionContext ec, Data... data_without_deferred) { + Timing tTotal = new Timing(true); + // add features and gradients to data + Data[] deferred_inputs = Arrays.stream(_deferredIds).mapToObj(id -> ec.getVariable(String.valueOf(id))).toArray(Data[]::new); + Data[] data = Arrays.copyOf(deferred_inputs, deferred_inputs.length + data_without_deferred.length); + System.arraycopy(data_without_deferred, 0, data, deferred_inputs.length, data_without_deferred.length); + FederatedResponse res = super.execute(ec, data); + + if (!res.isSuccessful()) { + return res; + } + + // encrypt model with SEAL + try { + Timing tEncrypt = DMLScript.STATISTICS ? new Timing(true) : null; + + ListObject model = (ListObject) res.getData()[0]; + ListObject encrypted_model = new ListObject(model); + IntStream.range(0, model.getLength()).forEach(matrix_idx -> { + CiphertextMatrix encrypted_matrix = ec.getSealClient().encrypt((MatrixObject) model.getData(matrix_idx)); + encrypted_model.set(matrix_idx, encrypted_matrix); + }); + + // overwrite model with encryption + res.getData()[0] = encrypted_model; + + if (tEncrypt != null) { + ParamServStatistics.accHEEncryptionTime((long)tEncrypt.stop()); + } + + // stop timing + DoubleObject gradientsTime = new DoubleObject(tTotal.stop()); + res.getData()[1] = gradientsTime; + } catch (Exception e) { + return new FederatedResponse(FederatedResponse.ResponseType.ERROR, new Object[] { e }); + } + return res; + } + } + + private static class HEComputePartialDecryption extends FederatedUDF { + private static final long serialVersionUID = -4535098129348794852L; + private final CiphertextMatrix[] _encrypted_sum; + + protected HEComputePartialDecryption(CiphertextMatrix[] encrypted_sum) { + super(new long[]{}); + _encrypted_sum = encrypted_sum; + } + + @Override + public FederatedResponse execute(ExecutionContext ec, Data... data) { + Timing tPartialDecrypt = DMLScript.STATISTICS ? new Timing(true) : null; + PlaintextMatrix[] result = new PlaintextMatrix[_encrypted_sum.length]; + IntStream.range(0, result.length).forEach(i -> { + result[i] = ec.getSealClient().partiallyDecrypt(_encrypted_sum[i]); + }); + if (tPartialDecrypt != null) { + ParamServStatistics.accHEPartialDecryptionTime((long)tPartialDecrypt.stop()); + } + return new FederatedResponse(FederatedResponse.ResponseType.SUCCESS, result); + } + + @Override + public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) { + return null; + } + } + + + public PlaintextMatrix[] getPartialDecryption(CiphertextMatrix[] encrypted_sum) { + Object udf = new HEComputePartialDecryption(encrypted_sum); + Future<FederatedResponse> udfResponse = _featuresData.executeFederatedOperation( + new FederatedRequest(RequestType.EXEC_UDF, _featuresData.getVarID(), udf)); + + try { + Object[] responseData = udfResponse.get().getData(); + return (PlaintextMatrix[]) responseData; + } catch(Exception e) { + throw new DMLRuntimeException("FederatedLocalPSThread: failed to execute UDF" + e.getMessage()); + } + } + // Statistics methods protected void accFedPSGradientWeightingTime(Timing time) { if (DMLScript.STATISTICS && time != null) @@ -608,4 +811,24 @@ protected void accBatchIndexingTime(Timing time) { protected void accGradientComputeTime(Timing time) { throw new NotImplementedException(); } + + public PublicKey getPartialPublicKey() { + return _partial_public_key; + } + + public void setPublicKey(PublicKey public_key) { + Future<FederatedResponse> res = _featuresData.executeFederatedOperation( + new FederatedRequest(RequestType.EXEC_UDF, _featuresData.getVarID(), + new SetPublicKeyFederatedWorker(public_key))); + + try { + FederatedResponse response = res.get(); + if(!response.isSuccessful()) + throw new DMLRuntimeException("FederatedLocalPSThread: SetPublicKey UDF failed"); + + } + catch(Exception e) { + throw new DMLRuntimeException("FederatedLocalPSThread: failed to execute Public Key Setup UDF" + e.getMessage()); + } + } } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/HEParamServer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/HEParamServer.java new file mode 100644 index 00000000000..577bf6c8205 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/HEParamServer.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.controlprogram.paramserv; + +import org.apache.sysds.api.DMLScript; +import org.apache.sysds.parser.Statement; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.PublicKey; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.SEALServer; +import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing; +import org.apache.sysds.runtime.instructions.cp.CiphertextMatrix; +import org.apache.sysds.runtime.instructions.cp.ListObject; +import org.apache.sysds.runtime.instructions.cp.PlaintextMatrix; +import org.apache.sysds.utils.NativeHelper; +import org.apache.sysds.utils.stats.ParamServStatistics; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * This class implements Homomorphic Encryption (HE) for LocalParamServer. It only supports modelAvg=true. + */ +public class HEParamServer extends LocalParamServer { + private int _thread_counter = 0; + private final List<FederatedPSControlThread> _threads; + private final List<Object> _result_buffer; // one per thread + private Object _result; + private final SEALServer _seal_server; + + public static HEParamServer create(ListObject model, String aggFunc, Statement.PSUpdateType updateType, + Statement.PSFrequency freq, ExecutionContext ec, int workerNum, String valFunc, int numBatchesPerEpoch, + MatrixObject valFeatures, MatrixObject valLabels, int nbatches) + { + NativeHEHelper.initialize(); + return new HEParamServer(model, aggFunc, updateType, freq, ec, + workerNum, valFunc, numBatchesPerEpoch, valFeatures, valLabels, nbatches); + } + + private HEParamServer(ListObject model, String aggFunc, Statement.PSUpdateType updateType, + Statement.PSFrequency freq, ExecutionContext ec, int workerNum, String valFunc, int numBatchesPerEpoch, + MatrixObject valFeatures, MatrixObject valLabels, int nbatches) + { + super(model, aggFunc, updateType, freq, ec, workerNum, valFunc, numBatchesPerEpoch, valFeatures, valLabels, nbatches, true); + + _seal_server = new SEALServer(); + + _threads = Collections.synchronizedList(new ArrayList<>(workerNum)); + for (int i = 0; i < getNumWorkers(); i++) { + _threads.add(null); + } + + _result_buffer = new ArrayList<>(workerNum); + resetResultBuffer(); + } + + public void registerThread(int thread_id, FederatedPSControlThread thread) { + _threads.set(thread_id, thread); + } + + private synchronized void resetResultBuffer() { + _result_buffer.clear(); + for (int i = 0; i < getNumWorkers(); i++) { + _result_buffer.add(null); + } + } + + public byte[] generateA() { + return _seal_server.generateA(); + } + + public PublicKey aggregatePartialPublicKeys(PublicKey[] partial_public_keys) { + return _seal_server.aggregatePartialPublicKeys(partial_public_keys); + } + + /** + * this method collects all T Objects from each worker into a list and then calls f once on this list to produce + * another T, which it returns. + */ + private synchronized <T,U> U collectAndDo(int workerId, T obj, Function<List<T>, U> f) { + _result_buffer.set(workerId, obj); + _thread_counter++; + + if (_thread_counter == getNumWorkers()) { + List<T> buf = _result_buffer.stream().map(x -> (T)x).collect(Collectors.toList()); + _result = f.apply(buf); + resetResultBuffer(); + _thread_counter = 0; + notifyAll(); + } else { + try { + wait(); + } catch (InterruptedException i) { + throw new RuntimeException("thread interrupted"); + } + } + + return (U) _result; + } + + private CiphertextMatrix[] homomorphicAggregation(List<ListObject> encrypted_models) { + Timing tAgg = DMLScript.STATISTICS ? new Timing(true) : null; + CiphertextMatrix[] result = new CiphertextMatrix[encrypted_models.get(0).getLength()]; + IntStream.range(0, encrypted_models.get(0).getLength()).forEach(matrix_idx -> { + CiphertextMatrix[] summands = new CiphertextMatrix[encrypted_models.size()]; + for (int i = 0; i < encrypted_models.size(); i++) { + summands[i] = (CiphertextMatrix) encrypted_models.get(i).getData(matrix_idx); + } + result[matrix_idx] = _seal_server.accumulateCiphertexts(summands);; + }); + if (tAgg != null) { + ParamServStatistics.accHEAccumulation((long)tAgg.stop()); + } + return result; + } + + private Void homomorphicAverage(CiphertextMatrix[] encrypted_sums, List<PlaintextMatrix[]> partial_decryptions) { + Timing tDecrypt = DMLScript.STATISTICS ? new Timing(true) : null; + + MatrixObject[] result = new MatrixObject[partial_decryptions.get(0).length]; + + IntStream.range(0, partial_decryptions.get(0).length).forEach(matrix_idx -> { + PlaintextMatrix[] partial_plaintexts = new PlaintextMatrix[partial_decryptions.size()]; + for (int i = 0; i < partial_decryptions.size(); i++) { + partial_plaintexts[i] = partial_decryptions.get(i)[matrix_idx]; + } + + result[matrix_idx] = _seal_server.average(encrypted_sums[matrix_idx], partial_plaintexts); + }); + + ListObject old_model = getResult(); + ListObject new_model = new ListObject(old_model); + for (int i = 0; i < new_model.getLength(); i++) { + new_model.set(i, result[i]); + } + + if (tDecrypt != null) { + ParamServStatistics.accHEDecryptionTime((long)tDecrypt.stop()); + } + + updateAndBroadcastModel(new_model, null); + return null; + } + + // this is only to be used in push() + private Timing commTimer; + private void startCommTimer() { + commTimer = new Timing(true); + } + private long stopCommTimer() { + return (long)commTimer.stop(); + } + // --------------------------------- + + @Override + public void push(int workerID, ListObject encrypted_model) { + // wait for all updates and sum them homomorphically + CiphertextMatrix[] homomorphic_sum = collectAndDo(workerID, encrypted_model, x -> { + CiphertextMatrix[] res = this.homomorphicAggregation(x); + this.startCommTimer(); + return res; + }); + + // get partial decryptions + PlaintextMatrix[] partial_decryption = _threads.get(workerID).getPartialDecryption(homomorphic_sum); + + // do average and update global model + collectAndDo(workerID, partial_decryption, x -> { + ParamServStatistics.accFedNetworkTime(this.stopCommTimer()); + return this.homomorphicAverage(homomorphic_sum, x); + }); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/LocalParamServer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/LocalParamServer.java index 50c76a0f427..9fd49ca0d10 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/LocalParamServer.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/LocalParamServer.java @@ -39,7 +39,7 @@ public static LocalParamServer create(ListObject model, String aggFunc, Statemen workerNum, valFunc, numBatchesPerEpoch, valFeatures, valLabels, nbatches, modelAvg); } - private LocalParamServer(ListObject model, String aggFunc, Statement.PSUpdateType updateType, + protected LocalParamServer(ListObject model, String aggFunc, Statement.PSUpdateType updateType, Statement.PSFrequency freq, ExecutionContext ec, int workerNum, String valFunc, int numBatchesPerEpoch, MatrixObject valFeatures, MatrixObject valLabels, int nbatches, boolean modelAvg) { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NativeHEHelper.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NativeHEHelper.java new file mode 100644 index 00000000000..7757ad722bb --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NativeHEHelper.java @@ -0,0 +1,100 @@ +package org.apache.sysds.runtime.controlprogram.paramserv; + +import org.apache.commons.lang.SystemUtils; +import org.apache.sysds.utils.NativeHelper; + +public class NativeHEHelper { + public static boolean initialize() { + String platform_suffix = (SystemUtils.IS_OS_WINDOWS ? "-Windows-AMD64.dll" : "-Linux-x86_64.so"); + String library_name = "libhe" + platform_suffix; + return NativeHelper.loadLibraryHelperFromResource(library_name); + } + + // ---------------------------------------------------------------------------------------------------------------- + // SEAL integration + // ---------------------------------------------------------------------------------------------------------------- + + // these are called by SEALClient + + /** + * generates a Client object + * @param a a constant generated by generateA + * @return a pointer to the native client object + */ + public static native long initClient(byte[] a); + + /** + * generates a partial public key + * stores a partial private key corresponding to the partial public key in client + * @param client A pointer to a Client, obtained from initClient + * @return a serialized partial public key + */ + public static native byte[] generatePartialPublicKey(long client); + + /** + * sets the public key and stores it in client + * @param client A pointer to a Client, obtained from initClient + * @param public_key serialized public key obtained from generatePartialPublicKey + */ + public static native void setPublicKey(long client, byte[] public_key); + + /** + * encrypts data with public key stored in client + * setPublicKey() must have been called before calling this + * @param client A pointer to a Client, obtained from initClient + * @param plaintexts array of double values to be encrypted + * @return serialized ciphertext + */ + public static native byte[] encrypt(long client, double[] plaintexts); + + /** + * partially decrypts ciphertexts with the partial private key. generatePartialPublicKey() must + * have been called before calling this function + * @param client A pointer to a Client, obtained from initClient + * @param ciphertext serialized ciphertext + * @return serialized partial decryption + */ + public static native byte[] partiallyDecrypt(long client, byte[] ciphertext); + + // ---------------------------------------------------------------------------------------------------------------- + + // these are called by SEALServer + + /** + * generates the Server object and returns a pointer to it + * @return pointer to a native Server object + */ + public static native long initServer(); + + /** + * this generates the a constant. in a future version we want to generate this together with the clients to prevent misuse + * @param server A pointer to a Server, obtained from initServer + * @return serialized a constant + */ + public static native byte[] generateA(long server); + + /** + * accumulates the given partial public keys into a public key, stores it in server and returns it + * @param server A pointer to a Server, obtained from initServer + * @param partial_public_keys array of serialized partial public keys + * @return serialized partial public key + */ + public static native byte[] aggregatePartialPublicKeys(long server, byte[][] partial_public_keys); + + /** + * accumulates the given ciphertexts into a sum ciphertext and returns it + * @param server A pointer to a Server, obtained from initServer + * @param ciphertexts array of serialized ciphertexts + * @return serialized accumulated ciphertext + */ + public static native byte[] accumulateCiphertexts(long server, byte[][] ciphertexts); + + /** + * averages the partial decryptions and returns the result + * @param server A pointer to a Server, obtained from initServer + * @param encrypted_sum the result of accumulateCiphertexts() + * @param partial_plaintexts the result of partiallyDecrypt of each ciphertext fed into accumulateCiphertexts + * @return average of original data + */ + public static native double[] average(long server, byte[] encrypted_sum, byte[][] partial_plaintexts); +} diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NetworkTrafficCounter.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NetworkTrafficCounter.java new file mode 100644 index 00000000000..d0ba01dba93 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NetworkTrafficCounter.java @@ -0,0 +1,23 @@ +package org.apache.sysds.runtime.controlprogram.paramserv; + +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.traffic.ChannelTrafficShapingHandler; +import java.util.function.BiConsumer; + +public class NetworkTrafficCounter extends ChannelTrafficShapingHandler { + private final BiConsumer<Long, Long> _fn; // (read, written) -> Void, logs bytes read and written + public NetworkTrafficCounter(BiConsumer<Long, Long> fn) { + // checkInterval of zero means that doAccounting will not be called + super( 0); + _fn = fn; + } + + // log bytes read/written after channel is closed + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + _fn.accept(trafficCounter.cumulativeReadBytes(), trafficCounter.cumulativeWrittenBytes()); + trafficCounter.resetCumulativeTime(); + super.channelInactive(ctx); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/ParamServer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/ParamServer.java index 009dc20a338..0e09fabf30b 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/ParamServer.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/ParamServer.java @@ -19,10 +19,7 @@ package org.apache.sysds.runtime.controlprogram.paramserv; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.stream.Collectors; @@ -326,30 +323,8 @@ protected synchronized void updateAverageModel(int workerID, ListObject model) { _accModels = ParamservUtils.accrueGradients(_accModels, weightParams, true); if(allFinished()) { - _model = setParams(_ec, _accModels, _model); - if (DMLScript.STATISTICS && tAgg != null) - ParamServStatistics.accAggregationTime((long) tAgg.stop()); - _accModels = null; //reset for next accumulation - - // This if has grown to be quite complex its function is rather simple. Validate at the end of each epoch - // In the BSP batch case that occurs after the sync counter reaches the number of batches and in the - // BSP epoch case every time - if(_numBatchesPerEpoch != -1 && (_freq == Statement.PSFrequency.EPOCH || (_freq == Statement.PSFrequency.BATCH && ++_syncCounter % _numBatchesPerEpoch == 0))) { - - if(LOG.isInfoEnabled()) - LOG.info("[+] PARAMSERV: completed EPOCH " + _epochCounter); - time_epoch(); - if(_validationPossible) { - validate(); - } - _epochCounter++; - _syncCounter = 0; - } - // Broadcast the updated model + updateAndBroadcastModel(_accModels, tAgg); resetFinishedStates(); - broadcastModel(true); - if(LOG.isDebugEnabled()) - LOG.debug("Global parameter is broadcasted successfully "); } break; } @@ -365,7 +340,33 @@ protected synchronized void updateAverageModel(int workerID, ListObject model) { } } - protected ListObject weightModels(ListObject params, int numWorkers) { + protected void updateAndBroadcastModel(ListObject new_model, Timing tAgg) { + _model = setParams(_ec, new_model, _model); + if (DMLScript.STATISTICS && tAgg != null) + ParamServStatistics.accAggregationTime((long) tAgg.stop()); + _accModels = null; //reset for next accumulation + + // This if has grown to be quite complex its function is rather simple. Validate at the end of each epoch + // In the BSP batch case that occurs after the sync counter reaches the number of batches and in the + // BSP epoch case every time + if(_numBatchesPerEpoch != -1 && (_freq == Statement.PSFrequency.EPOCH || (_freq == Statement.PSFrequency.BATCH && ++_syncCounter % _numBatchesPerEpoch == 0))) { + + if(LOG.isInfoEnabled()) + LOG.info("[+] PARAMSERV: completed EPOCH " + _epochCounter); + time_epoch(); + if(_validationPossible) { + validate(); + } + _epochCounter++; + _syncCounter = 0; + } + // Broadcast the updated model + broadcastModel(true); + if(LOG.isDebugEnabled()) + LOG.debug("Global parameter is broadcasted successfully "); + } + + protected ListObject weightModels(ListObject params, int numWorkers) { double _averagingFactor = 1d / numWorkers; if( _averagingFactor != 1) { @@ -472,6 +473,10 @@ private void validate() { ParamServStatistics.accValidationTime((long) tValidate.stop()); } + public int getNumWorkers() { + return _numWorkers; + } + public FunctionCallCPInstruction getAggInst() { return _inst; } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DataPartitionFederatedScheme.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DataPartitionFederatedScheme.java index 5bb3e12dcab..96979e3a5d8 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DataPartitionFederatedScheme.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DataPartitionFederatedScheme.java @@ -87,6 +87,7 @@ static List<MatrixObject> sliceFederatedMatrix(MatrixObject fedMatrix) { new MatrixCharacteristics(range.getSize(0), range.getSize(1)), Types.FileFormat.BINARY) ); + slice.setPrivacyConstraints(fedMatrix.getPrivacyConstraint()); // Create new federation map List<Pair<FederatedRange, FederatedData>> newFedHashMap = new ArrayList<>(); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/PublicKey.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/PublicKey.java new file mode 100644 index 00000000000..96fd415308e --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/PublicKey.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption; + +import java.io.Serializable; + +public class PublicKey implements Serializable { + private static final long serialVersionUID = 91289081237980123L; + + private final byte[] _data; + + public PublicKey(byte[] data) { + _data = data; + } + + public byte[] getData() { + return _data; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALClient.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALClient.java new file mode 100644 index 00000000000..935f2808af5 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALClient.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption; + +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.controlprogram.paramserv.NativeHEHelper; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.instructions.cp.CiphertextMatrix; +import org.apache.sysds.runtime.instructions.cp.PlaintextMatrix; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; + +import java.util.stream.IntStream; + +public class SEALClient { + public SEALClient(byte[] a) { + // TODO take params here, like slot_count etc. + ctx = NativeHEHelper.initClient(a); + } + + // this is a pointer to the context used by all native methods of this class + private final long ctx; + + + /** + * generates a partial public key + * stores a partial private key corresponding to the partial public key in ctx + * + * @return the partial public key + */ + public PublicKey generatePartialPublicKey() { + return new PublicKey(NativeHEHelper.generatePartialPublicKey(ctx)); + } + + /** + * sets the public key and stores it in ctx + * + * @param public_key the public key to set + */ + public void setPublicKey(PublicKey public_key) { + NativeHEHelper.setPublicKey(ctx, public_key.getData()); + } + + /** + * encrypts one block of data with public key stored statically and returns it + * setPublicKey() must have been called before calling this + * @param plaintext the MatrixObject to encrypt + * @return the encrypted matrix + */ + public CiphertextMatrix encrypt(MatrixObject plaintext) { + MatrixBlock mb = plaintext.acquireReadAndRelease(); + if (mb.isInSparseFormat()) { + mb.allocateSparseRowsBlock(); + mb.sparseToDense(); + } + DenseBlock db = mb.getDenseBlock(); + int[] dims = IntStream.range(0, db.numDims()).map(db::getDim).toArray(); + double[] raw_data = mb.getDenseBlockValues(); + return new CiphertextMatrix(dims, plaintext.getDataCharacteristics(), NativeHEHelper.encrypt(ctx, raw_data)); + } + + /** + * partially decrypts ciphertext with the partial private key. generatePartialPublicKey() must + * have been called before calling this function + * + * @param ciphertext the ciphertext to partially decrypt + * @return the partial decryption of ciphertext + */ + public PlaintextMatrix partiallyDecrypt(CiphertextMatrix ciphertext) { + return new PlaintextMatrix(ciphertext.getDims(), ciphertext.getDataCharacteristics(), NativeHEHelper.partiallyDecrypt(ctx, ciphertext.getData())); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java new file mode 100644 index 00000000000..d6265c7f6d7 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption; + +import org.apache.sysds.common.Types; +import org.apache.sysds.hops.OptimizerUtils; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.controlprogram.paramserv.NativeHEHelper; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.data.DenseBlockFactory; +import org.apache.sysds.runtime.instructions.cp.CiphertextMatrix; +import org.apache.sysds.runtime.instructions.cp.Encrypted; +import org.apache.sysds.runtime.instructions.cp.PlaintextMatrix; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.meta.MetaDataFormat; + +import java.util.Arrays; + +public class SEALServer { + public SEALServer() { + // TODO take params here, like slot_count etc. + ctx = NativeHEHelper.initServer(); + } + + // this is a pointer to the context used by all native methods of this class + private final long ctx; + private byte[] _a; + + /** + * this generates the a constant. in a future version we want to generate this together with the clients to prevent misuse + * @return serialized a constant + */ + public synchronized byte[] generateA() { + if (_a == null) { + _a = NativeHEHelper.generateA(ctx); + } + return _a; + } + + /** + * accumulates the given partial public keys into a public key, stores it in ctx and returns it + * @param partial_public_keys an array of partial public keys generated with SEALServer::generatePartialPublicKey + * @return the aggregated public key + */ + public PublicKey aggregatePartialPublicKeys(PublicKey[] partial_public_keys) { + return new PublicKey(NativeHEHelper.aggregatePartialPublicKeys(ctx, extractRawData(partial_public_keys))); + } + + /** + * accumulates the given ciphertext blocks into a sum ciphertext and returns it + * @param ciphertexts ciphertexts encrypted with the partial public keys + * @return the accumulated ciphertext (which is the homomorphic sum of ciphertexts) + */ + public CiphertextMatrix accumulateCiphertexts(CiphertextMatrix[] ciphertexts) { + return new CiphertextMatrix(ciphertexts[0].getDims(), ciphertexts[0].getDataCharacteristics(), NativeHEHelper.accumulateCiphertexts(ctx, extractRawData(ciphertexts))); + } + + /** + * averages the partial decryptions + * @param encrypted_sum is the result of accumulateCiphertexts() + * @param partial_plaintexts is the result of SEALServer::partiallyDecrypt of each ciphertext fed into accumulateCiphertexts + * @return the unencrypted, element-wise average of the original matrices + */ + public MatrixObject average(CiphertextMatrix encrypted_sum, PlaintextMatrix[] partial_plaintexts) { + double[] raw_result = NativeHEHelper.average(ctx, encrypted_sum.getData(), extractRawData(partial_plaintexts)); + int[] dims = encrypted_sum.getDims(); + int result_len = Arrays.stream(dims).reduce(1, (x,y) -> x*y); + DataCharacteristics dc = encrypted_sum.getDataCharacteristics(); + + DenseBlock new_dense_block = DenseBlockFactory.createDenseBlock(Arrays.copyOf(raw_result, result_len), dims); + MatrixBlock new_matrix_block = new MatrixBlock((int)dc.getRows(), (int)dc.getCols(), new_dense_block); + MatrixObject new_mo = new MatrixObject(Types.ValueType.FP64, OptimizerUtils.getUniqueTempFileName(), new MetaDataFormat(dc, Types.FileFormat.BINARY)); + new_mo.acquireModify(new_matrix_block); + new_mo.release(); + return new_mo; + } + + private static byte[][] extractRawData(Encrypted[] data) { + byte[][] raw_data = new byte[data.length][]; + for (int i = 0; i < data.length; i++) { + raw_data[i] = data[i].getData(); + } + return raw_data; + } + + // TODO: extract an interface for this and use it here + private static byte[][] extractRawData(PublicKey[] data) { + byte[][] raw_data = new byte[data.length][]; + for (int i = 0; i < data.length; i++) { + raw_data[i] = data[i].getData(); + } + return raw_data; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/CiphertextMatrix.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/CiphertextMatrix.java new file mode 100644 index 00000000000..1cbef9d488e --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/CiphertextMatrix.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.instructions.cp; + +import org.apache.sysds.common.Types; +import org.apache.sysds.runtime.meta.DataCharacteristics; + +/** + * This class abstracts over an encrypted matrix of ciphertexts. It stores the data as opaque byte array. The layout is unspecified. + */ +public class CiphertextMatrix extends Encrypted { + private static final long serialVersionUID = 1762936872261940616L; + + public CiphertextMatrix(int[] dims, DataCharacteristics dc, byte[] data) { + super(dims, dc, data, Types.DataType.ENCRYPTED_CIPHER); + } + + @Override + public String getDebugName() { + return "CiphertextMatrix " + getData().hashCode(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/Encrypted.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/Encrypted.java new file mode 100644 index 00000000000..eb7d1ea44a5 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/Encrypted.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.instructions.cp; + +import org.apache.sysds.common.Types; +import org.apache.sysds.runtime.meta.DataCharacteristics; + +/** + * This class abstracts over an encrypted data. It stores the data as opaque byte array. The layout is unspecified. + */ +public abstract class Encrypted extends Data { + private static final long serialVersionUID = 1762936872268046168L; + + private final int[] _dims; + private final DataCharacteristics _dc; + private final byte[] _data; + + public Encrypted(int[] dims, DataCharacteristics dc, byte[] data, Types.DataType dt) { + super(dt, Types.ValueType.UNKNOWN); + _dims = dims; + _dc = dc; + _data = data; + } + + public int[] getDims() { + return _dims; + } + + public DataCharacteristics getDataCharacteristics() { + return _dc; + } + + public byte[] getData() { + return _data; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ListObject.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ListObject.java index 38288178e4e..5c302fe80a9 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ListObject.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ListObject.java @@ -397,6 +397,18 @@ public void writeExternal(ObjectOutput out) throws IOException { ScalarObject so = (ScalarObject) d; out.writeObject(so.getStringValue()); break; + case ENCRYPTED_CIPHER: + case ENCRYPTED_PLAIN: + Encrypted e = (Encrypted) d; + int[] dims = e.getDims(); + dc = e.getDataCharacteristics(); + out.writeObject(dims); + out.writeObject(dc.getRows()); + out.writeObject(dc.getCols()); + out.writeObject(dc.getBlocksize()); + out.writeObject(dc.getNonZeros()); + out.writeObject(e.getData()); + break; default: throw new DMLRuntimeException("Unable to serialize datatype " + dataType); } @@ -463,6 +475,21 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept } d = so; break; + case ENCRYPTED_CIPHER: + case ENCRYPTED_PLAIN: + int[] dims = (int[]) in.readObject(); + rows = (long) in.readObject(); + cols = (long) in.readObject(); + blockSize = (int) in.readObject(); + nonZeros = (long) in.readObject(); + byte[] data = (byte[])in.readObject(); + DataCharacteristics dc = new MatrixCharacteristics(rows, cols, blockSize, nonZeros); + if (dataType == DataType.ENCRYPTED_CIPHER) { + d = new CiphertextMatrix(dims, dc, data); + } else { + d = new PlaintextMatrix(dims, dc, data); + } + break; default: throw new DMLRuntimeException("Unable to deserialize datatype " + dataType); } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParamservBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParamservBuiltinCPInstruction.java index 25353f60039..d16aa9ec4e2 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParamservBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParamservBuiltinCPInstruction.java @@ -19,28 +19,6 @@ package org.apache.sysds.runtime.instructions.cp; -import static org.apache.sysds.parser.Statement.PS_AGGREGATION_FUN; -import static org.apache.sysds.parser.Statement.PS_BATCH_SIZE; -import static org.apache.sysds.parser.Statement.PS_EPOCHS; -import static org.apache.sysds.parser.Statement.PS_FEATURES; -import static org.apache.sysds.parser.Statement.PS_FED_RUNTIME_BALANCING; -import static org.apache.sysds.parser.Statement.PS_FED_WEIGHTING; -import static org.apache.sysds.parser.Statement.PS_FREQUENCY; -import static org.apache.sysds.parser.Statement.PS_HYPER_PARAMS; -import static org.apache.sysds.parser.Statement.PS_LABELS; -import static org.apache.sysds.parser.Statement.PS_MODE; -import static org.apache.sysds.parser.Statement.PS_MODEL; -import static org.apache.sysds.parser.Statement.PS_MODELAVG; -import static org.apache.sysds.parser.Statement.PS_NBATCHES; -import static org.apache.sysds.parser.Statement.PS_PARALLELISM; -import static org.apache.sysds.parser.Statement.PS_SCHEME; -import static org.apache.sysds.parser.Statement.PS_SEED; -import static org.apache.sysds.parser.Statement.PS_UPDATE_FUN; -import static org.apache.sysds.parser.Statement.PS_UPDATE_TYPE; -import static org.apache.sysds.parser.Statement.PS_VAL_FEATURES; -import static org.apache.sysds.parser.Statement.PS_VAL_FUN; -import static org.apache.sysds.parser.Statement.PS_VAL_LABELS; - import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -71,25 +49,22 @@ import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext; -import org.apache.sysds.runtime.controlprogram.paramserv.FederatedPSControlThread; -import org.apache.sysds.runtime.controlprogram.paramserv.LocalPSWorker; -import org.apache.sysds.runtime.controlprogram.paramserv.LocalParamServer; -import org.apache.sysds.runtime.controlprogram.paramserv.ParamServer; -import org.apache.sysds.runtime.controlprogram.paramserv.ParamservUtils; -import org.apache.sysds.runtime.controlprogram.paramserv.SparkPSBody; -import org.apache.sysds.runtime.controlprogram.paramserv.SparkPSWorker; -import org.apache.sysds.runtime.controlprogram.paramserv.SparkParamservUtils; +import org.apache.sysds.runtime.controlprogram.paramserv.*; import org.apache.sysds.runtime.controlprogram.paramserv.dp.DataPartitionFederatedScheme; import org.apache.sysds.runtime.controlprogram.paramserv.dp.DataPartitionLocalScheme; import org.apache.sysds.runtime.controlprogram.paramserv.dp.FederatedDataPartitioner; import org.apache.sysds.runtime.controlprogram.paramserv.dp.LocalDataPartitioner; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.PublicKey; import org.apache.sysds.runtime.controlprogram.paramserv.rpc.PSRpcFactory; import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing; import org.apache.sysds.runtime.matrix.operators.Operator; +import org.apache.sysds.runtime.privacy.PrivacyConstraint; import org.apache.sysds.runtime.util.ProgramConverter; import org.apache.sysds.utils.stats.ParamServStatistics; +import static org.apache.sysds.parser.Statement.*; + public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruction { private static final Log LOG = LogFactory.getLog(ParamservBuiltinCPInstruction.class.getName()); @@ -102,6 +77,7 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc private static final PSUpdateType DEFAULT_TYPE = PSUpdateType.ASP; public static final int DEFAULT_NBATCHES = 1; private static final Boolean DEFAULT_MODELAVG = false; + private static final Boolean DEFAULT_HE = false; public ParamservBuiltinCPInstruction(Operator op, LinkedHashMap<String, String> paramsMap, CPOperand out, String opcode, String istr) { super(op, paramsMap, out, opcode, istr); @@ -188,23 +164,56 @@ private void runFederated(ExecutionContext ec) { MatrixObject val_features = (getParam(PS_VAL_FEATURES) != null) ? ec.getMatrixObject(getParam(PS_VAL_FEATURES)) : null; MatrixObject val_labels = (getParam(PS_VAL_LABELS) != null) ? ec.getMatrixObject(getParam(PS_VAL_LABELS)) : null; boolean modelAvg = Boolean.parseBoolean(getParam(PS_MODELAVG)); - ParamServer ps = createPS(PSModeType.FEDERATED, aggFunc, updateType, freq, workerNum, model, aggServiceEC, getValFunction(), - getNumBatchesPerEpoch(runtimeBalancing, result._balanceMetrics), val_features, val_labels, nbatches, modelAvg); + + // check if we need homomorphic encryption + boolean use_homomorphic_encryption_ = getHe(); + for (int i = 0; i < workerNum; i++) { + use_homomorphic_encryption_ = use_homomorphic_encryption_ || checkIsPrivate(result._pFeatures.get(i)); + use_homomorphic_encryption_ = use_homomorphic_encryption_ || checkIsPrivate(result._pLabels.get(i)); + } + final boolean use_homomorphic_encryption = use_homomorphic_encryption_; + if (use_homomorphic_encryption && !modelAvg) { + throw new DMLRuntimeException("can't use homomorphic encryption without modelAvg"); + } + + if (use_homomorphic_encryption && weighting) { + throw new DMLRuntimeException("can't use homomorphic encryption with weighting"); + } + + LocalParamServer ps = (LocalParamServer)createPS(PSModeType.FEDERATED, aggFunc, updateType, freq, workerNum, model, aggServiceEC, getValFunction(), + getNumBatchesPerEpoch(runtimeBalancing, result._balanceMetrics), val_features, val_labels, nbatches, modelAvg, use_homomorphic_encryption); // Create the local workers int finalNumBatchesPerEpoch = getNumBatchesPerEpoch(runtimeBalancing, result._balanceMetrics); List<FederatedPSControlThread> threads = IntStream.range(0, workerNum) .mapToObj(i -> new FederatedPSControlThread(i, updFunc, freq, runtimeBalancing, weighting, - getEpochs(), getBatchSize(), finalNumBatchesPerEpoch, federatedWorkerECs.get(i), ps, nbatches, modelAvg)) + getEpochs(), getBatchSize(), finalNumBatchesPerEpoch, federatedWorkerECs.get(i), ps, nbatches, modelAvg, use_homomorphic_encryption)) .collect(Collectors.toList()); if(workerNum != threads.size()) { throw new DMLRuntimeException("ParamservBuiltinCPInstruction: Federated data partitioning does not match threads!"); } + // Set features and lables for the control threads and write the program and instructions and hyperparams to the federated workers for (int i = 0; i < threads.size(); i++) { threads.get(i).setFeatures(result._pFeatures.get(i)); threads.get(i).setLabels(result._pLabels.get(i)); threads.get(i).setup(result._weightingFactors.get(i)); } + + if (use_homomorphic_encryption) { + // generate public key from partial public keys + PublicKey[] partial_public_keys = new PublicKey[threads.size()]; + for (int i = 0; i < threads.size(); i++) { + partial_public_keys[i] = threads.get(i).getPartialPublicKey(); + } + + // TODO: accumulate public keys with SEAL + PublicKey public_key = ((HEParamServer)ps).aggregatePartialPublicKeys(partial_public_keys); + + for (FederatedPSControlThread thread : threads) { + thread.setPublicKey(public_key); + } + } + if (DMLScript.STATISTICS) ParamServStatistics.accSetupTime((long) tSetup.stop()); @@ -479,21 +488,32 @@ private int getWorkerNum(PSModeType mode) { * @return parameter server */ private static ParamServer createPS(PSModeType mode, String aggFunc, PSUpdateType updateType, - PSFrequency freq, int workerNum, ListObject model, ExecutionContext ec, int nbatches, boolean modelAvg) + PSFrequency freq, int workerNum, ListObject model, ExecutionContext ec, int nbatches, boolean modelAvg) { return createPS(mode, aggFunc, updateType, freq, workerNum, model, ec, null, -1, null, null, nbatches, modelAvg); } + + private static ParamServer createPS(PSModeType mode, String aggFunc, PSUpdateType updateType, + PSFrequency freq, int workerNum, ListObject model, ExecutionContext ec, String valFunc, + int numBatchesPerEpoch, MatrixObject valFeatures, MatrixObject valLabels, int nbatches, boolean modelAvg) { + return createPS(mode, aggFunc, updateType, freq, workerNum, model, ec, valFunc, numBatchesPerEpoch, valFeatures, valLabels, nbatches, modelAvg, false); + } + // When this creation is used the parameter server is able to validate after each epoch private static ParamServer createPS(PSModeType mode, String aggFunc, PSUpdateType updateType, PSFrequency freq, int workerNum, ListObject model, ExecutionContext ec, String valFunc, - int numBatchesPerEpoch, MatrixObject valFeatures, MatrixObject valLabels, int nbatches, boolean modelAvg) + int numBatchesPerEpoch, MatrixObject valFeatures, MatrixObject valLabels, int nbatches, boolean modelAvg, boolean use_homomorphic_encryption) { switch (mode) { case FEDERATED: case LOCAL: case REMOTE_SPARK: - return LocalParamServer.create(model, aggFunc, updateType, freq, ec, workerNum, valFunc, numBatchesPerEpoch, valFeatures, valLabels, nbatches, modelAvg); + if (use_homomorphic_encryption) { + return HEParamServer.create(model, aggFunc, updateType, freq, ec, workerNum, valFunc, numBatchesPerEpoch, valFeatures, valLabels, nbatches); + } else { + return LocalParamServer.create(model, aggFunc, updateType, freq, ec, workerNum, valFunc, numBatchesPerEpoch, valFeatures, valLabels, nbatches, modelAvg); + } default: throw new DMLRuntimeException("Unsupported parameter server: " + mode.name()); } @@ -614,4 +634,15 @@ private int getNbatches() { } return Integer.parseInt(getParam(PS_NBATCHES)); } + + private boolean checkIsPrivate(MatrixObject obj) { + PrivacyConstraint pc = obj.getPrivacyConstraint(); + return pc != null && pc.hasPrivateElements(); + } + + private boolean getHe() { + if(!getParameterMap().containsKey(PS_HE)) + return DEFAULT_HE; + return Boolean.parseBoolean(getParam(PS_HE)); + } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/PlaintextMatrix.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/PlaintextMatrix.java new file mode 100644 index 00000000000..6fe2b3814f4 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/PlaintextMatrix.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.instructions.cp; + +import org.apache.sysds.common.Types; +import org.apache.sysds.runtime.meta.DataCharacteristics; + +/** + * This class abstracts over an encrypted matrix of ciphertexts. It stores the data as opaque byte array. The layout is unspecified. + */ +public class PlaintextMatrix extends Encrypted { + private static final long serialVersionUID = 5732436872261940616L; + + public PlaintextMatrix(int[] dims, DataCharacteristics dc, byte[] data) { + super(dims, dc, data, Types.DataType.ENCRYPTED_PLAIN); + } + + @Override + public String getDebugName() { + return "PlaintextMatrix " + getData().hashCode(); + } +} diff --git a/src/main/java/org/apache/sysds/utils/NativeHelper.java b/src/main/java/org/apache/sysds/utils/NativeHelper.java index 83869c23d2f..e86bd56b19d 100644 --- a/src/main/java/org/apache/sysds/utils/NativeHelper.java +++ b/src/main/java/org/apache/sysds/utils/NativeHelper.java @@ -44,14 +44,14 @@ * By default, it first tries to load Intel MKL, else tries to load OpenBLAS. */ public class NativeHelper { - + public enum NativeBlasState { NOT_ATTEMPTED_LOADING_NATIVE_BLAS, SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_IN_USE, SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_NOT_IN_USE, ATTEMPTED_LOADING_NATIVE_BLAS_UNSUCCESSFULLY } - + public static NativeBlasState CURRENT_NATIVE_BLAS_STATE = NativeBlasState.NOT_ATTEMPTED_LOADING_NATIVE_BLAS; private static String blasType; @@ -63,16 +63,16 @@ public enum NativeBlasState { /** * Called by Statistics to print the loaded BLAS. - * + * * @return empty string or the BLAS that is loaded */ public static String getCurrentBLAS() { return blasType != null ? blasType : ""; } - + /** * Called by runtime to check if the BLAS is available for exploitation - * + * * @return true if CURRENT_NATIVE_BLAS_STATE is SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_NOT_IN_USE else false */ public static boolean isNativeLibraryLoaded() { @@ -99,10 +99,10 @@ public static boolean isNativeLibraryLoaded() { } return CURRENT_NATIVE_BLAS_STATE == NativeBlasState.SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_IN_USE; } - + /** - * Initialize the native library before executing the DML program - * + * Initialize the native library before executing the DML program + * * @param customLibPath specified by sysds.native.blas.directory * @param userSpecifiedBLAS specified by sysds.native.blas */ @@ -121,22 +121,22 @@ else if(!isBLASLoaded() && isSupportedBLAS(userSpecifiedBLAS)) { performLoading(customLibPath, userSpecifiedBLAS); } } - + /** * Return true if the given BLAS type is supported. - * + * * @param userSpecifiedBLAS BLAS type specified via sysds.native.blas property * @return true if the userSpecifiedBLAS is auto | mkl | openblas, else false */ private static boolean isSupportedBLAS(String userSpecifiedBLAS) { - return userSpecifiedBLAS.equalsIgnoreCase("auto") || - userSpecifiedBLAS.equalsIgnoreCase("mkl") || + return userSpecifiedBLAS.equalsIgnoreCase("auto") || + userSpecifiedBLAS.equalsIgnoreCase("mkl") || userSpecifiedBLAS.equalsIgnoreCase("openblas"); } - + /** * Note: we only support 64 bit Java on x86 and AMD machine - * + * * @return true if the hardware architecture is supported */ private static boolean isSupportedArchitecture() { @@ -166,21 +166,21 @@ private static boolean isSupportedOS() { * SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_NOT_IN_USE */ private static boolean isBLASLoaded() { - return CURRENT_NATIVE_BLAS_STATE == NativeBlasState.SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_IN_USE || + return CURRENT_NATIVE_BLAS_STATE == NativeBlasState.SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_IN_USE || CURRENT_NATIVE_BLAS_STATE == NativeBlasState.SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_NOT_IN_USE; } - + /** * Check if we should attempt to perform loading. * If custom library path is provided, we should attempt to load again if not already loaded. - * - * @param customLibPath custom library path + * + * @param customLibPath custom library path * @return true if we should attempt to load blas again */ private static boolean shouldReload(String customLibPath) { boolean isValidBLASDirectory = customLibPath != null && !customLibPath.equalsIgnoreCase("none"); return CURRENT_NATIVE_BLAS_STATE == NativeBlasState.NOT_ATTEMPTED_LOADING_NATIVE_BLAS || - (isValidBLASDirectory && !isBLASLoaded()); + (isValidBLASDirectory && !isBLASLoaded()); } // Performing loading in a method instead of a static block will throw a detailed stack trace in case of fatal errors @@ -191,13 +191,13 @@ private static void performLoading(String customLibPath, String userSpecifiedBLA // attemptedLoading variable ensures that we don't try to load SystemDS and other dependencies // again and again especially in the parfor (hence the double-checking with synchronized). if(shouldReload(customLibPath) && isSupportedBLAS(userSpecifiedBLAS) && isSupportedArchitecture() - && isSupportedOS()) { + && isSupportedOS()) { long start = System.nanoTime(); synchronized(NativeHelper.class) { if(shouldReload(customLibPath)) { // Set attempted loading unsuccessful in case of exception CURRENT_NATIVE_BLAS_STATE = NativeBlasState.ATTEMPTED_LOADING_NATIVE_BLAS_UNSUCCESSFULLY; - String [] blas = new String[] { userSpecifiedBLAS }; + String [] blas = new String[] { userSpecifiedBLAS }; if(userSpecifiedBLAS.equalsIgnoreCase("auto")) { blas = new String[] { "mkl", "openblas" }; } @@ -206,7 +206,7 @@ && isSupportedOS()) { String platform_suffix = (SystemUtils.IS_OS_WINDOWS ? "-Windows-AMD64.dll" : "-Linux-x86_64.so"); String library_name = "libsystemds_" + blasType + platform_suffix; if(loadLibraryHelperFromResource(library_name) || - loadBLAS(customLibPath, library_name,"Loading native helper with customLibPath.")) + loadBLAS(customLibPath, library_name,"Loading native helper with customLibPath.")) { LOG.info("Using native blas: " + blasType + getNativeBLASPath()); CURRENT_NATIVE_BLAS_STATE = NativeBlasState.SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_IN_USE; @@ -215,15 +215,15 @@ && isSupportedOS()) { } } double timeToLoadInMilliseconds = (System.nanoTime()-start)*1e-6; - if(timeToLoadInMilliseconds > 1000) + if(timeToLoadInMilliseconds > 1000) LOG.warn("Time to load native blas: " + timeToLoadInMilliseconds + " milliseconds."); } else if(LOG.isDebugEnabled() && !isSupportedBLAS(userSpecifiedBLAS)) { LOG.debug("Using internal Java BLAS as native BLAS support instead of the configuration " + - "'sysds.native.blas'=" + userSpecifiedBLAS + "."); + "'sysds.native.blas'=" + userSpecifiedBLAS + "."); } } - + private static boolean checkAndLoadBLAS(String customLibPath, String [] listBLAS) { if(customLibPath != null && customLibPath.equalsIgnoreCase("none")) customLibPath = null; @@ -250,10 +250,10 @@ else if (blas.equalsIgnoreCase("openblas")) { } return isLoaded; } - + /** * Useful method for debugging. - * + * * @return empty string (if !LOG.isDebugEnabled()) or the path from where openblas or mkl is loaded. */ private static String getNativeBLASPath() { @@ -287,8 +287,8 @@ public static int getMaxNumThreads() { /** * Attempts to load native BLAS - * - * @param customLibPath can be null (if we want to only want to use LD_LIBRARY_PATH), else the + * + * @param customLibPath can be null (if we want to only want to use LD_LIBRARY_PATH), else the * @param blas can be gomp, openblas or mkl_rt * @param optionalMsg message for debugging * @return true if successfully loaded BLAS @@ -300,8 +300,8 @@ public static boolean loadBLAS(String customLibPath, String blas, String optiona try { // This fixes libPath if it already contained a prefix/suffix and mapLibraryName added another one. libPath = libPath.replace("liblibsystemds", "libsystemds") - .replace(".dll.dll", ".dll") - .replace(".so.so", ".so"); + .replace(".dll.dll", ".dll") + .replace(".so.so", ".so"); System.load(libPath); LOG.info("Loaded the library:" + libPath); return true; @@ -321,7 +321,7 @@ public static boolean loadBLAS(String customLibPath, String blas, String optiona catch (UnsatisfiedLinkError e) { LOG.debug("java.library.path: " + System.getProperty("java.library.path")); LOG.debug("Unable to load " + blas + (optionalMsg == null ? "" : (" (" + optionalMsg + ")")) + - " \n Message from exception was: " + e.getMessage()); + " \n Message from exception was: " + e.getMessage()); return false; } } @@ -355,13 +355,13 @@ public static boolean loadLibraryHelperFromResource(String libFileName) { } // TODO: Add pmm, wsloss, mmchain, etc. - + //double-precision matrix multiply dense-dense public static native long dmmdd(double [] m1, double [] m2, double [] ret, int m1rlen, int m1clen, int m2clen, - int numThreads); + int numThreads); //single-precision matrix multiply dense-dense public static native long smmdd(FloatBuffer m1, FloatBuffer m2, FloatBuffer ret, int m1rlen, int m1clen, int m2clen, - int numThreads); + int numThreads); //transpose-self matrix multiply public static native long tsmm(double[] m1, double[] ret, int m1rlen, int m1clen, boolean leftTrans, int numThreads); @@ -374,27 +374,27 @@ public static native long smmdd(FloatBuffer m1, FloatBuffer m2, FloatBuffer ret, // Returns -1 if failures or returns number of nonzeros // Called by DnnCPInstruction if both input and filter are dense - public static native long conv2dDense(double [] input, double [] filter, double [] ret, int N, int C, int H, int W, - int K, int R, int S, int stride_h, int stride_w, int pad_h, int pad_w, int P, int Q, int numThreads); + public static native long conv2dDense(double [] input, double [] filter, double [] ret, int N, int C, int H, int W, + int K, int R, int S, int stride_h, int stride_w, int pad_h, int pad_w, int P, int Q, int numThreads); public static native long dconv2dBiasAddDense(double [] input, double [] bias, double [] filter, double [] ret, int N, - int C, int H, int W, int K, int R, int S, int stride_h, int stride_w, int pad_h, int pad_w, int P, int Q, - int numThreads); + int C, int H, int W, int K, int R, int S, int stride_h, int stride_w, int pad_h, int pad_w, int P, int Q, + int numThreads); public static native long sconv2dBiasAddDense(FloatBuffer input, FloatBuffer bias, FloatBuffer filter, FloatBuffer ret, - int N, int C, int H, int W, int K, int R, int S, int stride_h, int stride_w, int pad_h, int pad_w, int P, int Q, - int numThreads); + int N, int C, int H, int W, int K, int R, int S, int stride_h, int stride_w, int pad_h, int pad_w, int P, int Q, + int numThreads); // Called by DnnCPInstruction if both input and filter are dense public static native long conv2dBackwardFilterDense(double [] input, double [] dout, double [] ret, int N, int C, - int H, int W, int K, int R, int S, int stride_h, int stride_w, - int pad_h, int pad_w, int P, int Q, int numThreads); + int H, int W, int K, int R, int S, int stride_h, int stride_w, + int pad_h, int pad_w, int P, int Q, int numThreads); // If both filter and dout are dense, then called by DnnCPInstruction // Else, called by LibMatrixDNN's thread if filter is dense. dout[n] is converted to dense if sparse. public static native long conv2dBackwardDataDense(double [] filter, double [] dout, double [] ret, int N, int C, - int H, int W, int K, int R, int S, int stride_h, int stride_w, - int pad_h, int pad_w, int P, int Q, int numThreads); + int H, int W, int K, int R, int S, int stride_h, int stride_w, + int pad_h, int pad_w, int P, int Q, int numThreads); // Currently only supported with numThreads = 1 and sparse input // Called by LibMatrixDNN's thread if input is sparse. dout[n] is converted to dense if sparse. @@ -415,4 +415,4 @@ public static native boolean conv2dSparse(int apos, int alen, int[] aix, double[ // different tradeoffs. In current implementation, we always use GetPrimitiveArrayCritical as it has proven to be // fastest. We can revisit this decision later and hence I would not recommend removing this method. private static native void setMaxNumThreads(int numThreads); -} +} \ No newline at end of file diff --git a/src/main/java/org/apache/sysds/utils/Statistics.java b/src/main/java/org/apache/sysds/utils/Statistics.java index 77b63a99213..aece9b655a6 100644 --- a/src/main/java/org/apache/sysds/utils/Statistics.java +++ b/src/main/java/org/apache/sysds/utils/Statistics.java @@ -674,6 +674,8 @@ public static String display(int maxHeavyHitters) if(DMLScript.FED_STATISTICS) { sb.append("\n"); sb.append(FederatedStatistics.displayStatistics(DMLScript.FED_STATISTICS_COUNT)); + sb.append("\n"); + sb.append(ParamServStatistics.displayFloStatistics()); } return sb.toString(); diff --git a/src/main/java/org/apache/sysds/utils/stats/ParamServStatistics.java b/src/main/java/org/apache/sysds/utils/stats/ParamServStatistics.java index 0d97bfd0c63..8eb26a19637 100644 --- a/src/main/java/org/apache/sysds/utils/stats/ParamServStatistics.java +++ b/src/main/java/org/apache/sysds/utils/stats/ParamServStatistics.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.LongAdder; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing; public class ParamServStatistics { @@ -41,6 +42,14 @@ public class ParamServStatistics { private static final LongAdder fedWorkerComputingTime = new LongAdder(); private static final LongAdder fedGradientWeightingTime = new LongAdder(); private static final LongAdder fedCommunicationTime = new LongAdder(); + private static final LongAdder fedNetworkTime = new LongAdder(); // measures exactly how long it takes netty to send & receive data + // Homomorphic encryption specifics (time is in milli sec) + private static final LongAdder heEncryption = new LongAdder(); // SEALClient::encrypt + private static final LongAdder heAccumulation = new LongAdder(); // SEALServer::accumulateCiphertexts + private static final LongAdder hePartialDecryption = new LongAdder(); // SEALClient::partiallyDecrypt + private static final LongAdder heDecryption = new LongAdder(); // SEALServer::average + + private static final LongAdder fedAggregation = new LongAdder(); // SEALServer::average public static void incWorkerNumber() { numWorkers.increment(); @@ -110,6 +119,14 @@ public static void accFedWorkerComputing(long t) { fedWorkerComputingTime.add(t); } + public static void accFedNetworkTime(long t) { + fedNetworkTime.add(t); + } + + public static void accFedAggregation(long t) { + fedAggregation.add(t); + } + public static void accFedGradientWeightingTime(long t) { fedGradientWeightingTime.add(t); } @@ -118,6 +135,22 @@ public static void accFedCommunicationTime(long t) { fedCommunicationTime.add(t); } + public static void accHEEncryptionTime(long t) { + heEncryption.add(t); + } + + public static void accHEAccumulation(long t) { + heAccumulation.add(t); + } + + public static void accHEPartialDecryptionTime(long t) { + hePartialDecryption.add(t); + } + + public static void accHEDecryptionTime(long t) { + heDecryption.add(t); + } + public static void reset() { executionTime.reset(); numWorkers.reset(); @@ -133,6 +166,12 @@ public static void reset() { fedWorkerComputingTime.reset(); fedGradientWeightingTime.reset(); fedCommunicationTime.reset(); + fedNetworkTime.reset(); + heEncryption.reset(); + heAccumulation.reset(); + hePartialDecryption.reset(); + heDecryption.reset(); + fedAggregation.reset(); } public static String displayStatistics() { @@ -168,4 +207,16 @@ private static String displayFedPSStatistics() { sb.append(String.format("PS fed grad. weigh. time (cum):\t%.3f secs.\n", fedGradientWeightingTime.doubleValue() / 1000)); return sb.toString(); } + + public static String displayFloStatistics() { + StringBuilder sb = new StringBuilder(); + sb.append(String.format("PS fed network time (cum):\t%.3f secs.\n", fedNetworkTime.doubleValue() / 1000)); + sb.append(String.format("PS fed agg time:\t%.3f secs.\n", fedAggregation.doubleValue() / 1000)); + sb.append(String.format("Paramserv grad compute time:\t%.3f secs.\n", gradientComputeTime.doubleValue() / 1000)); + sb.append(String.format("HE PS encryption time:\t%.3f secs.\n", heEncryption.doubleValue() / 1000)); + sb.append(String.format("HE PS accumulation time:\t%.3f secs.\n", heAccumulation.doubleValue() / 1000)); + sb.append(String.format("HE PS partial decryption time:\t%.3f secs.\n", hePartialDecryption.doubleValue() / 1000)); + sb.append(String.format("HE PS decryption time:\t%.3f secs.\n", heDecryption.doubleValue() / 1000)); + return sb.toString(); + } } diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 6ebff8eacd0..c5f7d1a54bb 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -652,6 +652,12 @@ protected void writeFederatedInputMatrix(String name, FederationMap fedMap){ */ protected void rowFederateLocallyAndWriteInputMatrixWithMTD(String name, double[][] matrix, int numFederatedWorkers, List<Integer> ports, double[][] ranges) + { + rowFederateLocallyAndWriteInputMatrixWithMTD(name, matrix, numFederatedWorkers, ports, ranges, null); + } + + protected void rowFederateLocallyAndWriteInputMatrixWithMTD(String name, + double[][] matrix, int numFederatedWorkers, List<Integer> ports, double[][] ranges, PrivacyConstraint privacyConstraint) { // check matrix non empty if(matrix.length == 0 || matrix[0].length == 0) @@ -677,7 +683,7 @@ protected void rowFederateLocallyAndWriteInputMatrixWithMTD(String name, // write slice writeInputMatrixWithMTD(path, Arrays.copyOfRange(matrix, (int)lowerBound, (int)upperBound), false, new MatrixCharacteristics((long) examplesForWorkerI, ncol, - OptimizerUtils.DEFAULT_BLOCKSIZE, (long) examplesForWorkerI * ncol)); + OptimizerUtils.DEFAULT_BLOCKSIZE, (long) examplesForWorkerI * ncol), privacyConstraint); // generate fedmap entry FederatedRange range = new FederatedRange(new long[]{(long) lowerBound, 0}, new long[]{(long) upperBound, ncol}); @@ -688,7 +694,7 @@ false, new MatrixCharacteristics((long) examplesForWorkerI, ncol, federatedMatrixObject.setFedMapping(new FederationMap(FederationUtils.getNextFedDataID(), fedHashMap)); federatedMatrixObject.getFedMapping().setType(FType.ROW); - writeInputFederatedWithMTD(name, federatedMatrixObject, null); + writeInputFederatedWithMTD(name, federatedMatrixObject, privacyConstraint); } protected double[][] generateBalancedFederatedRowRanges(int numFederatedWorkers, int dataSetSize) { diff --git a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java new file mode 100644 index 00000000000..25bc5b4ae77 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.federated.paramserv; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.hops.codegen.SpoofCompiler; +import org.apache.sysds.runtime.controlprogram.paramserv.NativeHEHelper; +import org.apache.sysds.runtime.privacy.PrivacyConstraint; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.utils.Statistics; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(value = Parameterized.class) +@net.jcip.annotations.NotThreadSafe +public class EncryptedFederatedParamservTest extends AutomatedTestBase { + // private static final Log LOG = LogFactory.getLog(EncryptedFederatedParamservTest.class.getName()); + private final static String TEST_DIR = "functions/federated/paramserv/"; + private final static String TEST_NAME = "EncryptedFederatedParamservTest"; + private final static String TEST_CLASS_DIR = TEST_DIR + EncryptedFederatedParamservTest.class.getSimpleName() + "/"; + + private final String _networkType; + private final int _numFederatedWorkers; + private final int _dataSetSize; + private final int _epochs; + private final int _batch_size; + private final double _eta; + private final String _utype; + private final String _freq; + private final String _scheme; + private final String _runtime_balancing; + private final String _weighting; + private final String _data_distribution; + private final int _seed; + + // parameters + @Parameterized.Parameters + public static Collection<Object[]> parameters() { + return Arrays.asList(new Object[][] { + // Network type, number of federated workers, data set size, batch size, epochs, learning rate, update type, update frequency + // basic functionality + //{"TwoNN", 4, 60000, 32, 4, 0.01, "BSP", "BATCH", "KEEP_DATA_ON_WORKER", "NONE" , "false","BALANCED", 200}, + + // One important point is that we do the model averaging in the case of BSP + {"TwoNN", 2, 4, 1, 1, 0.01, "BSP", "BATCH", "KEEP_DATA_ON_WORKER", "BASELINE", "false", "IMBALANCED", 200}, + {"CNN", 2, 4, 1, 1, 0.01, "BSP", "EPOCH", "KEEP_DATA_ON_WORKER", "BASELINE", "false", "IMBALANCED", 200}, + //{"TwoNN", 5, 1000, 100, 1, 0.01, "BSP", "BATCH", "KEEP_DATA_ON_WORKER", "NONE", "true", "BALANCED", 200}, + + /* + // runtime balancing + {"TwoNN", 2, 4, 1, 4, 0.01, "BSP", "BATCH", "KEEP_DATA_ON_WORKER", "CYCLE_MIN", "true", "IMBALANCED", 200}, + {"TwoNN", 2, 4, 1, 4, 0.01, "BSP", "EPOCH", "KEEP_DATA_ON_WORKER", "CYCLE_MIN", "true", "IMBALANCED", 200}, + {"TwoNN", 2, 4, 1, 4, 0.01, "BSP", "BATCH", "KEEP_DATA_ON_WORKER", "CYCLE_AVG", "true", "IMBALANCED", 200}, + {"TwoNN", 2, 4, 1, 4, 0.01, "BSP", "EPOCH", "KEEP_DATA_ON_WORKER", "CYCLE_AVG", "true", "IMBALANCED", 200}, + {"TwoNN", 2, 4, 1, 4, 0.01, "BSP", "BATCH", "KEEP_DATA_ON_WORKER", "CYCLE_MAX", "true", "IMBALANCED", 200}, + {"TwoNN", 2, 4, 1, 4, 0.01, "BSP", "EPOCH", "KEEP_DATA_ON_WORKER", "CYCLE_MAX", "true", "IMBALANCED", 200}, + + // data partitioning + {"TwoNN", 2, 4, 1, 1, 0.01, "BSP", "BATCH", "SHUFFLE", "CYCLE_AVG", "true", "IMBALANCED", 200}, + {"TwoNN", 2, 4, 1, 1, 0.01, "BSP", "BATCH", "REPLICATE_TO_MAX", "NONE", "true", "IMBALANCED", 200}, + {"TwoNN", 2, 4, 1, 1, 0.01, "BSP", "BATCH", "SUBSAMPLE_TO_MIN", "NONE", "true", "IMBALANCED", 200}, + {"TwoNN", 2, 4, 1, 1, 0.01, "BSP", "BATCH", "BALANCE_TO_AVG", "NONE", "true", "IMBALANCED", 200}, + + // balanced tests + {"CNN", 5, 1000, 100, 2, 0.01, "BSP", "EPOCH", "KEEP_DATA_ON_WORKER", "NONE", "true", "BALANCED", 200} + */ + }); + } + + public EncryptedFederatedParamservTest(String networkType, int numFederatedWorkers, int dataSetSize, int batch_size, + int epochs, double eta, String utype, String freq, String scheme, String runtime_balancing, String weighting, String data_distribution, int seed) { + try { + NativeHEHelper.initialize(); + } catch (Exception e) { + throw e; + } + _networkType = networkType; + _numFederatedWorkers = numFederatedWorkers; + _dataSetSize = dataSetSize; + _batch_size = batch_size; + _epochs = epochs; + _eta = eta; + _utype = utype; + _freq = freq; + _scheme = scheme; + _runtime_balancing = runtime_balancing; + _weighting = weighting; + _data_distribution = data_distribution; + _seed = seed; + } + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME)); + } + + @Test + public void EncryptedfederatedParamservSingleNode() { + EncryptedfederatedParamserv(ExecMode.SINGLE_NODE, true); + } + + @Test + public void EncryptedfederatedParamservHybrid() { + EncryptedfederatedParamserv(ExecMode.HYBRID, true); + } + + private void EncryptedfederatedParamserv(ExecMode mode, boolean modelAvg) { + // Warning Statistics accumulate in unit test + // config + getAndLoadTestConfiguration(TEST_NAME); + String HOME = SCRIPT_DIR + TEST_DIR; + setOutputBuffering(true); + + int C = 1, Hin = 28, Win = 28; + int numLabels = 10; + + ExecMode platformOld = setExecMode(mode); + + try { + // start threads + List<Integer> ports = new ArrayList<>(); + List<Thread> threads = new ArrayList<>(); + for(int i = 0; i < _numFederatedWorkers; i++) { + ports.add(getRandomAvailablePort()); + threads.add(startLocalFedWorkerThread(ports.get(i), + i==(_numFederatedWorkers-1) ? FED_WORKER_WAIT : FED_WORKER_WAIT_S)); + } + + // generate test data + double[][] features = generateDummyMNISTFeatures(_dataSetSize, C, Hin, Win); + double[][] labels = generateDummyMNISTLabels(_dataSetSize, numLabels); + String featuresName = ""; + String labelsName = ""; + + PrivacyConstraint privacyConstraint = new PrivacyConstraint(PrivacyConstraint.PrivacyLevel.Private); + + // federate test data balanced or imbalanced + if(_data_distribution.equals("IMBALANCED")) { + featuresName = "X_IMBALANCED_" + _numFederatedWorkers; + labelsName = "y_IMBALANCED_" + _numFederatedWorkers; + double[][] ranges = {{0,1}, {1,4}}; + rowFederateLocallyAndWriteInputMatrixWithMTD(featuresName, features, _numFederatedWorkers, ports, ranges, privacyConstraint); + rowFederateLocallyAndWriteInputMatrixWithMTD(labelsName, labels, _numFederatedWorkers, ports, ranges, privacyConstraint); + } + else { + featuresName = "X_BALANCED_" + _numFederatedWorkers; + labelsName = "y_BALANCED_" + _numFederatedWorkers; + double[][] ranges = generateBalancedFederatedRowRanges(_numFederatedWorkers, features.length); + rowFederateLocallyAndWriteInputMatrixWithMTD(featuresName, features, _numFederatedWorkers, ports, ranges, privacyConstraint); + rowFederateLocallyAndWriteInputMatrixWithMTD(labelsName, labels, _numFederatedWorkers, ports, ranges, privacyConstraint); + } + + try { + //wait for all workers to be setup + Thread.sleep(FED_WORKER_WAIT); + } + catch(InterruptedException e) { + e.printStackTrace(); + } + + // dml name + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + // generate program args + List<String> programArgsList = new ArrayList<>(Arrays.asList("-stats", + "-nvargs", + "features=" + input(featuresName), + "labels=" + input(labelsName), + "epochs=" + _epochs, + "batch_size=" + _batch_size, + "eta=" + _eta, + "utype=" + _utype, + "freq=" + _freq, + "scheme=" + _scheme, + "runtime_balancing=" + _runtime_balancing, + "weighting=" + _weighting, + "network_type=" + _networkType, + "channels=" + C, + "hin=" + Hin, + "win=" + Win, + "seed=" + _seed, + "modelAvg=" + Boolean.toString(modelAvg).toUpperCase())); + + programArgs = programArgsList.toArray(new String[0]); + String log = runTest(null).toString(); + Assert.assertEquals("Test Failed \n" + log, 0, Statistics.getNoOfExecutedSPInst()); + + // shut down threads + for(int i = 0; i < _numFederatedWorkers; i++) { + TestUtils.shutdownThreads(threads.get(i)); + } + } + finally { + resetExecMode(platformOld); + } + } + + /** + * Generates an feature matrix that has the same format as the MNIST dataset, + * but is completely random and normalized + * + * @param numExamples Number of examples to generate + * @param C Channels in the input data + * @param Hin Height in Pixels of the input data + * @param Win Width in Pixels of the input data + * @return a dummy MNIST feature matrix + */ + private double[][] generateDummyMNISTFeatures(int numExamples, int C, int Hin, int Win) { + // Seed -1 takes the time in milliseconds as a seed + // Sparsity 1 means no sparsity + return getRandomMatrix(numExamples, C*Hin*Win, 0, 1, 1, -1); + } + + /** + * Generates an label matrix that has the same format as the MNIST dataset, but is completely random and consists + * of one hot encoded vectors as rows + * + * @param numExamples Number of examples to generate + * @param numLabels Number of labels to generate + * @return a dummy MNIST lable matrix + */ + private double[][] generateDummyMNISTLabels(int numExamples, int numLabels) { + // Seed -1 takes the time in milliseconds as a seed + // Sparsity 1 means no sparsity + return getRandomMatrix(numExamples, numLabels, 0, 1, 1, -1); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java b/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java new file mode 100644 index 00000000000..58600887c71 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.homomorphicEncryption; + +import org.apache.sysds.common.Types; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.controlprogram.paramserv.NativeHEHelper; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.PublicKey; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.SEALClient; +import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.SEALServer; +import org.apache.sysds.runtime.instructions.cp.CiphertextMatrix; +import org.apache.sysds.runtime.instructions.cp.PlaintextMatrix; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.meta.MetaDataFormat; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +public class InOutTest extends AutomatedTestBase { + private final static String TEST_NAME = "InOutTest"; + private final static String TEST_DIR = "functions/data/"; + private final static String TEST_CLASS_DIR = TEST_DIR + InOutTest.class.getSimpleName() + "/"; + + private final int num_clients = 3; + + private final int rows = 100; + private final int cols = 200; + private final long seed = 42; + + @Override + public void setUp() { + try { + NativeHEHelper.initialize(); + } catch (Exception e) { + throw e; + } + + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "C" }) ); + } + + @Test + public void endToEndTest() { + SEALServer server = new SEALServer(); + + SEALClient[] clients = new SEALClient[num_clients]; + PublicKey[] partial_pub_keys = new PublicKey[num_clients]; + for (int i = 0; i < num_clients; i++) { + clients[i] = new SEALClient(server.generateA()); + partial_pub_keys[i] = clients[i].generatePartialPublicKey(); + } + + PublicKey public_key = server.aggregatePartialPublicKeys(partial_pub_keys); + + MatrixObject[] plaintexts = new MatrixObject[num_clients]; + CiphertextMatrix[] ciphertexts = new CiphertextMatrix[num_clients]; + for (int i = 0; i < num_clients; i++) { + MatrixBlock mb = TestUtils.generateTestMatrixBlock(rows, cols, -100, 100, 1.0, seed+i); + MatrixObject mo = new MatrixObject(Types.ValueType.FP64, null); + mo.setMetaData(new MetaDataFormat(new MatrixCharacteristics(rows, cols), Types.FileFormat.BINARY)); + mo.acquireModify(mb); + mo.release(); + plaintexts[i] = mo; + + clients[i].setPublicKey(public_key); + ciphertexts[i] = clients[i].encrypt(plaintexts[i]); + } + + CiphertextMatrix encrypted_sum = server.accumulateCiphertexts(ciphertexts); + + PlaintextMatrix[] partial_decryptions = new PlaintextMatrix[num_clients]; + for (int i = 0; i < num_clients; i++) { + partial_decryptions[i] = clients[i].partiallyDecrypt(encrypted_sum); + } + + MatrixObject result = server.average(encrypted_sum, partial_decryptions); + + double[] expected_raw_result = new double[rows*cols]; + double[][] plaintexts_raw = new double[num_clients][]; + for (int i = 0; i < num_clients; i++) { + plaintexts_raw[i] = plaintexts[i].acquireReadAndRelease().getDenseBlockValues(); + } + for (int x = 0; x < rows * cols; x++) { + double sum = 0.0; + for (int i = 0; i < num_clients; i++) { + sum += plaintexts_raw[i][x]; + } + expected_raw_result[x] = sum / num_clients; + } + + double[] raw_result = result.acquireReadAndRelease().getDenseBlockValues(); + assert result.getNumRows() == rows; + assert result.getNumColumns() == cols; + assert raw_result.length == rows*cols; + TestUtils.compareMatrices(raw_result, expected_raw_result, 5e-8); + } +} diff --git a/src/test/scripts/functions/federated/paramserv/EncryptedFederatedParamservTest.dml b/src/test/scripts/functions/federated/paramserv/EncryptedFederatedParamservTest.dml new file mode 100644 index 00000000000..b8021867dc9 --- /dev/null +++ b/src/test/scripts/functions/federated/paramserv/EncryptedFederatedParamservTest.dml @@ -0,0 +1,61 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +source("src/test/scripts/functions/federated/paramserv/TwoNN.dml") as TwoNN +source("src/test/scripts/functions/federated/paramserv/TwoNNModelAvg.dml") as TwoNNModelAvg +source("src/test/scripts/functions/federated/paramserv/CNN.dml") as CNN +source("src/test/scripts/functions/federated/paramserv/CNNModelAvg.dml") as CNNModelAvg + + +# create federated input matrices +features = read($features) +labels = read($labels) + +if($network_type == "TwoNN") { + if(!as.logical($modelAvg)) { + model = TwoNN::train_paramserv(features, labels, matrix(0, rows=100, cols=784), matrix(0, rows=100, cols=10), 0, $epochs, $utype, $freq, $batch_size, $scheme, $runtime_balancing, $weighting, $eta, $seed) + print("Test results:") + [loss_test, accuracy_test] = TwoNN::validate(matrix(0, rows=100, cols=784), matrix(0, rows=100, cols=10), model, list()) + print("[+] test loss: " + loss_test + ", test accuracy: " + accuracy_test + "\n") + } + else if (as.logical($modelAvg)){ + model = TwoNNModelAvg::train_paramserv(features, labels, matrix(0, rows=100, cols=784), matrix(0, rows=100, cols=10), 0, $epochs, $utype, $freq, $batch_size, $scheme, $runtime_balancing, $weighting, $eta, $seed, $modelAvg) + print("Test results:") + [loss_test, accuracy_test] = TwoNNModelAvg::validate(matrix(0, rows=100, cols=784), matrix(0, rows=100, cols=10), model, list()) + print("[+] test loss: " + loss_test + ", test accuracy: " + accuracy_test + "\n") + } +} +else if($network_type == "CNN") { + if(!as.logical($modelAvg)) { + model = CNN::train_paramserv(features, labels, matrix(0, rows=100, cols=784), matrix(0, rows=100, cols=10), 0, $epochs, $utype, $freq, $batch_size, $scheme, $runtime_balancing, $weighting, $eta, $channels, $hin, $win, $seed) + print("Test results:") + hyperparams = list(learning_rate=$eta, C=$channels, Hin=$hin, Win=$win) + [loss_test, accuracy_test] = CNN::validate(matrix(0, rows=100, cols=784), matrix(0, rows=100, cols=10), model, hyperparams) + print("[+] test loss: " + loss_test + ", test accuracy: " + accuracy_test + "\n") + } + else if (as.logical($modelAvg)){ + model = CNNModelAvg::train_paramserv(features, labels, matrix(0, rows=100, cols=784), matrix(0, rows=100, cols=10), 0, $epochs, $utype, $freq, $batch_size, $scheme, $runtime_balancing, $weighting, $eta, $channels, $hin, $win, $seed, $modelAvg) + print("Test results:") + hyperparams = list(learning_rate=$eta, C=$channels, Hin=$hin, Win=$win) + [loss_test, accuracy_test] = CNNModelAvg::validate(matrix(0, rows=100, cols=784), matrix(0, rows=100, cols=10), model, hyperparams) + print("[+] test loss: " + loss_test + ", test accuracy: " + accuracy_test + "\n") + } +} From a718ccebcd34b4f88c15c00f1fbb93b98b69b9e6 Mon Sep 17 00:00:00 2001 From: sebwrede <swrede@know-center.at> Date: Thu, 2 Jun 2022 19:12:00 +0200 Subject: [PATCH 2/3] Add Licenses --- src/main/cpp/he/he.cpp | 21 ++++++++++++++++++- .../paramserv/NativeHEHelper.java | 19 +++++++++++++++++ .../paramserv/NetworkTrafficCounter.java | 19 +++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/main/cpp/he/he.cpp b/src/main/cpp/he/he.cpp index f9bad7e9846..71ff571af12 100644 --- a/src/main/cpp/he/he.cpp +++ b/src/main/cpp/he/he.cpp @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + #include "he.h" #include "libhe.h" @@ -276,4 +295,4 @@ JNIEXPORT jdoubleArray JNICALL Java_org_apache_sysds_runtime_controlprogram_para } return result; -} \ No newline at end of file +} diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NativeHEHelper.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NativeHEHelper.java index 7757ad722bb..38e4dec553b 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NativeHEHelper.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NativeHEHelper.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.sysds.runtime.controlprogram.paramserv; import org.apache.commons.lang.SystemUtils; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NetworkTrafficCounter.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NetworkTrafficCounter.java index d0ba01dba93..f823b9d3be3 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NetworkTrafficCounter.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/NetworkTrafficCounter.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.sysds.runtime.controlprogram.paramserv; import io.netty.channel.ChannelHandler; From 2664c6a0266c69fad8c032e89ff09ca5bb2221b4 Mon Sep 17 00:00:00 2001 From: sebwrede <swrede@know-center.at> Date: Thu, 2 Jun 2022 21:45:08 +0200 Subject: [PATCH 3/3] Ignore HE Tests --- .../federated/paramserv/EncryptedFederatedParamservTest.java | 3 +++ .../sysds/test/functions/homomorphicEncryption/InOutTest.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java index 25bc5b4ae77..250358d4087 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java @@ -33,6 +33,7 @@ import org.apache.sysds.test.TestUtils; import org.apache.sysds.utils.Statistics; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -122,11 +123,13 @@ public void setUp() { } @Test + @Ignore public void EncryptedfederatedParamservSingleNode() { EncryptedfederatedParamserv(ExecMode.SINGLE_NODE, true); } @Test + @Ignore public void EncryptedfederatedParamservHybrid() { EncryptedfederatedParamserv(ExecMode.HYBRID, true); } diff --git a/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java b/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java index 58600887c71..5bb952d9ed9 100644 --- a/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java +++ b/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java @@ -33,6 +33,7 @@ import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; +import org.junit.Ignore; import org.junit.Test; public class InOutTest extends AutomatedTestBase { @@ -59,6 +60,7 @@ public void setUp() { } @Test + @Ignore public void endToEndTest() { SEALServer server = new SEALServer();