diff --git a/algorithms.py b/algorithms.py index 22a53ba..65d06b6 100644 --- a/algorithms.py +++ b/algorithms.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from abc import ABC, abstractmethod +import os import time import pandas as pd import numpy as np @@ -311,9 +312,21 @@ def get_slices(self, n_slices, X, y): def fit(self, data, args): params = self.configure(data, args) - n_workers = None if args.gpus < 0 else args.gpus - cluster = LocalCUDACluster(n_workers=n_workers, - local_directory=args.root) + clusterargs={ + 'n_workers':int(os.environ.get("NUM_WORKERS", None if args.gpus < 0 else args.gpus)), + 'local_directory':args.root, + 'memory_limit':os.environ.get("DEVICE_MEMORY_LIMIT", None), + 'device_memory_limit':os.getenv('DASK_DEVICE_MEMORY_LIMIT',None), + 'protocol':"ucx" if os.environ.get("CLUSTER_MODE", "TCP")=="NVLINK" else "tcp", + 'enable_tcp_over_ucx':os.environ.get("CLUSTER_MODE", "TCP")=="NVLINK", + 'enable_nvlink':os.environ.get("CLUSTER_MODE", "TCP")=="NVLINK", + 'enable_infiniband':os.getenv('CLUSTER_CONFIG_TYPE', "").endswith("ib"), + 'enable_rdmacm':bool(os.getenv('ENABLE_RDMACM', False)), + 'jit_unspill':True, + 'rmm_pool_size':os.environ.get("POOL_SIZE", "29GB") + } + + cluster = LocalCUDACluster( n_workers=clusterargs['n_workers'], ) client = Client(cluster) n_partitions = len(client.scheduler_info()['workers']) X_sliced, y_sliced = self.get_slices(n_partitions, diff --git a/runme.py b/runme.py index 89ff84b..f3dc1a0 100755 --- a/runme.py +++ b/runme.py @@ -31,6 +31,7 @@ import json import ast import psutil +import datetime import algorithms from metrics import get_metrics from datasets import prepare_dataset @@ -126,6 +127,7 @@ def main(): args.cpus = get_number_processors(args) args.extra = ast.literal_eval(args.extra) print_sys_info(args) + ts=datetime.datetime.utcnow().strftime('%Y%m%d.%H%M%S%f') if args.warmup: benchmark(args, os.path.join(args.root, "fraud"), "fraud") if args.dataset == 'all': @@ -133,7 +135,8 @@ def main(): results = {} for dataset in args.dataset.split(","): folder = os.path.join(args.root, dataset) - results.update({dataset: benchmark(args, folder, dataset)}) + results.update({ 'timestamp_utc': ts, + dataset: benchmark(args, folder, dataset)}) print(json.dumps({dataset: results[dataset]}, indent=2, sort_keys=True)) output = json.dumps(results, indent=2, sort_keys=True) output_file = open(args.output, "w")