From 6334d1f75201871b13f9f5b46d6b6c3b51fcaa92 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 20:57:15 -0500 Subject: [PATCH 001/113] adding code to make ddp work --- main.py | 57 +++++++++++++++++++++++++++------------------------ main_utils.py | 29 ++++++++------------------ 2 files changed, 39 insertions(+), 47 deletions(-) diff --git a/main.py b/main.py index cc1af83e..90ea1070 100644 --- a/main.py +++ b/main.py @@ -6,56 +6,58 @@ def main(): print("\n\nBeginning of process.") print_time() set_seed(parser_args.seed * parser_args.trial_num) - #set_seed(parser_args.seed + parser_args.trial_num - 1) + # set_seed(parser_args.seed + parser_args.trial_num - 1) - # parser_args.distributed = parser_args.world_size > 1 or parser_args.multiprocessing_distributed + # world size = ngpus_per_node since we are assuming single node ngpus_per_node = torch.cuda.device_count() if parser_args.multiprocessing_distributed: - setup_distributed(ngpus_per_node) - mp.spawn(main_worker, nprocs=ngpus_per_node, - args=(ngpus_per_node,), join=True) + assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" + mp.spawn(main_worker, args=(ngpus_per_node,), nprocs=ngpus_per_node, join=True) else: # Simply call main_worker function main_worker(parser_args.gpu, ngpus_per_node) def main_worker(gpu, ngpus_per_node): - train, validate, modifier = get_trainer(parser_args) + # NOTE: gpu = rank in the multiprocessing setting parser_args.gpu = gpu + if parser_args.gpu is not None: print("Use GPU: {} for training".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: - parser_args.rank = parser_args.rank * ngpus_per_node + parser_args.gpu - # When using a single GPU per process and per DistributedDataParallel, we need to divide the batch size - # ourselves based on the total number of GPUs we have + parser_args.rank = parser_args.gpu + setup_distributed(rank, ngpus_per_node) + # if using ddp, divide batch size per gpu parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) - parser_args.num_workers = int( - (parser_args.num_workers + ngpus_per_node - 1) / ngpus_per_node) - # Since we have ngpus_per_node processes per node, the total world_size - # needs to be adjusted accordingly - parser_args.world_size = ngpus_per_node * parser_args.world_size - idty_str = get_idty_str(parser_args) - if parser_args.subfolder is not None: - if not os.path.isdir('results/'): - os.mkdir('results/') - result_subroot = 'results/' + parser_args.subfolder + '/' - if not os.path.isdir(result_subroot): - os.mkdir(result_subroot) - result_root = result_subroot + '/results_' + idty_str + '/' - else: - result_root = 'results/results_' + idty_str + '/' - if not os.path.isdir(result_root): - os.mkdir(result_root) + train, validate, modifier = get_trainer(parser_args) model = get_model(parser_args) - print_model(model, parser_args) + + if parser_args == 0: + idty_str = get_idty_str(parser_args) + if parser_args.subfolder is not None: + if not os.path.isdir('results/'): + os.mkdir('results/') + result_subroot = 'results/' + parser_args.subfolder + '/' + if not os.path.isdir(result_subroot): + os.mkdir(result_subroot) + result_root = result_subroot + '/results_' + idty_str + '/' + else: + result_root = 'results/results_' + idty_str + '/' + + if not os.path.isdir(result_root): + os.mkdir(result_root) + print_model(model, parser_args) if parser_args.weight_training: model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) model = switch_to_wt(model) + model = set_gpu(parser_args, model) + if parser_args.pretrained: pretrained(parser_args.pretrained, model) if parser_args.pretrained2: @@ -64,6 +66,7 @@ def main_worker(gpu, ngpus_per_node): pretrained(parser_args.pretrained2, model2) else: model2 = None + optimizer = get_optimizer(parser_args, model) data = get_dataset(parser_args) scheduler = get_scheduler(optimizer, parser_args.lr_policy) diff --git a/main_utils.py b/main_utils.py index 7f54a7cd..193e9661 100644 --- a/main_utils.py +++ b/main_utils.py @@ -504,15 +504,12 @@ def get_mask(model): return mask, flat_tensor -def setup_distributed(ngpus_per_node): - # for debugging - # os.environ['NCCL_DEBUG'] = 'INFO' - # os.environ['TORCH_DISTRIBUTED_DEBUG'] = 'INFO' - - # setup environment +def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' os.environ['MASTER_PORT'] = '29500' + dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) + def cleanup_distributed(): torch.distributed.destroy_process_group() @@ -886,21 +883,13 @@ def get_trainer(parser_args): def set_gpu(parser_args, model): assert torch.cuda.is_available(), "CPU-only experiments currently unsupported" - if parser_args.gpu is not None: - torch.cuda.set_device(parser_args.gpu) - model.cuda(parser_args.gpu) + # torch.cuda.set_device(parser_args.gpu) + model.to(parser_args.gpu) - if parser_args.multiprocessing_distributed: - torch.distributed.init_process_group( - backend=parser_args.dist_backend, - init_method='env://', - world_size=parser_args.world_size, - rank=parser_args.rank - ) - model = nn.parallel.DistributedDataParallel( - model, device_ids=[parser_args.gpu], find_unused_parameters=True) - else: - device = torch.device("cpu") + if parser_args.multiprocessing_distributed: + # TODO: not sure about find_unused_parameters. Need to check + model = nn.parallel.DistributedDataParallel( + model, device_ids=[parser_args.gpu], find_unused_parameters=True) return model From 9e1f0de4d9aa9c1098ce22db3bd81930159d9e64 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 21:06:28 -0500 Subject: [PATCH 002/113] trying out how args work in ddp --- ddp_args_helper.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ ddp_poc.py | 2 ++ 2 files changed, 55 insertions(+) create mode 100644 ddp_args_helper.py diff --git a/ddp_args_helper.py b/ddp_args_helper.py new file mode 100644 index 00000000..73324903 --- /dev/null +++ b/ddp_args_helper.py @@ -0,0 +1,53 @@ +import argparse +import sys +import yaml + +from configs import parser as _parser + +global parser_args + +class ArgsHelper: + def parse_arguments(self, jupyter_mode=False): + parser = argparse.ArgumentParser(description="Pruning random networks") + + # Config/Hyperparameters + parser.add_argument( + "--gpu", + type=int, + default=0, + help="gpu" + ) + parser.add_argument( + "--name", + default=None, + type=str, + help="Name of experiment" + ) + + if jupyter_mode: + args = parser.parse_args("") + else: + args = parser.parse_args() + self.get_config(args, jupyter_mode) + + return args + + def isNotebook(self): + try: + shell = get_ipython().__class__.__name__ + if shell == 'ZMQInteractiveShell': + return True # Jupyter notebook or qtconsole + elif shell == 'TerminalInteractiveShell': + return False # Terminal running IPython + else: + return False # Other type (?) + except NameError: + return False # Probably standard Python interpreter + + def get_args(self, jupyter_mode=False): + global parser_args + jupyter_mode = self.isNotebook() + parser_args = self.parse_arguments(jupyter_mode) + +argshelper = ArgsHelper() +argshelper.get_args() diff --git a/ddp_poc.py b/ddp_poc.py index 04a21e52..f06a311e 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -13,6 +13,7 @@ import re from torch.nn.parallel import DistributedDataParallel as DDP +from ddp_args_helper import parser_args def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' @@ -60,6 +61,7 @@ def get_model_norm(model): def demo_basic(rank, world_size): print(f"Running basic DDP example on rank {rank}.") + print("Parser args: gpu={}, name={}".format(parser_args.gpu, parser_args.name)) setup(rank, world_size) # create model and move it to GPU with id rank From f616ab6117e55030bba9489d1f63fcda6c0c5ff6 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 21:12:12 -0500 Subject: [PATCH 003/113] trying out stuff with parser_args in ddp setting --- ddp_args_helper.py | 3 +-- ddp_poc.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ddp_args_helper.py b/ddp_args_helper.py index 73324903..04909374 100644 --- a/ddp_args_helper.py +++ b/ddp_args_helper.py @@ -19,7 +19,7 @@ def parse_arguments(self, jupyter_mode=False): ) parser.add_argument( "--name", - default=None, + default="blah", type=str, help="Name of experiment" ) @@ -28,7 +28,6 @@ def parse_arguments(self, jupyter_mode=False): args = parser.parse_args("") else: args = parser.parse_args() - self.get_config(args, jupyter_mode) return args diff --git a/ddp_poc.py b/ddp_poc.py index f06a311e..be34223e 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -62,6 +62,8 @@ def get_model_norm(model): def demo_basic(rank, world_size): print(f"Running basic DDP example on rank {rank}.") print("Parser args: gpu={}, name={}".format(parser_args.gpu, parser_args.name)) + print("Setting gpu now, let's see what happens") + parser_args.gpu = rank setup(rank, world_size) # create model and move it to GPU with id rank @@ -96,6 +98,7 @@ def demo_basic(rank, world_size): for epoch in range(15): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) + print("Local Rank: {} | Parser args: gpu={}, Name={}".format(parser_args.gpu, parser_args.name)) if epoch % 3 == 0: # prune model print("Rank: {} | Gonna try to prune model".format(rank)) From e6b93a698bbd6bfe35fce0766972d6d0336ce2a7 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 21:16:06 -0500 Subject: [PATCH 004/113] adding ddp_utils --- ddp_poc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ddp_poc.py b/ddp_poc.py index be34223e..6fb105ce 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -14,6 +14,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP from ddp_args_helper import parser_args +from ddp_utils import do_something def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' @@ -131,6 +132,8 @@ def demo_basic(rank, world_size): loss.backward() optimizer.step() print("End of epoch total batch sizes: {}".format(total_data_size)) + + do_something_outside() # optimizer.zero_grad() # outputs = ddp_model(torch.randn(20, 10)) # labels = torch.randn(20, 5).to(rank) From f202a42886f6fc60f83ec7fdc2ce8c6c9609d7dd Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 21:17:16 -0500 Subject: [PATCH 005/113] adding ddp_utils --- ddp_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 ddp_utils.py diff --git a/ddp_utils.py b/ddp_utils.py new file mode 100644 index 00000000..0ed4b747 --- /dev/null +++ b/ddp_utils.py @@ -0,0 +1,6 @@ +from args_helper import parser_args + +def do_something(rank): + print("Re-imported parser_args, time to see if something funky happened.") + print("Local Rank: {} | parser_args.gpu={}, parser_args.name={}".format(parser_args.gpu, parser_args.name)) + return -1 \ No newline at end of file From 513b8a6397511fbd8b6550d08d8fce9cdb361f2d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 21:24:55 -0500 Subject: [PATCH 006/113] looks like reimporting doesn't hurt the global values --- ddp_poc.py | 6 +++--- ddp_utils.py | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 6fb105ce..5e291cf0 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -14,7 +14,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP from ddp_args_helper import parser_args -from ddp_utils import do_something +from ddp_utils import do_something_outside def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' @@ -99,7 +99,7 @@ def demo_basic(rank, world_size): for epoch in range(15): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) - print("Local Rank: {} | Parser args: gpu={}, Name={}".format(parser_args.gpu, parser_args.name)) + print("Local Rank: {} | Parser args: gpu={}, Name={}".format(rank, parser_args.gpu, parser_args.name)) if epoch % 3 == 0: # prune model print("Rank: {} | Gonna try to prune model".format(rank)) @@ -133,7 +133,7 @@ def demo_basic(rank, world_size): optimizer.step() print("End of epoch total batch sizes: {}".format(total_data_size)) - do_something_outside() + do_something_outside(rank) # optimizer.zero_grad() # outputs = ddp_model(torch.randn(20, 10)) # labels = torch.randn(20, 5).to(rank) diff --git a/ddp_utils.py b/ddp_utils.py index 0ed4b747..5f7a0c72 100644 --- a/ddp_utils.py +++ b/ddp_utils.py @@ -1,6 +1,5 @@ -from args_helper import parser_args +from ddp_args_helper import parser_args -def do_something(rank): - print("Re-imported parser_args, time to see if something funky happened.") - print("Local Rank: {} | parser_args.gpu={}, parser_args.name={}".format(parser_args.gpu, parser_args.name)) - return -1 \ No newline at end of file +def do_something_outside(rank): + print("Re-imported parser_args, time to see if something funky happened. --> Local Rank: {} | parser_args.gpu={}, parser_args.name={}".format(rank, parser_args.gpu, parser_args.name)) + return -1 From 02a6627c1036644d17040cd8f09c8c791d86424b Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 22:06:26 -0500 Subject: [PATCH 007/113] adding more code to implement ddp --- data/imagenet.py | 52 ++++++++----- main.py | 186 ++++++++++++++++++++++------------------------- main_utils.py | 56 +++++++------- 3 files changed, 151 insertions(+), 143 deletions(-) diff --git a/data/imagenet.py b/data/imagenet.py index e8ec5aee..0379647a 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -2,8 +2,9 @@ import torch from torchvision import datasets, transforms - import torch.multiprocessing +from args_helper import parser_args +from torch.utils.data import random_split torch.multiprocessing.set_sharing_strategy("file_system") @@ -11,13 +12,12 @@ class ImageNet: def __init__(self, args): super(ImageNet, self).__init__() -# data_root = os.path.join(args.data, "imagenet") data_root = args.data use_cuda = torch.cuda.is_available() # Data loading code - kwargs = {"num_workers": args.num_workers, "pin_memory": True} if use_cuda else {} + kwargs = {"num_workers": parser_args.num_workers, "pin_memory": True} if use_cuda else {} # Data loading code traindir = os.path.join(data_root, "train") @@ -27,7 +27,7 @@ def __init__(self, args): mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ) - train_dataset = datasets.ImageFolder( + dataset = datasets.ImageFolder( traindir, transforms.Compose( [ @@ -39,6 +39,27 @@ def __init__(self, args): ), ) + test_dataset = datasets.ImageFolder( + valdir, + transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ] + ), + ) + + if parser_args.use_full_data: + train_dataset = dataset + # use_full_data => we are not tuning hyperparameters + validation_dataset = test_dataset + else: + val_size = 10000 + train_size = len(dataset) - val_size + train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) + if args.multiprocessing_distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: @@ -47,25 +68,22 @@ def __init__(self, args): self.train_loader = torch.utils.data.DataLoader( train_dataset, - batch_size=args.batch_size, + batch_size=parser_args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, **kwargs ) self.val_loader = torch.utils.data.DataLoader( - datasets.ImageFolder( - valdir, - transforms.Compose( - [ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - normalize, - ] - ), - ), - batch_size=args.batch_size, + test_dataset, + batch_size=parser_args.batch_size, shuffle=False, **kwargs ) + + self.actual_val_loader = torch.utils.data.DataLoader( + validation_dataset, + batch_size=parser_args.batch_size, + shuffle=True, + **kwargs + ) diff --git a/main.py b/main.py index 90ea1070..2b7ef2cd 100644 --- a/main.py +++ b/main.py @@ -70,7 +70,7 @@ def main_worker(gpu, ngpus_per_node): optimizer = get_optimizer(parser_args, model) data = get_dataset(parser_args) scheduler = get_scheduler(optimizer, parser_args.lr_policy) - #lr_policy = get_policy(parser_args.lr_policy)(optimizer, parser_args) + # lr_policy = get_policy(parser_args.lr_policy)(optimizer, parser_args) if parser_args.label_smoothing is None: criterion = nn.CrossEntropyLoss().cuda() else: @@ -112,10 +112,9 @@ def main_worker(gpu, ngpus_per_node): print("Overriding prune_rate to {}".format(parser_args.prune_rate)) #if parser_args.dataset == 'TinyImageNet': # print_num_dataset(data) - if not parser_args.weight_training: - print_layers(parser_args, model) - - + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed:: + if not parser_args.weight_training: + print_layers(parser_args, model) if parser_args.mixed_precision: scaler = torch.cuda.amp.GradScaler(enabled=True) # mixed precision @@ -142,7 +141,7 @@ def main_worker(gpu, ngpus_per_node): for epoch in range(parser_args.start_epoch, parser_args.epochs): if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) - #lr_policy(epoch, iteration=None) + # lr_policy(epoch, iteration=None) modifier(parser_args, epoch, model) cur_lr = get_lr(optimizer) @@ -166,37 +165,40 @@ def main_worker(gpu, ngpus_per_node): scheduler.step() # evaluate on validation set - start_validation = time.time() - if parser_args.algo in ['hc', 'hc_iter']: - br_acc1, br_acc5, br_acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding - print('Acc before rounding: {}'.format(br_acc1)) - acc_avg = 0 - for num_trial in range(parser_args.num_test): - cp_model = round_model(model, parser_args.round, noise=parser_args.noise, - ratio=parser_args.noise_ratio, rank=parser_args.gpu) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + start_validation = time.time() + if parser_args.algo in ['hc', 'hc_iter']: + br_acc1, br_acc5, br_acc10 = validate( + data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding + print('Acc before rounding: {}'.format(br_acc1)) + acc_avg = 0 + for num_trial in range(parser_args.num_test): + cp_model = round_model(model, parser_args.round, noise=parser_args.noise, + ratio=parser_args.noise_ratio, rank=parser_args.gpu) + acc1, acc5, acc10 = validate( + data.val_loader, cp_model, criterion, parser_args, writer, epoch) + acc_avg += acc1 + acc_avg /= parser_args.num_test + acc1 = acc_avg + print('Acc after rounding: {}'.format(acc1)) + val_acc1, val_acc5, val_acc10 = validate( + data.actual_val_loader, cp_model, criterion, parser_args, writer, epoch) + print('Validation Acc after rounding: {}'.format(val_acc1)) + else: acc1, acc5, acc10 = validate( - data.val_loader, cp_model, criterion, parser_args, writer, epoch) - acc_avg += acc1 - acc_avg /= parser_args.num_test - acc1 = acc_avg - print('Acc after rounding: {}'.format(acc1)) - val_acc1, val_acc5, val_acc10 = validate( - data.actual_val_loader, cp_model, criterion, parser_args, writer, epoch) - print('Validation Acc after rounding: {}'.format(val_acc1)) - else: - acc1, acc5, acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, epoch) - print('Acc: {}'.format(acc1)) - validation_time.update((time.time() - start_validation) / 60) + data.val_loader, model, criterion, parser_args, writer, epoch) + print('Acc: {}'.format(acc1)) + validation_time.update((time.time() - start_validation) / 60) # prune the model every T_{prune} epochs if not parser_args.weight_training and parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: prune(model) if parser_args.checkpoint_at_prune: - save_checkpoint_at_prune(model, parser_args) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + save_checkpoint_at_prune(model, parser_args) # get model sparsity + # DDP_TODO: This part could be a problem. What if models get out of sync? if not parser_args.weight_training: if parser_args.bottom_k_on_forward: cp_model = copy.deepcopy(model) @@ -212,7 +214,9 @@ def main_worker(gpu, ngpus_per_node): else: # haven't written a weight sparsity function yet avg_sparsity = -1 - print('Model avg sparsity: {}'.format(avg_sparsity)) + + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print('Model avg sparsity: {}'.format(avg_sparsity)) # if model has been "short-circuited", then no point in continuing training if avg_sparsity == 0: @@ -227,92 +231,78 @@ def main_worker(gpu, ngpus_per_node): break # update all results lists - epoch_list.append(epoch) - if parser_args.algo in ['hc', 'hc_iter']: - test_acc_before_round_list.append(br_acc1) - else: - # no before rounding for EP/weight training - test_acc_before_round_list.append(-1) - test_acc_list.append(acc1) - val_acc_list.append(val_acc1) - train_acc_list.append(train_acc1) - reg_loss_list.append(reg_loss) - model_sparsity_list.append(avg_sparsity) - - epoch_time.update((time.time() - end_epoch) / 60) - progress_overall.display(epoch) - progress_overall.write_to_tensorboard( - writer, prefix="diagnostics", global_step=epoch - ) - - if parser_args.conv_type == "SampleSubnetConv": - count = 0 - sum_pr = 0.0 - for n, m in model.named_modules(): - if isinstance(m, SampleSubnetConv): - # avg pr across 10 samples - pr = 0.0 - for _ in range(10): - pr += ( - (torch.rand_like(m.clamped_scores) >= m.clamped_scores) - .float() - .mean() - .item() - ) - pr /= 10.0 - writer.add_scalar("pr/{}".format(n), pr, epoch) - sum_pr += pr - count += 1 - - parser_args.prune_rate = sum_pr / count - writer.add_scalar("pr/average", parser_args.prune_rate, epoch) - - writer.add_scalar("test/lr", cur_lr, epoch) - end_epoch = time.time() - - if parser_args.algo in ['hc', 'hc_iter']: - results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, - 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) - else: - results_df = pd.DataFrame( - {'epoch': epoch_list, 'test_acc': test_acc_list, 'model_sparsity': model_sparsity_list}) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + epoch_list.append(epoch) + if parser_args.algo in ['hc', 'hc_iter']: + test_acc_before_round_list.append(br_acc1) + else: + # no before rounding for EP/weight training + test_acc_before_round_list.append(-1) + test_acc_list.append(acc1) + val_acc_list.append(val_acc1) + train_acc_list.append(train_acc1) + reg_loss_list.append(reg_loss) + model_sparsity_list.append(avg_sparsity) + + epoch_time.update((time.time() - end_epoch) / 60) + progress_overall.display(epoch) + progress_overall.write_to_tensorboard( + writer, prefix="diagnostics", global_step=epoch + ) + + writer.add_scalar("test/lr", cur_lr, epoch) + end_epoch = time.time() + + if parser_args.algo in ['hc', 'hc_iter']: + results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, + 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) + else: + results_df = pd.DataFrame( + {'epoch': epoch_list, 'test_acc': test_acc_list, 'model_sparsity': model_sparsity_list}) - if parser_args.results_filename: - results_filename = parser_args.results_filename - else: - results_filename = result_root + 'acc_and_sparsity.csv' - print("Writing results into: {}".format(results_filename)) - results_df.to_csv(results_filename, index=False) + if parser_args.results_filename: + results_filename = parser_args.results_filename + else: + results_filename = result_root + 'acc_and_sparsity.csv' + print("Writing results into: {}".format(results_filename)) + results_df.to_csv(results_filename, index=False) - # save checkpoint before fine-tuning - #torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + # save checkpoint before fine-tuning + torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') - print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") - print_time() + print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") + print_time() # finetune weights + # DDP_TODO: Check how DDP works with copy deepcopy cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: - print("Beginning fine-tuning") + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("Beginning fine-tuning") cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root) # print out the final acc - eval_and_print(validate, data.val_loader, cp_model, criterion, - parser_args, writer=None, description='final model after finetuning') - # save checkpoint after fine-tuning - #torch.save(cp_model.state_dict(), result_root + 'model_after_finetune.pth') + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + eval_and_print(validate, data.val_loader, cp_model, criterion, + parser_args, writer=None, description='final model after finetuning') + # save checkpoint after fine-tuning + torch.save(cp_model.state_dict(), result_root + 'model_after_finetune.pth') else: - print("Skipping finetuning!!!") + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("Skipping finetuning!!!") if not parser_args.skip_sanity_checks: do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root) else: - print("Skipping sanity checks!!!") + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("Skipping sanity checks!!!") - print("\n\nEnd of process. Exiting") - print_time() + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("\n\nEnd of process. Exiting") + print_time() if parser_args.multiprocessing_distributed: cleanup_distributed() diff --git a/main_utils.py b/main_utils.py index 193e9661..f4b26906 100644 --- a/main_utils.py +++ b/main_utils.py @@ -319,7 +319,6 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) # lr_policy(epoch, iteration=None) - # modifier(parser_args, epoch, model) cur_lr = get_lr(optimizer) print('epoch: {}, lr: {}'.format(epoch, cur_lr)) @@ -331,34 +330,35 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b train_time.update((time.time() - start_train) / 60) # evaluate on validation set - start_validation = time.time() - acc1, acc5, acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, epoch) - val_acc1, val_acc5, val_acc10 = validate( - data.actual_val_loader, model, criterion, parser_args, writer, epoch) - validation_time.update((time.time() - start_validation) / 60) - # copy & paste the sparsity of prev. epoch - avg_sparsity = model_sparsity_list[-1] - - # update all results lists - epoch_list.append(epoch) - test_acc_before_round_list.append(-1) - test_acc_list.append(acc1) - val_acc_list.append(val_acc1) - train_acc_list.append(train_acc1) - reg_loss_list.append(reg_loss) - model_sparsity_list.append(avg_sparsity) - - epoch_time.update((time.time() - end_epoch) / 60) - progress_overall.display(epoch) - progress_overall.write_to_tensorboard( - writer, prefix="diagnostics", global_step=epoch - ) - writer.add_scalar("test/lr", cur_lr, epoch) - end_epoch = time.time() + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + start_validation = time.time() + acc1, acc5, acc10 = validate( + data.val_loader, model, criterion, parser_args, writer, epoch) + val_acc1, val_acc5, val_acc10 = validate( + data.actual_val_loader, model, criterion, parser_args, writer, epoch) + validation_time.update((time.time() - start_validation) / 60) + # copy & paste the sparsity of prev. epoch + avg_sparsity = model_sparsity_list[-1] + + # update all results lists + epoch_list.append(epoch) + test_acc_before_round_list.append(-1) + test_acc_list.append(acc1) + val_acc_list.append(val_acc1) + train_acc_list.append(train_acc1) + reg_loss_list.append(reg_loss) + model_sparsity_list.append(avg_sparsity) + + epoch_time.update((time.time() - end_epoch) / 60) + progress_overall.display(epoch) + progress_overall.write_to_tensorboard( + writer, prefix="diagnostics", global_step=epoch + ) + writer.add_scalar("test/lr", cur_lr, epoch) + end_epoch = time.time() - results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, - 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) + results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, + 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) if not chg_mask and not chg_weight: results_filename = result_root + 'acc_and_sparsity.csv' # elif chg_weight and shuffle: From 7f1e0f6b48aae20d717d6a00862eb3449f99a912 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 14:35:53 -0500 Subject: [PATCH 008/113] testing ddp with deepcopy --- ddp_poc.py | 27 +++++++++++++++++++++------ main.py | 2 ++ main_utils.py | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 5e291cf0..87414b96 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -15,6 +15,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP from ddp_args_helper import parser_args from ddp_utils import do_something_outside +import copy def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' @@ -97,7 +98,7 @@ def demo_basic(rank, world_size): device = torch.device("cuda:{}".format(rank)) - for epoch in range(15): + for epoch in range(5): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) print("Local Rank: {} | Parser args: gpu={}, Name={}".format(rank, parser_args.gpu, parser_args.name)) if epoch % 3 == 0: @@ -134,11 +135,25 @@ def demo_basic(rank, world_size): print("End of epoch total batch sizes: {}".format(total_data_size)) do_something_outside(rank) - # optimizer.zero_grad() - # outputs = ddp_model(torch.randn(20, 10)) - # labels = torch.randn(20, 5).to(rank) - # loss_fn(outputs, labels).backward() - # optimizer.step() + + print("Local rank: {} | Entering barrier".format(rank)) + dist.barrier() + print("Local rank: {} | Past barrier".format(rank)) + cp_model = copy.deepcopy(model) + print("Local rank: {} | Copied Model".format(rank)) + + for data in train_loader: + print("Rank: {} | Copied Model Norm: {}".format(rank, get_model_norm(cp_model))) + inputs, labels = data[0].to(device).reshape(-1, 32*32*3), data[1].to(device) + # print("Device: {} | Batch Size: {} | Label sum: {}".format(rank, data[1].shape[0], torch.sum(data[1]))) + total_data_size[rank] += data[1].shape[0] + optimizer.zero_grad() + outputs = cp_model(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + print("End of epoch total batch sizes: {}".format(total_data_size)) + cleanup() diff --git a/main.py b/main.py index 2b7ef2cd..5d810dde 100644 --- a/main.py +++ b/main.py @@ -276,6 +276,8 @@ def main_worker(gpu, ngpus_per_node): # finetune weights # DDP_TODO: Check how DDP works with copy deepcopy + dist.barrier() + cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: diff --git a/main_utils.py b/main_utils.py index f4b26906..37a25a73 100644 --- a/main_utils.py +++ b/main_utils.py @@ -18,7 +18,7 @@ import torch.backends.cudnn as cudnn import torch.optim import torch.utils.data -import torch.utils.data.distributed +import torch.utils.data.distributed as dist import torch.multiprocessing as mp import sys From 6cc557bdd4c06253a784d42f5016b58a8c2a9af2 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 15:24:19 -0500 Subject: [PATCH 009/113] looks like copy.deepcopy causes sync issues --- ddp_poc.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 87414b96..5efc1b63 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -98,7 +98,7 @@ def demo_basic(rank, world_size): device = torch.device("cuda:{}".format(rank)) - for epoch in range(5): + for epoch in range(2): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) print("Local Rank: {} | Parser args: gpu={}, Name={}".format(rank, parser_args.gpu, parser_args.name)) if epoch % 3 == 0: @@ -120,7 +120,7 @@ def demo_basic(rank, world_size): print("-" * 75) ddp_model.train() - total_data_size = [0, 0] + total_data_size = [0, 0, 0, 0] for data in train_loader: print("Rank: {} | Model Norm: {}".format(rank, get_model_norm(ddp_model))) @@ -142,6 +142,7 @@ def demo_basic(rank, world_size): cp_model = copy.deepcopy(model) print("Local rank: {} | Copied Model".format(rank)) + optimizer = optim.SGD(cp_model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-5) for data in train_loader: print("Rank: {} | Copied Model Norm: {}".format(rank, get_model_norm(cp_model))) inputs, labels = data[0].to(device).reshape(-1, 32*32*3), data[1].to(device) @@ -154,6 +155,11 @@ def demo_basic(rank, world_size): optimizer.step() print("End of epoch total batch sizes: {}".format(total_data_size)) + print("Local rank: {} | Entering barrier".format(rank)) + dist.barrier() + print("Local rank: {} | Past barrier".format(rank)) + print("Rank: {} | Copied Model Norm: {}".format(rank, get_model_norm(cp_model))) + cleanup() From dd0ba73848da714b61f506217223eb214410eae9 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 15:38:51 -0500 Subject: [PATCH 010/113] fixing typo in ddp deepcopy --- ddp_poc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddp_poc.py b/ddp_poc.py index 5efc1b63..021aa4ea 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -139,7 +139,7 @@ def demo_basic(rank, world_size): print("Local rank: {} | Entering barrier".format(rank)) dist.barrier() print("Local rank: {} | Past barrier".format(rank)) - cp_model = copy.deepcopy(model) + cp_model = copy.deepcopy(ddp_model) print("Local rank: {} | Copied Model".format(rank)) optimizer = optim.SGD(cp_model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-5) From 1d21c5a227ad3b6ff6a194591aa7969de9a76619 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 16:23:47 -0500 Subject: [PATCH 011/113] finishing up ddp for sanity checks and finetune --- data/cifar.py | 19 +++++++++++++++--- data/imagenet.py | 2 +- main.py | 15 ++++++++++----- main_utils.py | 50 +++++++++++++++++++++++++++++++++--------------- 4 files changed, 62 insertions(+), 24 deletions(-) diff --git a/data/cifar.py b/data/cifar.py index 696ff388..5a5159ff 100644 --- a/data/cifar.py +++ b/data/cifar.py @@ -23,6 +23,11 @@ def __init__(self, args): mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262] ) + if args.multiprocessing_distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + dataset = torchvision.datasets.CIFAR10( root=data_root, train=True, @@ -54,13 +59,21 @@ def __init__(self, args): train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) self.train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=parser_args.batch_size, shuffle=True, **kwargs + train_dataset, + batch_size=parser_args.batch_size, + shuffle=(train_sampler is None), + sampler=train_sampler, + **kwargs ) self.val_loader = torch.utils.data.DataLoader( - test_dataset, batch_size=parser_args.batch_size, shuffle=False, **kwargs + test_dataset, + batch_size=parser_args.batch_size, + shuffle=False, **kwargs ) self.actual_val_loader = torch.utils.data.DataLoader( - validation_dataset, batch_size=parser_args.batch_size, shuffle=True, **kwargs + validation_dataset, + batch_size=parser_args.batch_size, + shuffle=False, **kwargs ) diff --git a/data/imagenet.py b/data/imagenet.py index 0379647a..a78ac4f4 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -84,6 +84,6 @@ def __init__(self, args): self.actual_val_loader = torch.utils.data.DataLoader( validation_dataset, batch_size=parser_args.batch_size, - shuffle=True, + shuffle=False, **kwargs ) diff --git a/main.py b/main.py index 5d810dde..d0d10603 100644 --- a/main.py +++ b/main.py @@ -112,7 +112,7 @@ def main_worker(gpu, ngpus_per_node): print("Overriding prune_rate to {}".format(parser_args.prune_rate)) #if parser_args.dataset == 'TinyImageNet': # print_num_dataset(data) - if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed:: + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: if not parser_args.weight_training: print_layers(parser_args, model) @@ -131,9 +131,9 @@ def main_worker(gpu, ngpus_per_node): do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, reg_loss_list, model_sparsity_list, parser_args.results_root) - #cp_model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, + # cp_model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, # ratio=parser_args.noise_ratio, rank=parser_args.gpu) - #print(get_model_sparsity(cp_model)) + # print(get_model_sparsity(cp_model)) return @@ -275,8 +275,10 @@ def main_worker(gpu, ngpus_per_node): print_time() # finetune weights - # DDP_TODO: Check how DDP works with copy deepcopy + # DDP works surprisingly well with copy deepcopy. Might cause memory issues TODO + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: @@ -294,10 +296,13 @@ def main_worker(gpu, ngpus_per_node): if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: print("Skipping finetuning!!!") + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if not parser_args.skip_sanity_checks: do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root) - else: if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: print("Skipping sanity checks!!!") diff --git a/main_utils.py b/main_utils.py index 37a25a73..61bb83ca 100644 --- a/main_utils.py +++ b/main_utils.py @@ -94,6 +94,11 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b print("Beginning Sanity Checks:") # do the sanity check for shuffled mask/weights, reinit weights print("Sanity Check 1: Weight Reinit") + + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root, reinit=True, chg_weight=True) @@ -105,6 +110,11 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b reg_loss_list, model_sparsity_list, result_root, shuffle=True, chg_weight=True) ''' print("Sanity Check 2: Mask Reshuffle") + + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root, shuffle=True, chg_mask=True) @@ -262,12 +272,17 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b post_round_sparsity = get_model_sparsity(model) # apply reinit/shuffling masks/weights (if necessary) + # DDP_TODO: Check if this works well model = redraw(model, shuffle=shuffle, reinit=reinit, invert=invert, chg_mask=chg_mask, chg_weight=chg_weight) # switch to weight training mode (turn on the requires_grad for weight/bias, and turn off the requires_grad for other parameters) model = switch_to_wt(model) + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + # not to use score regulaization during the weight training parser_args.regularization = False @@ -298,20 +313,24 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b train, validate, modifier = get_trainer(parser_args) # check the performance of loaded model (after rounding) - acc1, acc5, acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, parser_args.epochs-1) - val_acc1, val_acc5, val_acc10 = validate( - data.actual_val_loader, model, criterion, parser_args, writer, parser_args.epochs-1) - train_acc1, train_acc5, train_acc10 = validate( - data.train_loader, model, criterion, parser_args, writer, parser_args.epochs-1) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + acc1, acc5, acc10 = validate( + data.val_loader, model, criterion, parser_args, writer, parser_args.epochs-1) + val_acc1, val_acc5, val_acc10 = validate( + data.actual_val_loader, model, criterion, parser_args, writer, parser_args.epochs-1) + train_acc1, train_acc5, train_acc10 = validate( + data.train_loader, model, criterion, parser_args, writer, parser_args.epochs-1) + avg_sparsity = post_round_sparsity - epoch_list.append(parser_args.epochs-1) - test_acc_before_round_list.append(-1) - test_acc_list.append(acc1) - val_acc_list.append(val_acc1) - train_acc_list.append(train_acc1) - reg_loss_list.append(0.0) - model_sparsity_list.append(avg_sparsity) + + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + epoch_list.append(parser_args.epochs-1) + test_acc_before_round_list.append(-1) + test_acc_list.append(acc1) + val_acc_list.append(val_acc1) + train_acc_list.append(train_acc1) + reg_loss_list.append(0.0) + model_sparsity_list.append(avg_sparsity) end_epoch = time.time() for epoch in range(parser_args.epochs, parser_args.epochs*2): @@ -372,8 +391,9 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b else: raise NotImplementedError - print("Writing results into: {}".format(results_filename)) - results_df.to_csv(results_filename, index=False) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("Writing results into: {}".format(results_filename)) + results_df.to_csv(results_filename, index=False) scheduler.step() return model From 691cffacd9abade09c0ad84a7983153dd15c728f Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 16:40:29 -0500 Subject: [PATCH 012/113] minor bugfixes for ddp --- cifar_exec.sh | 8 ++--- configs/ddp_debug/conf1.yml | 64 +++++++++++++++++++++++++++++++++++++ data/cifar.py | 10 +++--- data/imagenet.py | 4 +-- main.py | 2 +- main_utils.py | 2 +- trainers/default.py | 1 - 7 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 configs/ddp_debug/conf1.yml diff --git a/cifar_exec.sh b/cifar_exec.sh index 43eefb4f..7d7a1efd 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -64,15 +64,15 @@ BLOCK #:< "$log_root$trial$log_end" 2>&1 & diff --git a/configs/ddp_debug/conf1.yml b/configs/ddp_debug/conf1.yml new file mode 100644 index 00000000..23e5733c --- /dev/null +++ b/configs/ddp_debug/conf1.yml @@ -0,0 +1,64 @@ +# subfolder: target_sparsity_0_59_unflagT_real + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 20 +wd: 0.0 +momentum: 0.9 +batch_size: 512 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + + + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 + +# ===== Hardware setup ===== # +workers: 8 +# gpu: 1 +multiprocessing_distributed: True + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/data/cifar.py b/data/cifar.py index 5a5159ff..a782b917 100644 --- a/data/cifar.py +++ b/data/cifar.py @@ -23,11 +23,6 @@ def __init__(self, args): mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262] ) - if args.multiprocessing_distributed: - train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) - else: - train_sampler = None - dataset = torchvision.datasets.CIFAR10( root=data_root, train=True, @@ -58,6 +53,11 @@ def __init__(self, args): train_size = len(dataset) - val_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) + if parser_args.multiprocessing_distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + self.train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=parser_args.batch_size, diff --git a/data/imagenet.py b/data/imagenet.py index a78ac4f4..5e978d27 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -12,7 +12,7 @@ class ImageNet: def __init__(self, args): super(ImageNet, self).__init__() - data_root = args.data + data_root = parser_args.data use_cuda = torch.cuda.is_available() @@ -60,7 +60,7 @@ def __init__(self, args): train_size = len(dataset) - val_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) - if args.multiprocessing_distributed: + if parser_args.multiprocessing_distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None diff --git a/main.py b/main.py index d0d10603..b7e1c364 100644 --- a/main.py +++ b/main.py @@ -28,7 +28,7 @@ def main_worker(gpu, ngpus_per_node): if parser_args.multiprocessing_distributed: parser_args.rank = parser_args.gpu - setup_distributed(rank, ngpus_per_node) + setup_distributed(parser_args.rank, ngpus_per_node) # if using ddp, divide batch size per gpu parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) diff --git a/main_utils.py b/main_utils.py index 61bb83ca..06caaaf9 100644 --- a/main_utils.py +++ b/main_utils.py @@ -18,7 +18,7 @@ import torch.backends.cudnn as cudnn import torch.optim import torch.utils.data -import torch.utils.data.distributed as dist +import torch.distributed as dist import torch.multiprocessing as mp import sys diff --git a/trainers/default.py b/trainers/default.py index 8325fccb..6597353f 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -87,7 +87,6 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # compute gradient and do SGD step optimizer.zero_grad() - import ipdb; ipdb.set_trace() if scaler is None: loss.backward() optimizer.step() From ffca64810cb1ba85a4366fcb130068c3fee18659 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 16:43:27 -0500 Subject: [PATCH 013/113] removing .cuda() from the codebase --- main.py | 2 +- utils/net_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b7e1c364..87172299 100644 --- a/main.py +++ b/main.py @@ -72,7 +72,7 @@ def main_worker(gpu, ngpus_per_node): scheduler = get_scheduler(optimizer, parser_args.lr_policy) # lr_policy = get_policy(parser_args.lr_policy)(optimizer, parser_args) if parser_args.label_smoothing is None: - criterion = nn.CrossEntropyLoss().cuda() + criterion = nn.CrossEntropyLoss() else: criterion = LabelSmoothing(smoothing=parser_args.label_smoothing) # if isinstance(model, nn.parallel.DistributedDataParallel): diff --git a/utils/net_utils.py b/utils/net_utils.py index 46462981..58434248 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -563,7 +563,7 @@ def get_regularization_loss(model, regularizer='L2', lmbda=1, alpha=1, alpha_pri def get_special_reg_sum(layer): # reg_loss = \sum_{i} w_i^2 * p_i(1-p_i) # NOTE: alpha = alpha' = 1 here. Change if needed. - reg_sum = torch.tensor(0.).cuda() + reg_sum = torch.tensor(0.).to(parser_args.gpu) w_i = layer.weight p_i = layer.scores reg_sum += torch.sum(torch.pow(w_i, 2) * @@ -576,7 +576,7 @@ def get_special_reg_sum(layer): return reg_sum #pdb.set_trace() - regularization_loss = torch.tensor(0.).cuda() + regularization_loss = torch.tensor(0.).to(parser_args.gpu) if regularizer == 'L2': # reg_loss = ||p||_2^2 for name, params in model.named_parameters(): From e1c9810e358252bbc9e635529db8591871c56b95 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 17:10:09 -0500 Subject: [PATCH 014/113] trying to catch the rank=0 leak --- main.py | 2 +- main_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 87172299..43ab73d2 100644 --- a/main.py +++ b/main.py @@ -35,7 +35,7 @@ def main_worker(gpu, ngpus_per_node): train, validate, modifier = get_trainer(parser_args) model = get_model(parser_args) - if parser_args == 0: + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: idty_str = get_idty_str(parser_args) if parser_args.subfolder is not None: if not os.path.isdir('results/'): diff --git a/main_utils.py b/main_utils.py index 06caaaf9..694e101e 100644 --- a/main_utils.py +++ b/main_utils.py @@ -903,7 +903,7 @@ def get_trainer(parser_args): def set_gpu(parser_args, model): assert torch.cuda.is_available(), "CPU-only experiments currently unsupported" - # torch.cuda.set_device(parser_args.gpu) + torch.cuda.set_device(parser_args.gpu) model.to(parser_args.gpu) if parser_args.multiprocessing_distributed: From 64cbf57c4719eeaa7951fc163d0295da2251231c Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 17:43:28 -0500 Subject: [PATCH 015/113] I think ddp works! --- configs/ddp_debug/conf1.yml | 1 + utils/net_utils.py | 6 +++--- utils/utils.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/configs/ddp_debug/conf1.yml b/configs/ddp_debug/conf1.yml index 23e5733c..7e02189e 100644 --- a/configs/ddp_debug/conf1.yml +++ b/configs/ddp_debug/conf1.yml @@ -56,6 +56,7 @@ lmbda: 0.0001 # 1e-4 workers: 8 # gpu: 1 multiprocessing_distributed: True +mixed_precision: True # ===== Checkpointing ===== # checkpoint_at_prune: False diff --git a/utils/net_utils.py b/utils/net_utils.py index 58434248..9e5828d9 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -348,9 +348,9 @@ def round_model(model, round_scheme, noise=False, ratio=0.0, rank=None): params.data = (params.data + delta) % 2 ''' - if isinstance(model, nn.parallel.DistributedDataParallel): - cp_model = nn.parallel.DistributedDataParallel( - cp_model, device_ids=[rank], find_unused_parameters=True) + # if isinstance(model, nn.parallel.DistributedDataParallel): + # cp_model = nn.parallel.DistributedDataParallel( + # cp_model, device_ids=[rank], find_unused_parameters=True) return cp_model diff --git a/utils/utils.py b/utils/utils.py index 2ed92dba..bc4c509a 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -24,9 +24,9 @@ def set_seed(seed): torch.cuda.manual_seed_all(seed) np.random.seed(seed) os.environ['PYTHONHASHSEED'] = str(seed) - # making sure GPU runs are deterministic even if they are slower + # set this=True if you want deterministic runs torch.backends.cudnn.deterministic = False - # this causes the code to vary across runs. I don't want that for now. + # set this=False if you want deterministic runs torch.backends.cudnn.benchmark = True print("Seeded everything: {}".format(seed)) From eb12a2d874b719e85c1445f2753a8ebb59247e95 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:00:53 -0500 Subject: [PATCH 016/113] skipping checkpoint, trying to get rid of tqdm --- cifar_exec.sh | 2 +- configs/ddp_debug/conf1.yml | 2 -- main.py | 30 +++++++++++++++----------- main_utils.py | 21 +++++++++++-------- trainers/default.py | 42 ++++++++++++++++++------------------- 5 files changed, 52 insertions(+), 45 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 7d7a1efd..6ba68efa 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -72,7 +72,7 @@ subfolder_root="ddp_debug_" for trial in 1 do - CUDA_VISIBLE_DEVICES=0,2,3 python main.py \ + CUDA_VISIBLE_DEVICES=0,1 python main.py \ --config "$conf_file$conf_end" \ --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 & diff --git a/configs/ddp_debug/conf1.yml b/configs/ddp_debug/conf1.yml index 7e02189e..d94092b8 100644 --- a/configs/ddp_debug/conf1.yml +++ b/configs/ddp_debug/conf1.yml @@ -37,8 +37,6 @@ init: signed_constant score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True - - # ===== Rounding ===== # round: naive noise: True diff --git a/main.py b/main.py index 43ab73d2..95bdda4b 100644 --- a/main.py +++ b/main.py @@ -46,6 +46,9 @@ def main_worker(gpu, ngpus_per_node): result_root = result_subroot + '/results_' + idty_str + '/' else: result_root = 'results/results_' + idty_str + '/' + else: + idty_str = get_idty_str(parser_args) + result_root = 'results/results_' + idty_str + '/' if not os.path.isdir(result_root): os.mkdir(result_root) @@ -266,19 +269,21 @@ def main_worker(gpu, ngpus_per_node): results_filename = result_root + 'acc_and_sparsity.csv' print("Writing results into: {}".format(results_filename)) results_df.to_csv(results_filename, index=False) + print("Local rank: {} | About to enter save model logic".format(parser_args.gpu)) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + # save checkpoint before fine-tuning + # torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') - if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - # save checkpoint before fine-tuning - torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') - - print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") - print_time() + print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") + print("Local rank: {}".format(parser_args.gpu)) + print_time() # finetune weights # DDP works surprisingly well with copy deepcopy. Might cause memory issues TODO - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: @@ -296,9 +301,10 @@ def main_worker(gpu, ngpus_per_node): if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: print("Skipping finetuning!!!") - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) if not parser_args.skip_sanity_checks: do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, diff --git a/main_utils.py b/main_utils.py index 694e101e..55a93499 100644 --- a/main_utils.py +++ b/main_utils.py @@ -95,9 +95,10 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b # do the sanity check for shuffled mask/weights, reinit weights print("Sanity Check 1: Weight Reinit") - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, @@ -111,9 +112,10 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b ''' print("Sanity Check 2: Mask Reshuffle") - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, @@ -279,9 +281,10 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b # switch to weight training mode (turn on the requires_grad for weight/bias, and turn off the requires_grad for other parameters) model = switch_to_wt(model) - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) # not to use score regulaization during the weight training parser_args.regularization = False diff --git a/trainers/default.py b/trainers/default.py index 6597353f..8b11a9e9 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -1,11 +1,11 @@ import time import torch -import tqdm +# import tqdm import copy import pdb from utils.eval_utils import accuracy -from utils.logging import AverageMeter, ProgressMeter +# from utils.logging import AverageMeter, ProgressMeter from utils.net_utils import get_regularization_loss, prune, get_layers from torch import optim @@ -15,17 +15,17 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler=None): - batch_time = AverageMeter("Time", ":6.3f") - data_time = AverageMeter("Data", ":6.3f") - losses = AverageMeter("Loss", ":.3f") - top1 = AverageMeter("Acc@1", ":6.2f") - top5 = AverageMeter("Acc@5", ":6.2f") - top10 = AverageMeter("Acc@10", ":6.2f") - progress = ProgressMeter( - len(train_loader), - [batch_time, data_time, losses, top1, top5], - prefix=f"Epoch: [{epoch}]", - ) + # batch_time = AverageMeter("Time", ":6.3f") + # data_time = AverageMeter("Data", ":6.3f") + # losses = AverageMeter("Loss", ":.3f") + # top1 = AverageMeter("Acc@1", ":6.2f") + # top5 = AverageMeter("Acc@5", ":6.2f") + # top10 = AverageMeter("Acc@10", ":6.2f") + # progress = ProgressMeter( + # len(train_loader), + # [batch_time, data_time, losses, top1, top5], + # prefix=f"GPU:[{args.gpu}] | Epoch: [{epoch}]", + # ) # switch to train mode model.train() @@ -120,14 +120,14 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler def validate(val_loader, model, criterion, args, writer, epoch): - batch_time = AverageMeter("Time", ":6.3f", write_val=False) - losses = AverageMeter("Loss", ":.3f", write_val=False) - top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) - top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) - top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) - progress = ProgressMeter( - len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " - ) + # batch_time = AverageMeter("Time", ":6.3f", write_val=False) + # losses = AverageMeter("Loss", ":.3f", write_val=False) + # top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) + # top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) + # top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) + # progress = ProgressMeter( + # len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " + # ) # switch to evaluate mode model.eval() From f4fbff04a39f6042224594cf343eada119d5020c Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:26:40 -0500 Subject: [PATCH 017/113] trying to eliminate tqdm and progress meters I think it's just slowing things down and is known to cause memory leaks --- main_utils.py | 36 ++++++++++-------- trainers/default.py | 91 ++++++++++++++++++++++++++++----------------- utils/logging.py | 2 +- 3 files changed, 78 insertions(+), 51 deletions(-) diff --git a/main_utils.py b/main_utils.py index 55a93499..7180c3c7 100644 --- a/main_utils.py +++ b/main_utils.py @@ -25,7 +25,7 @@ import re from utils.conv_type import FixedSubnetConv, SampleSubnetConv -from utils.logging import AverageMeter, ProgressMeter +# from utils.logging import AverageMeter, ProgressMeter from utils.net_utils import ( set_model_prune_rate, freeze_model_weights, @@ -349,7 +349,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b train_acc1, train_acc5, train_acc10, reg_loss = train( data.train_loader, model, criterion, optimizer, epoch, parser_args, writer=writer ) - train_time.update((time.time() - start_train) / 60) + # train_time.update((time.time() - start_train) / 60) + train_time = (time.time() - start_train) / 60 # evaluate on validation set if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: @@ -358,7 +359,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b data.val_loader, model, criterion, parser_args, writer, epoch) val_acc1, val_acc5, val_acc10 = validate( data.actual_val_loader, model, criterion, parser_args, writer, epoch) - validation_time.update((time.time() - start_validation) / 60) + # validation_time.update((time.time() - start_validation) / 60) + validation_time = (time.time() - start_validation) / 60 # copy & paste the sparsity of prev. epoch avg_sparsity = model_sparsity_list[-1] @@ -371,12 +373,12 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b reg_loss_list.append(reg_loss) model_sparsity_list.append(avg_sparsity) - epoch_time.update((time.time() - end_epoch) / 60) - progress_overall.display(epoch) - progress_overall.write_to_tensorboard( - writer, prefix="diagnostics", global_step=epoch - ) - writer.add_scalar("test/lr", cur_lr, epoch) + # epoch_time.update((time.time() - end_epoch) / 60) + # progress_overall.display(epoch) + # progress_overall.write_to_tensorboard( + # writer, prefix="diagnostics", global_step=epoch + # ) + # writer.add_scalar("test/lr", cur_lr, epoch) end_epoch = time.time() results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, @@ -439,12 +441,16 @@ def get_settings(parser_args): parser_args.ckpt_base_dir = ckpt_base_dir writer = SummaryWriter(log_dir=log_base_dir) # writer = None - epoch_time = AverageMeter("epoch_time", ":.4f", write_avg=False) - validation_time = AverageMeter("validation_time", ":.4f", write_avg=False) - train_time = AverageMeter("train_time", ":.4f", write_avg=False) - progress_overall = ProgressMeter( - 1, [epoch_time, validation_time, train_time], prefix="Overall Timing" - ) + # epoch_time = AverageMeter("epoch_time", ":.4f", write_avg=False) + # validation_time = AverageMeter("validation_time", ":.4f", write_avg=False) + # train_time = AverageMeter("train_time", ":.4f", write_avg=False) + # progress_overall = ProgressMeter( + # 1, [epoch_time, validation_time, train_time], prefix="Overall Timing" + # ) + epoch_time = 0 + validation_time = 0 + train_time = 0 + progress_overall = None return run_base_dir, ckpt_base_dir, log_base_dir, writer, epoch_time, validation_time, train_time, progress_overall diff --git a/trainers/default.py b/trainers/default.py index 8b11a9e9..4df27010 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -26,6 +26,10 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # [batch_time, data_time, losses, top1, top5], # prefix=f"GPU:[{args.gpu}] | Epoch: [{epoch}]", # ) + top1 = 0 + top5 = 0 + top10 = 0 + num_images = 0 # switch to train mode model.train() @@ -33,16 +37,16 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler batch_size = train_loader.batch_size num_batches = len(train_loader) end = time.time() - for i, (images, target) in tqdm.tqdm( - enumerate(train_loader), ascii=True, total=len(train_loader) - ): + # for i, (images, target) in tqdm.tqdm( + # enumerate(train_loader), ascii=True, total=len(train_loader) + # ): + for i, (images, target) in enumerate(train_loader): # measure data loading time - data_time.update(time.time() - end) - #print(images.shape, target.shape) - - if args.gpu is not None: - images = images.cuda(args.gpu, non_blocking=True) + data_time = time.time() - end + # print("Data Time: {}".format(data_time)) + # print(images.shape, target.shape) + images = images.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) # update score thresholds for global ep @@ -80,10 +84,15 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # measure accuracy and record loss acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - losses.update(loss.item(), images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - top10.update(acc10.item(), images.size(0)) + # losses.update(loss.item(), images.size(0)) + # top1.update(acc1.item(), images.size(0)) + # top5.update(acc5.item(), images.size(0)) + # top10.update(acc10.item(), images.size(0)) + # compute weighted sum for each accuracy so we can average it later + top1 += acc1.item() * images.size(0) + top5 += acc5.item() * images.size(0) + top10 += acc10.item() * images.size(0) + num_images += images.size(0) # compute gradient and do SGD step optimizer.zero_grad() @@ -96,14 +105,17 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler scaler.update() # measure elapsed time - batch_time.update(time.time() - end) + # batch_time.update(time.time() - end) + batch_time = time.time() - end end = time.time() if i % args.print_freq == 0: t = (num_batches * epoch + i) * batch_size - progress.display(i) - progress.write_to_tensorboard( - writer, prefix="train", global_step=t) + # progress.display(i) + # progress.write_to_tensorboard( + # writer, prefix="train", global_step=t) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item()), batch_time) + # before completing training, clean up model based on latest scores # update score thresholds for global ep @@ -116,7 +128,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) - return top1.avg, top5.avg, top10.avg, regularization_loss.item() + return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() def validate(val_loader, model, criterion, args, writer, epoch): @@ -128,18 +140,20 @@ def validate(val_loader, model, criterion, args, writer, epoch): # progress = ProgressMeter( # len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " # ) + top1 = 0 + top5 = 0 + top10 = 0 # switch to evaluate mode model.eval() with torch.no_grad(): end = time.time() - for i, (images, target) in tqdm.tqdm( - enumerate(val_loader), ascii=True, total=len(val_loader) - ): - if args.gpu is not None: - images = images.cuda(args.gpu, non_blocking=True) - + # for i, (images, target) in tqdm.tqdm( + # enumerate(val_loader), ascii=True, total=len(val_loader) + # ): + for i, (images, target) in enumerate(val_loader): + images = images.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) #print(images.shape, target.shape) @@ -151,26 +165,33 @@ def validate(val_loader, model, criterion, args, writer, epoch): # measure accuracy and record loss acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - losses.update(loss.item(), images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - top10.update(acc10.item(), images.size(0)) + # losses.update(loss.item(), images.size(0)) + # top1.update(acc1.item(), images.size(0)) + # top5.update(acc5.item(), images.size(0)) + # top10.update(acc10.item(), images.size(0)) + # compute weighted sum for each accuracy so we can average it later + top1 += acc1.item() * images.size(0) + top5 += acc5.item() * images.size(0) + top10 += acc10.item() * images.size(0) + num_images += images.size(0) # measure elapsed time - batch_time.update(time.time() - end) + # batch_time.update(time.time() - end) + batch_time = time.time() - end end = time.time() if i % args.print_freq == 0: - progress.display(i) + # progress.display(i) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item()), batch_time) - progress.display(len(val_loader)) + # progress.display(len(val_loader)) - if writer is not None: - progress.write_to_tensorboard( - writer, prefix="test", global_step=epoch) + # if writer is not None: + # progress.write_to_tensorboard( + # writer, prefix="test", global_step=epoch) - print("Model top1 Accuracy: {}".format(top1.avg)) - return top1.avg, top5.avg, top10.avg + print("Model top1 Accuracy: {}".format(top1/num_images)) + return top1/num_images, top5/num_images, top10/num_images def modifier(args, epoch, model): diff --git a/utils/logging.py b/utils/logging.py index 8cce423a..7f809ba8 100644 --- a/utils/logging.py +++ b/utils/logging.py @@ -1,5 +1,5 @@ import abc -import tqdm +# import tqdm # from torch.utils.tensorboard import SummaryWriter From 69048e0bac37eb3ac6335f9882eff3ef13b50025 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:30:56 -0500 Subject: [PATCH 018/113] trainer doesn't see parser_args --- main_utils.py | 2 ++ trainers/default.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/main_utils.py b/main_utils.py index 7180c3c7..991b8178 100644 --- a/main_utils.py +++ b/main_utils.py @@ -379,6 +379,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b # writer, prefix="diagnostics", global_step=epoch # ) # writer.add_scalar("test/lr", cur_lr, epoch) + epoch_time = (time.time() - end_epoch) / 60 + print("GPU:{} | Epoch: {} | Acc={} | Epoch Time={}".format(parser_args.gpu, epoch, acc1, epoch_time)) end_epoch = time.time() results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, diff --git a/trainers/default.py b/trainers/default.py index 4df27010..0cd8e5eb 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -114,7 +114,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # progress.display(i) # progress.write_to_tensorboard( # writer, prefix="train", global_step=t) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item()), batch_time) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item()), batch_time) # before completing training, clean up model based on latest scores @@ -182,7 +182,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): if i % args.print_freq == 0: # progress.display(i) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item()), batch_time) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item()), batch_time) # progress.display(len(val_loader)) From 3834b3d70018600406a9a2775bb957c8f65cd697 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:46:35 -0500 Subject: [PATCH 019/113] still removing leftover calls to tqdm and progresmeters --- main.py | 6 ++++-- trainers/default.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 95bdda4b..0cf1f349 100644 --- a/main.py +++ b/main.py @@ -164,7 +164,8 @@ def main_worker(gpu, ngpus_per_node): train_acc1, train_acc5, train_acc10, reg_loss = train( data.train_loader, model, criterion, optimizer, epoch, parser_args, writer=writer, scaler=scaler ) - train_time.update((time.time() - start_train) / 60) + # train_time.update((time.time() - start_train) / 60) + train_time = (time.time() - start_train) / 60 scheduler.step() # evaluate on validation set @@ -191,7 +192,8 @@ def main_worker(gpu, ngpus_per_node): acc1, acc5, acc10 = validate( data.val_loader, model, criterion, parser_args, writer, epoch) print('Acc: {}'.format(acc1)) - validation_time.update((time.time() - start_validation) / 60) + # validation_time.update((time.time() - start_validation) / 60) + validation_time = (time.time() - start_validation) / 60 # prune the model every T_{prune} epochs if not parser_args.weight_training and parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: diff --git a/trainers/default.py b/trainers/default.py index 0cd8e5eb..8b1754c9 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -114,7 +114,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # progress.display(i) # progress.write_to_tensorboard( # writer, prefix="train", global_step=t) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item()), batch_time) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) # before completing training, clean up model based on latest scores @@ -143,6 +143,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): top1 = 0 top5 = 0 top10 = 0 + num_images = 0 # switch to evaluate mode model.eval() @@ -182,7 +183,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): if i % args.print_freq == 0: # progress.display(i) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item()), batch_time) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) # progress.display(len(val_loader)) From 3ba1f9344e032ec97df4242f105b16838a25e54f Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:49:20 -0500 Subject: [PATCH 020/113] removing more instances of progressmeter and replacing them with vanilla prints --- main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 0cf1f349..b92e5efa 100644 --- a/main.py +++ b/main.py @@ -166,6 +166,7 @@ def main_worker(gpu, ngpus_per_node): ) # train_time.update((time.time() - start_train) / 60) train_time = (time.time() - start_train) / 60 + scheduler.step() # evaluate on validation set @@ -249,13 +250,15 @@ def main_worker(gpu, ngpus_per_node): reg_loss_list.append(reg_loss) model_sparsity_list.append(avg_sparsity) - epoch_time.update((time.time() - end_epoch) / 60) - progress_overall.display(epoch) - progress_overall.write_to_tensorboard( - writer, prefix="diagnostics", global_step=epoch - ) + # epoch_time.update((time.time() - end_epoch) / 60) + epoch_time = (time.time() - end_epoch) / 60 + # progress_overall.display(epoch) + # progress_overall.write_to_tensorboard( + # writer, prefix="diagnostics", global_step=epoch + # ) + print("GPU:{} | Epoch: {} | Acc={} | Epoch Time={}".format(parser_args.gpu, epoch, acc1, epoch_time)) - writer.add_scalar("test/lr", cur_lr, epoch) + # writer.add_scalar("test/lr", cur_lr, epoch) end_epoch = time.time() if parser_args.algo in ['hc', 'hc_iter']: From 77fbe481cfd12f42b596a29d17bb0f8689962b06 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 17:24:12 -0500 Subject: [PATCH 021/113] fixing dir creation bug --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b92e5efa..5116f2f5 100644 --- a/main.py +++ b/main.py @@ -46,13 +46,14 @@ def main_worker(gpu, ngpus_per_node): result_root = result_subroot + '/results_' + idty_str + '/' else: result_root = 'results/results_' + idty_str + '/' - else: - idty_str = get_idty_str(parser_args) - result_root = 'results/results_' + idty_str + '/' if not os.path.isdir(result_root): os.mkdir(result_root) print_model(model, parser_args) + else: + idty_str = get_idty_str(parser_args) + result_root = 'results/results_' + idty_str + '/' + if parser_args.weight_training: model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, From 1ffc0000ad8fce0eb178f708f154aa1b5c189c50 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 17:25:47 -0500 Subject: [PATCH 022/113] adding config for serial debug --- configs/ddp_debug/conf2.yml | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 configs/ddp_debug/conf2.yml diff --git a/configs/ddp_debug/conf2.yml b/configs/ddp_debug/conf2.yml new file mode 100644 index 00000000..3df27c06 --- /dev/null +++ b/configs/ddp_debug/conf2.yml @@ -0,0 +1,62 @@ +# subfolder: target_sparsity_0_59_unflagT_real + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 20 +wd: 0.0 +momentum: 0.9 +batch_size: 512 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 + +# ===== Hardware setup ===== # +workers: 8 +# gpu: 3 +mixed_precision: True + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From 53b90c0d5541c76c1b97022030bd6953a996401e Mon Sep 17 00:00:00 2001 From: root Date: Thu, 28 Apr 2022 21:15:38 -0500 Subject: [PATCH 023/113] stuff works, removing an unnecessary barrier --- configs/ddp_debug/conf2.yml | 2 +- data/cifar.py | 2 +- main_utils.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/ddp_debug/conf2.yml b/configs/ddp_debug/conf2.yml index 3df27c06..6ac12696 100644 --- a/configs/ddp_debug/conf2.yml +++ b/configs/ddp_debug/conf2.yml @@ -52,7 +52,7 @@ lmbda: 0.0001 # 1e-4 # ===== Hardware setup ===== # workers: 8 -# gpu: 3 +gpu: 0 mixed_precision: True # ===== Checkpointing ===== # diff --git a/data/cifar.py b/data/cifar.py index a782b917..2ee881ed 100644 --- a/data/cifar.py +++ b/data/cifar.py @@ -17,7 +17,7 @@ def __init__(self, args): use_cuda = torch.cuda.is_available() # Data loading code - kwargs = {"num_workers": parser_args.workers, "pin_memory": True} if use_cuda else {} + kwargs = {"num_workers": parser_args.num_workers, "pin_memory": True} if use_cuda else {} normalize = transforms.Normalize( mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262] diff --git a/main_utils.py b/main_utils.py index 991b8178..1439182e 100644 --- a/main_utils.py +++ b/main_utils.py @@ -95,10 +95,10 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b # do the sanity check for shuffled mask/weights, reinit weights print("Sanity Check 1: Weight Reinit") - if parser_args.multiprocessing_distributed: - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + # if parser_args.multiprocessing_distributed: + # print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + # dist.barrier() + # print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, From 10b1ee4424ef9e711e73ed9e0fe3f7d1bb49cf22 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 29 Apr 2022 16:59:55 +0000 Subject: [PATCH 024/113] adding imagenet configs and exec script --- .../resnet50/imagenet/resnet50_sparsity_5.yml | 68 +++++++++++++++++++ .../resnet50_sc_hypercube_reg_exp1.yml | 0 .../resnet50_sc_hypercube_reg_exp2.yml | 0 .../resnet50_sc_hypercube_reg_exp3.yml | 0 imagenet_exec.sh | 29 ++++++++ main_utils.py | 4 +- 6 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml rename configs/hypercube/resnet50/{ => old_configs}/resnet50_sc_hypercube_reg_exp1.yml (100%) rename configs/hypercube/resnet50/{ => old_configs}/resnet50_sc_hypercube_reg_exp2.yml (100%) rename configs/hypercube/resnet50/{ => old_configs}/resnet50_sc_hypercube_reg_exp3.yml (100%) create mode 100644 imagenet_exec.sh diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml new file mode 100644 index 00000000..4eb571f9 --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml @@ -0,0 +1,68 @@ +# subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/ILSVRC2012/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.004 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 12 +multiprocessing_distributed: True +mixed_precision: True +# gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp1.yml b/configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp1.yml similarity index 100% rename from configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp1.yml rename to configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp1.yml diff --git a/configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp2.yml b/configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp2.yml similarity index 100% rename from configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp2.yml rename to configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp2.yml diff --git a/configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp3.yml b/configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp3.yml similarity index 100% rename from configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp3.yml rename to configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp3.yml diff --git a/imagenet_exec.sh b/imagenet_exec.sh new file mode 100644 index 00000000..3e992ff9 --- /dev/null +++ b/imagenet_exec.sh @@ -0,0 +1,29 @@ +# Running trials in parallel +# NOTE: make sure to delete/comment subfolder from the config file or else it may not work +:< "$log_root$trial$log_end" 2>&1 & + + python main.py \ + --config "$conf_file" \ + --trial-num $trial \ + --invert-sanity-check \ + --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +done + +BLOCK + +conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" +log_root="resnet50_sp5" +log_end="_log" +python main.py \ + --config "$conf_file" > "$log_root$log_end" 2>&1 & diff --git a/main_utils.py b/main_utils.py index 1439182e..1af1101b 100644 --- a/main_utils.py +++ b/main_utils.py @@ -11,7 +11,7 @@ import random import time import pandas as pd -from torch.utils.tensorboard import SummaryWriter +# from torch.utils.tensorboard import SummaryWriter import torch import torch.nn as nn import torch.nn.parallel @@ -441,7 +441,7 @@ def get_settings(parser_args): run_base_dir, ckpt_base_dir, log_base_dir = get_directories(parser_args) parser_args.ckpt_base_dir = ckpt_base_dir - writer = SummaryWriter(log_dir=log_base_dir) + writer = None # SummaryWriter(log_dir=log_base_dir) # writer = None # epoch_time = AverageMeter("epoch_time", ":.4f", write_avg=False) # validation_time = AverageMeter("validation_time", ":.4f", write_avg=False) From 38550af457cb9e5f5ec4ed33b2cc978e82a2d3bf Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 3 May 2022 02:48:51 +0000 Subject: [PATCH 025/113] setting find_unused_parameters=False and adding config for sp10 --- .../imagenet/resnet50_sparsity_10.yml | 68 +++++++++++++++++++ main_utils.py | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml new file mode 100644 index 00000000..8a5bf08e --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml @@ -0,0 +1,68 @@ +# subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/ILSVRC2012/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.04 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 10 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000001 # 1e-7 + +# ===== Hardware setup ===== # +workers: 10 +multiprocessing_distributed: True +mixed_precision: True +# gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/main_utils.py b/main_utils.py index 1af1101b..11242bda 100644 --- a/main_utils.py +++ b/main_utils.py @@ -920,7 +920,7 @@ def set_gpu(parser_args, model): if parser_args.multiprocessing_distributed: # TODO: not sure about find_unused_parameters. Need to check model = nn.parallel.DistributedDataParallel( - model, device_ids=[parser_args.gpu], find_unused_parameters=True) + model, device_ids=[parser_args.gpu], find_unused_parameters=False) return model From 50805e31aa22fe5ed89cc32fb552bf49fb6e9837 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 4 May 2022 20:30:52 +0000 Subject: [PATCH 026/113] pushing config for sp15 imagenet --- cifar_exec.sh | 8 +-- .../imagenet/resnet50_sparsity_15.yml | 68 +++++++++++++++++++ .../resnet50/imagenet/resnet50_sparsity_5.yml | 2 +- main_utils.py | 2 +- 4 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml diff --git a/cifar_exec.sh b/cifar_exec.sh index 6ba68efa..8986d2f8 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -64,15 +64,15 @@ BLOCK #:< "$log_root$trial$log_end" 2>&1 & diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml new file mode 100644 index 00000000..c36fd337 --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml @@ -0,0 +1,68 @@ +# subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/ILSVRC2012/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.04 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 10 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00000005 # 5e-8 + +# ===== Hardware setup ===== # +workers: 12 +multiprocessing_distributed: True +mixed_precision: True +# gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml index 4eb571f9..a7cf06bd 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml @@ -19,7 +19,7 @@ data: /home/ubuntu/ILSVRC2012/ optimizer: sgd lr: 0.4 #0.01 lr_policy: cosine_lr #constant_lr #multistep_lr -fine_tune_lr: 0.004 +fine_tune_lr: 0.04 fine_tune_lr_policy: multistep_lr # ===== Network training config ===== # diff --git a/main_utils.py b/main_utils.py index 11242bda..64f7b3dc 100644 --- a/main_utils.py +++ b/main_utils.py @@ -537,7 +537,7 @@ def get_mask(model): def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' - os.environ['MASTER_PORT'] = '29500' + os.environ['MASTER_PORT'] = '29600' dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) From fb1eab0e32a5f31a5a70b0d867810af5dff8b176 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 4 May 2022 21:33:08 +0000 Subject: [PATCH 027/113] minor tweaks to configs for imagenet --- .../imagenet/resnet50_sparsity_10.yml | 2 +- .../imagenet/resnet50_sparsity_15.yml | 2 +- .../imagenet/resnet50_sparsity_25.yml | 68 +++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_25.yml diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml index 8a5bf08e..6d6ea32a 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml @@ -56,7 +56,7 @@ regularization: L2 lmbda: 0.0000001 # 1e-7 # ===== Hardware setup ===== # -workers: 10 +workers: 12 multiprocessing_distributed: True mixed_precision: True # gpu: 1 diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml index c36fd337..5a9b6a0a 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml @@ -35,7 +35,7 @@ bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK # enter target sparsity here -target_sparsity: 10 +target_sparsity: 15 # decide if you want to "unflag" unflag_before_finetune: False init: signed_constant diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_25.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_25.yml new file mode 100644 index 00000000..4adf0655 --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_25.yml @@ -0,0 +1,68 @@ +# subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/ILSVRC2012/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.04 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 25 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00000001 # 1e-8 + +# ===== Hardware setup ===== # +workers: 12 +multiprocessing_distributed: True +mixed_precision: True +# gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From 5ccc7a32c3c7467a2bf2168273373b8022ad6ee9 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 4 May 2022 16:45:49 -0500 Subject: [PATCH 028/113] adding port as param for multiple runs --- args_helper.py | 58 ++++++++++++++++++++++++++++++++++---------------- main_utils.py | 2 +- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/args_helper.py b/args_helper.py index 94370efa..c73d83e2 100644 --- a/args_helper.py +++ b/args_helper.py @@ -878,35 +878,52 @@ def parse_arguments(self, jupyter_mode=False): default=0, help="Use mixed precision or not" ) - parser.add_argument('--transformer_emsize', type=int, default=200, - help='size of word embeddings') - parser.add_argument('--transformer_nhid', type=int, default=200, - help='number of hidden units per layer') - parser.add_argument('--transformer_nlayers', type=int, default=2, - help='number of layers') - parser.add_argument('--transformer_clip', type=float, default=0.25, - help='gradient clipping') - parser.add_argument('--transformer_bptt', type=int, default=35, - help='sequence length') - parser.add_argument('--transformer_dropout', type=float, default=0.2, - help='dropout applied to layers (0 = no dropout)') - parser.add_argument('--transformer_nhead', type=int, default=2, - help='the number of heads in the encoder/decoder of the transformer model') - + parser.add_argument('--transformer_emsize', + type=int, default=200, + help='size of word embeddings' + ) + parser.add_argument('--transformer_nhid', + type=int, + default=200, + help='number of hidden units per layer' + ) + parser.add_argument('--transformer_nlayers', + type=int, + default=2, + help='number of layers' + ) + parser.add_argument('--transformer_clip', + type=float, + default=0.25, + help='gradient clipping' + ) + parser.add_argument('--transformer_bptt', + type=int, + default=35, + help='sequence length' + ) + parser.add_argument('--transformer_dropout', + type=float, + default=0.2, + help='dropout applied to layers (0 = no dropout)' + ) + parser.add_argument('--transformer_nhead', + type=int, + default=2, + help='the number of heads in the encoder/decoder of the transformer model' + ) parser.add_argument( "--only-sanity", action="store_true", default=False, help="Only run sanity checks on the files in specific directory or subdirectories" ) - parser.add_argument( "--invert-sanity-check", action="store_true", default=False, help="Enable this to run the inverted sanity check (for HC)" ) - parser.add_argument( "--sanity-folder", default=None, @@ -914,7 +931,6 @@ def parse_arguments(self, jupyter_mode=False): metavar="PATH", help="directory(s) to access for only sanity check", ) - parser.add_argument( "--sr-version", default=1, @@ -927,6 +943,12 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="Enable this use full train data and not leave anything for validation" ) + parser.add_argument( + "--port", + default=29500, + type=int, + help="Specify port to use for DDP", + ) if jupyter_mode: args = parser.parse_args("") diff --git a/main_utils.py b/main_utils.py index 64f7b3dc..a95ef5da 100644 --- a/main_utils.py +++ b/main_utils.py @@ -537,7 +537,7 @@ def get_mask(model): def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' - os.environ['MASTER_PORT'] = '29600' + os.environ['MASTER_PORT'] = parser_args.port dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) From 9052870128c86c16df4ab6ee5be30f2a4a5cee4c Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 6 May 2022 13:42:17 -0500 Subject: [PATCH 029/113] updating port to be a str not int --- main_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main_utils.py b/main_utils.py index a95ef5da..4cfdf812 100644 --- a/main_utils.py +++ b/main_utils.py @@ -537,7 +537,7 @@ def get_mask(model): def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' - os.environ['MASTER_PORT'] = parser_args.port + os.environ['MASTER_PORT'] = '{}'.format(parser_args.port) dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) From 2e17a43aa659b32ee7a81a7142e8e3e2347bdc42 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sat, 7 May 2022 12:22:20 -0500 Subject: [PATCH 030/113] pushing psutil memory logs --- main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/main.py b/main.py index 5116f2f5..4119197b 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ from main_utils import * +import psutil def main(): @@ -143,6 +144,7 @@ def main_worker(gpu, ngpus_per_node): # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): + print("STARTING TRAINING: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) # lr_policy(epoch, iteration=None) @@ -162,9 +164,11 @@ def main_worker(gpu, ngpus_per_node): # train for one epoch start_train = time.time() + print("BEFORE TRAIN LOOP: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) train_acc1, train_acc5, train_acc10, reg_loss = train( data.train_loader, model, criterion, optimizer, epoch, parser_args, writer=writer, scaler=scaler ) + print("AFTER TRAIN LOOP: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) # train_time.update((time.time() - start_train) / 60) train_time = (time.time() - start_train) / 60 @@ -173,6 +177,7 @@ def main_worker(gpu, ngpus_per_node): # evaluate on validation set if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: start_validation = time.time() + print("BEFORE VAL LOOP: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) if parser_args.algo in ['hc', 'hc_iter']: br_acc1, br_acc5, br_acc10 = validate( data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding @@ -196,10 +201,13 @@ def main_worker(gpu, ngpus_per_node): print('Acc: {}'.format(acc1)) # validation_time.update((time.time() - start_validation) / 60) validation_time = (time.time() - start_validation) / 60 + print("AFTER VAL LOOP: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) # prune the model every T_{prune} epochs if not parser_args.weight_training and parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: + print("BEFORE PRUNE: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) prune(model) + print("AFTER PRUNE: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) if parser_args.checkpoint_at_prune: if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: save_checkpoint_at_prune(model, parser_args) @@ -275,6 +283,7 @@ def main_worker(gpu, ngpus_per_node): results_filename = result_root + 'acc_and_sparsity.csv' print("Writing results into: {}".format(results_filename)) results_df.to_csv(results_filename, index=False) + print("AFTER TRAINING: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) print("Local rank: {} | About to enter save model logic".format(parser_args.gpu)) if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: # save checkpoint before fine-tuning From bb604abd164a5a6796091f90dcdedff1cf542bf7 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sat, 7 May 2022 12:36:59 -0500 Subject: [PATCH 031/113] adding more memory logs --- trainers/default.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trainers/default.py b/trainers/default.py index 8b1754c9..4ba71154 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -40,6 +40,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # for i, (images, target) in tqdm.tqdm( # enumerate(train_loader), ascii=True, total=len(train_loader) # ): + print("(TRAINER)BEFORE TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) for i, (images, target) in enumerate(train_loader): # measure data loading time data_time = time.time() - end @@ -116,17 +117,19 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # writer, prefix="train", global_step=t) print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) - + print("(TRAINER)AFTER TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) # before completing training, clean up model based on latest scores # update score thresholds for global ep if args.algo in ['global_ep', 'global_ep_iter']: prune(model, update_thresholds_only=True) if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: + print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) for name, params in model.named_parameters(): if "score" in name: scores = params with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) + print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() From 7b21e75d29cc04b8ae2e854274fb5d71eed12a9a Mon Sep 17 00:00:00 2001 From: root Date: Sat, 7 May 2022 12:52:17 -0500 Subject: [PATCH 032/113] adding more psutil logs --- cifar_exec.sh | 6 +++--- .../resnet20_sparsity_1_44_unflagT_real.yml | 4 +++- trainers/default.py | 16 ++++++++++------ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 8986d2f8..570f8208 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -64,11 +64,11 @@ BLOCK #:< Date: Sat, 7 May 2022 18:09:14 +0000 Subject: [PATCH 033/113] updating memory logs only for master --- trainers/default.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index 3952c30a..6592a30b 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -41,7 +41,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # for i, (images, target) in tqdm.tqdm( # enumerate(train_loader), ascii=True, total=len(train_loader) # ): - print("(TRAINER)BEFORE TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + if args.gpu == 0: + print("(TRAINER)BEFORE TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) for i, (images, target) in enumerate(train_loader): # measure data loading time data_time = time.time() - end @@ -75,16 +76,19 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler raise NotImplementedError # please check finetune_loss repo regularization_loss = torch.tensor(0) + if args.gpu == 0: + print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) if args.regularization: regularization_loss =\ get_regularization_loss(model, regularizer=args.regularization, lmbda=args.lmbda, alpha=args.alpha, alpha_prime=args.alpha_prime) - + if args.gpu == 0: + print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) #print('regularization_loss: ', regularization_loss) loss += regularization_loss - - print("(TRAINER)BEFORE ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + if args.gpu == 0: + print("(TRAINER)BEFORE ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # measure accuracy and record loss acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) # losses.update(loss.item(), images.size(0)) @@ -97,7 +101,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler top10 += acc10.item() * images.size(0) num_images += images.size(0) - print("(TRAINER)AFTER ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + if args.gpu == 0: + print("(TRAINER)AFTER ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # compute gradient and do SGD step optimizer.zero_grad() @@ -127,13 +132,15 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler if args.algo in ['global_ep', 'global_ep_iter']: prune(model, update_thresholds_only=True) if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: - print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + if args.gpu == 0: + print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) for name, params in model.named_parameters(): if "score" in name: scores = params with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) - print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + if args.gpu == 0: + print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() From 2e199d46c421ce63ad31980d36193e2ff34370a8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 8 May 2022 00:25:15 +0000 Subject: [PATCH 034/113] updating some net_utils functions which are recreating ddp objects --- trainers/default.py | 30 ++++++++++++++---------------- utils/net_utils.py | 27 ++++++++++++++++----------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index 6592a30b..647300be 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -9,7 +9,7 @@ from utils.net_utils import get_regularization_loss, prune, get_layers from torch import optim -import psutil +import psutil, sys __all__ = ["train", "validate", "modifier"] @@ -41,9 +41,11 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # for i, (images, target) in tqdm.tqdm( # enumerate(train_loader), ascii=True, total=len(train_loader) # ): - if args.gpu == 0: - print("(TRAINER)BEFORE TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + print("(TRAINER)BEFORE TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) for i, (images, target) in enumerate(train_loader): + # print("(TRAINER)AFTER LOADING IMAGES: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("Is it just the image?") + # print("(TRAINER): Size of images: {}".format(sys.getsizeof(images))) # measure data loading time data_time = time.time() - end # print("Data Time: {}".format(data_time)) @@ -56,12 +58,14 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler if args.algo in ['global_ep', 'global_ep_iter']: prune(model, update_thresholds_only=True) + # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) if args.algo in ['hc', 'hc_iter', 'pt'] and i % args.project_freq == 0 and not args.differentiate_clamp: for name, params in model.named_parameters(): if "score" in name: scores = params with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) + # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # compute output if scaler is None: @@ -76,19 +80,16 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler raise NotImplementedError # please check finetune_loss repo regularization_loss = torch.tensor(0) - if args.gpu == 0: - print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) if args.regularization: regularization_loss =\ get_regularization_loss(model, regularizer=args.regularization, lmbda=args.lmbda, alpha=args.alpha, alpha_prime=args.alpha_prime) - if args.gpu == 0: - print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) #print('regularization_loss: ', regularization_loss) loss += regularization_loss - if args.gpu == 0: - print("(TRAINER)BEFORE ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("(TRAINER)BEFORE ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # measure accuracy and record loss acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) # losses.update(loss.item(), images.size(0)) @@ -101,8 +102,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler top10 += acc10.item() * images.size(0) num_images += images.size(0) - if args.gpu == 0: - print("(TRAINER)AFTER ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("(TRAINER)AFTER ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # compute gradient and do SGD step optimizer.zero_grad() @@ -126,21 +126,19 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # writer, prefix="train", global_step=t) print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) - print("(TRAINER)AFTER TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("(TRAINER)AFTER TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # before completing training, clean up model based on latest scores # update score thresholds for global ep if args.algo in ['global_ep', 'global_ep_iter']: prune(model, update_thresholds_only=True) if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: - if args.gpu == 0: - print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) for name, params in model.named_parameters(): if "score" in name: scores = params with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) - if args.gpu == 0: - print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() diff --git a/utils/net_utils.py b/utils/net_utils.py index 9e5828d9..d300c9a5 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -18,9 +18,11 @@ # return layer objects of conv layers and linear layers so we can parse them # efficiently -def get_layers(arch='Conv4', model=None): - if isinstance(model, nn.parallel.DistributedDataParallel): - model = model.module +def get_layers(arch='Conv4', dist_model=None): + if isinstance(dist_model, nn.parallel.DistributedDataParallel): + model = dist_model.module + else: + model = dist_model if arch == 'Conv4': conv_layers = [model.convs[0], model.convs[2], model.convs[5], model.convs[7]] @@ -312,11 +314,14 @@ def forward(self, model, temperature=1.0): # rounds model by round_scheme and returns the rounded model def round_model(model, round_scheme, noise=False, ratio=0.0, rank=None): print("Rounding model with scheme: {}".format(round_scheme)) + cp_model = copy.deepcopy(model) if isinstance(model, nn.parallel.DistributedDataParallel): - cp_model = copy.deepcopy(model.module) + # cp_model = copy.deepcopy(model.module) + named_params = cp_model.module.named_parameters() else: - cp_model = copy.deepcopy(model) - for name, params in cp_model.named_parameters(): + # cp_model = copy.deepcopy(model) + named_params = cp_model.named_parameters() + for name, params in named_params: if ".score" in name: if noise: delta = torch.randn_like(params.data)*ratio @@ -350,7 +355,7 @@ def round_model(model, round_scheme, noise=False, ratio=0.0, rank=None): # if isinstance(model, nn.parallel.DistributedDataParallel): # cp_model = nn.parallel.DistributedDataParallel( - # cp_model, device_ids=[rank], find_unused_parameters=True) + # cp_model, device_ids=[parser_args.gpu], find_unused_parameters=False) return cp_model @@ -475,8 +480,8 @@ def prune(model, update_thresholds_only=False, update_scores=False): # returns avg_sparsity = number of non-zero weights! def get_model_sparsity(model, threshold=0): - if isinstance(model, nn.parallel.DistributedDataParallel): - model = model.module + # if isinstance(model, nn.parallel.DistributedDataParallel): + # model = model.module conv_layers, linear_layers = get_layers(parser_args.arch, model) numer = 0 denom = 0 @@ -556,8 +561,8 @@ def get_layer_sparsity(layer, threshold=0): def get_regularization_loss(model, regularizer='L2', lmbda=1, alpha=1, alpha_prime=1): - if isinstance(model, nn.parallel.DistributedDataParallel): - model = model.module + # if isinstance(model, nn.parallel.DistributedDataParallel): + # model = model.module conv_layers, linear_layers = get_layers(parser_args.arch, model) def get_special_reg_sum(layer): From e3a4b8c3dd61179e143b0345ea236acdb60497ff Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sat, 7 May 2022 19:45:25 -0500 Subject: [PATCH 035/113] updating validate. I think that's where the leak is --- trainers/default.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index 647300be..cb4535ef 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -132,13 +132,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler if args.algo in ['global_ep', 'global_ep_iter']: prune(model, update_thresholds_only=True) if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: - print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) for name, params in model.named_parameters(): if "score" in name: scores = params with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) - print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() @@ -166,8 +166,8 @@ def validate(val_loader, model, criterion, args, writer, epoch): # enumerate(val_loader), ascii=True, total=len(val_loader) # ): for i, (images, target) in enumerate(val_loader): - images = images.cuda(args.gpu, non_blocking=True) - target = target.cuda(args.gpu, non_blocking=True) + images = images.to(args.gpu) + target = target.to(args.gpu) #print(images.shape, target.shape) From da11ccbf74006fdda00f25829f4b63cd927d0040 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sat, 7 May 2022 19:52:01 -0500 Subject: [PATCH 036/113] adding more fixes to hopefully handle the memory leak --- main.py | 9 ++++++--- main_utils.py | 10 ++++++++-- utils/net_utils.py | 3 ++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 4119197b..da5117eb 100644 --- a/main.py +++ b/main.py @@ -221,9 +221,12 @@ def main_worker(gpu, ngpus_per_node): avg_sparsity = get_model_sparsity(cp_model) elif parser_args.algo in ['hc', 'hc_iter']: # Round before checking sparsity - cp_model = round_model(model, parser_args.round, noise=parser_args.noise, - ratio=parser_args.noise_ratio, rank=parser_args.gpu) - avg_sparsity = get_model_sparsity(cp_model) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + cp_model = round_model(model, parser_args.round, noise=parser_args.noise, + ratio=parser_args.noise_ratio, rank=parser_args.gpu) + avg_sparsity = get_model_sparsity(cp_model) + else: + avg_sparsity = -1 else: avg_sparsity = get_model_sparsity(model) else: diff --git a/main_utils.py b/main_utils.py index 4cfdf812..07251bd0 100644 --- a/main_utils.py +++ b/main_utils.py @@ -264,12 +264,18 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) # check sparsity - post_round_sparsity = get_model_sparsity(model) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + post_round_sparsity = get_model_sparsity(model) + else: + post_round_sparsity = -1 else: # round the score (in the model itself) model = round_model(model, round_scheme=parser_args.round, noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) - post_round_sparsity = get_model_sparsity(model) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + post_round_sparsity = get_model_sparsity(model) + else: + post_round_sparsity = -1 elif parser_args.algo in ['ep']: post_round_sparsity = get_model_sparsity(model) diff --git a/utils/net_utils.py b/utils/net_utils.py index d300c9a5..6c960df6 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -132,7 +132,8 @@ def get_layers(arch='Conv4', dist_model=None): def redraw(model, shuffle=False, reinit=False, invert=False, chg_mask=False, chg_weight=False): - cp_model = copy.deepcopy(model) + # cp_model = copy.deepcopy(model) + cp_model = model conv_layers, linear_layers = get_layers(parser_args.arch, cp_model) for layer in (conv_layers + linear_layers): if shuffle: From c9b64cc83e0f42729ce474ea00f340368a7479ac Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 8 May 2022 01:14:36 +0000 Subject: [PATCH 037/113] updating val logs with gpu --- main.py | 4 ++-- trainers/default.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index da5117eb..e1024026 100644 --- a/main.py +++ b/main.py @@ -177,7 +177,7 @@ def main_worker(gpu, ngpus_per_node): # evaluate on validation set if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: start_validation = time.time() - print("BEFORE VAL LOOP: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + print("BEFORE VAL LOOP: GPU: {} | Epoch {} | Memory Usage: {}".format(parser_args.gpu, epoch, psutil.virtual_memory())) if parser_args.algo in ['hc', 'hc_iter']: br_acc1, br_acc5, br_acc10 = validate( data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding @@ -201,7 +201,7 @@ def main_worker(gpu, ngpus_per_node): print('Acc: {}'.format(acc1)) # validation_time.update((time.time() - start_validation) / 60) validation_time = (time.time() - start_validation) / 60 - print("AFTER VAL LOOP: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + print("AFTER VAL LOOP: GPU: {} | Epoch {} | Memory Usage: {}".format(parser_args.gpu, epoch, psutil.virtual_memory())) # prune the model every T_{prune} epochs if not parser_args.weight_training and parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: diff --git a/trainers/default.py b/trainers/default.py index cb4535ef..774cc9f4 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -126,7 +126,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # writer, prefix="train", global_step=t) print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) - # print("(TRAINER)AFTER TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + print("(TRAINER)AFTER TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # before completing training, clean up model based on latest scores # update score thresholds for global ep if args.algo in ['global_ep', 'global_ep_iter']: From f112628ab56336f899714592e6a8ad96ca6cdbe6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 8 May 2022 15:45:42 +0000 Subject: [PATCH 038/113] adding debug loop --- main.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/main.py b/main.py index e1024026..af1a83be 100644 --- a/main.py +++ b/main.py @@ -151,6 +151,20 @@ def main_worker(gpu, ngpus_per_node): modifier(parser_args, epoch, model) cur_lr = get_lr(optimizer) + # print("Skipping training, just gonna round") + # print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, + # ratio=parser_args.noise_ratio, rank=parser_args.gpu) + # if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + # acc1, acc5, acc10 = validate(data.val_loader, cp_model, criterion, parser_args, writer, epoch) + # else: + # acc1 = -1 + # print("GPU: {} | acc1={}".format(parser_args.gpu, acc1)) + # print("After Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + # dist.barrier() + # continue + + # save the score at the beginning of training epoch, so if we set parser.args.rewind_to_epoch to 0 # that means we save the initialization of score if parser_args.rewind_score and parser_args.rewind_to_epoch == epoch: From a0603b3181d57148b1c45b09f96ab13ddc4af168 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 8 May 2022 15:46:12 +0000 Subject: [PATCH 039/113] adding memory log to help debug --- resnet50_sp5_log | 17832 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 17832 insertions(+) create mode 100644 resnet50_sp5_log diff --git a/resnet50_sp5_log b/resnet50_sp5_log new file mode 100644 index 00000000..c8afe019 --- /dev/null +++ b/resnet50_sp5_log @@ -0,0 +1,17832 @@ +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Namespace(algo='hc_iter', alpha=1.0, alpha_prime=1.0, arch='ResNet50', batch_size=1024, bias=False, bn_type='NonAffineBatchNorm', bottom_k_on_forward=False, checkpoint_at_prune=False, chg_mask=False, chg_weight=False, ckpt_interval=-1, config='configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml', conv_type='SubnetConv', data='/home/ubuntu/ILSVRC2012/', data_dir='../data', dataset='ImageNet', dense_training=False, differentiate_clamp=False, dist_backend='nccl', epochs=88, evaluate=False, evaluate_only=False, fine_tune_lr=0.04, fine_tune_lr_policy='multistep_lr', fine_tune_optimizer='sgd', fine_tune_wd=0.0001, first_layer_dense=False, first_layer_type=None, fixed_init=False, flips=None, freeze_weights=True, gamma=0.1, gpu=0, hc_period=1, hc_quantized=True, hc_warmup=9999, hidden_size=500, how_to_connect='prob', how_to_prune='random', imp_no_rewind=False, imp_resume_epoch=-1, imp_resume_iter=-1, imp_resume_round=-1, imp_rewind_iter=1000, imp_rewind_model='short_imp/Liu_checkpoint_model_correct.pth', imp_rounds=-1, init='signed_constant', interpolate='prob', invert_sanity_check=False, iter_period=5, iter_start=0, label_smoothing=None, lam_finetune_loss=-1, last_layer_dense=False, lmbda=1e-06, load_ckpt=None, log_dir=None, log_interval=2, loss='cross-entropy-loss', lr=0.4, lr_adjust=50, lr_gamma=0.1, lr_policy='cosine_lr', max_iter=100000, metric='loss', milestones=[50, 100, 150, 200], mixed_precision=True, mode='fan_in', mode_connect=False, mode_connect_filename=None, momentum=0.9, multiprocessing_distributed=True, name='resnet50_imagenet', nesterov=False, no_bn_decay=False, no_cuda=False, noise=True, noise_ratio=0, nonlinearity='relu', num_round=1, num_step_finetune=10, num_test=1, num_trial=1, num_workers=4, only_sanity=False, optimizer='sgd', override_prune_rate=False, plot_hc_convergence=False, port=29500, pretrained=None, pretrained2=None, print_freq=10, project_freq=1, prune_rate=0.5, prune_type='BottomK', pruning_strategy=None, quantize_threshold=0.5, random_subnet=False, rank=-1, regularization='L2', reinit=False, results_filename=None, resume=None, rewind_score=False, rewind_to_epoch=-1, round='naive', run_idx=None, sanity_folder=None, save_every=-1, save_model=False, save_plot_data=False, scale_fan=False, score_init='unif', score_init_constant=None, seed=42, seed_fixed_init=24, shift=0.0, shuffle=False, skip_fine_tune=False, skip_sanity_checks=True, smart_ratio=-1, sr_version=1, start_epoch=None, start_from_nothing=False, subfolder=None, submask_size=1, target_sparsity=5, td=0.99, temp=100000, trainer='default', transformer_bptt=35, transformer_clip=0.25, transformer_dropout=0.2, transformer_emsize=200, transformer_nhead=2, transformer_nhid=200, transformer_nlayers=2, trial_num=1, unflag_before_finetune=False, use_full_data=False, warmup_length=0, wd=0.0, weight_training=False, width=1.0, width_mult=1.0, workers=12, world_size=-1, **{'compare-rounding': False}) + + +Beginning of process. + + +-------------------------------------- +TIME: The current time is: Sun May 8 01:33:46 2022 +TIME: The current time in seconds is: 1651973626.3474913 +-------------------------------------- + + +Seeded everything: 42 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Use GPU: 2 for training +=> Using trainer from trainers.default +=> Creating model 'ResNet50' +==> Conv Type: SubnetConv +==> BN Type: NonAffineBatchNorm +==> Building first layer +==> Setting prune rate of network to 0.5 +=> Rough estimate model params 25502912 +=> Freezing model weights +=> Getting ImageNet dataset +scheduler: use cosine learning rate decay, with max epochs 88 +Computing prune_rate for target_sparsity 5 with iter_period 5 +Setting prune_rate to 0.16156611126074005 +STARTING TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241612877824, percent=6.2, used=13615095808, free=102254149632, active=115461599232, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +BEFORE TRAIN LOOP: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241612361728, percent=6.2, used=13615611904, free=102253633536, active=115461599232, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=241611845632, percent=6.2, used=13616128000, free=102253117440, active=115462115328, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Use GPU: 1 for training +=> Using trainer from trainers.default +=> Creating model 'ResNet50' +==> Conv Type: SubnetConv +==> BN Type: NonAffineBatchNorm +==> Building first layer +==> Setting prune rate of network to 0.5 +=> Rough estimate model params 25502912 +=> Freezing model weights +=> Getting ImageNet dataset +scheduler: use cosine learning rate decay, with max epochs 88 +Computing prune_rate for target_sparsity 5 with iter_period 5 +Setting prune_rate to 0.16156611126074005 +STARTING TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241611845632, percent=6.2, used=13616128000, free=102253117440, active=115462115328, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +BEFORE TRAIN LOOP: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241611329536, percent=6.2, used=13616644096, free=102252601344, active=115462631424, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=241610297344, percent=6.2, used=13617676288, free=102251569152, active=115463147520, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Use GPU: 0 for training +=> Using trainer from trainers.default +=> Creating model 'ResNet50' +==> Conv Type: SubnetConv +==> BN Type: NonAffineBatchNorm +==> Building first layer +==> Setting prune rate of network to 0.5 +=> Rough estimate model params 25502912 +=> Freezing model weights +conv1.scores 9408 +layer1.0.conv1.scores 4096 +layer1.0.conv2.scores 36864 +layer1.0.conv3.scores 16384 +layer1.0.downsample.0.scores 16384 +layer1.1.conv1.scores 16384 +layer1.1.conv2.scores 36864 +layer1.1.conv3.scores 16384 +layer1.2.conv1.scores 16384 +layer1.2.conv2.scores 36864 +layer1.2.conv3.scores 16384 +layer2.0.conv1.scores 32768 +layer2.0.conv2.scores 147456 +layer2.0.conv3.scores 65536 +layer2.0.downsample.0.scores 131072 +layer2.1.conv1.scores 65536 +layer2.1.conv2.scores 147456 +layer2.1.conv3.scores 65536 +layer2.2.conv1.scores 65536 +layer2.2.conv2.scores 147456 +layer2.2.conv3.scores 65536 +layer2.3.conv1.scores 65536 +layer2.3.conv2.scores 147456 +layer2.3.conv3.scores 65536 +layer3.0.conv1.scores 131072 +layer3.0.conv2.scores 589824 +layer3.0.conv3.scores 262144 +layer3.0.downsample.0.scores 524288 +layer3.1.conv1.scores 262144 +layer3.1.conv2.scores 589824 +layer3.1.conv3.scores 262144 +layer3.2.conv1.scores 262144 +layer3.2.conv2.scores 589824 +layer3.2.conv3.scores 262144 +layer3.3.conv1.scores 262144 +layer3.3.conv2.scores 589824 +layer3.3.conv3.scores 262144 +layer3.4.conv1.scores 262144 +layer3.4.conv2.scores 589824 +layer3.4.conv3.scores 262144 +layer3.5.conv1.scores 262144 +layer3.5.conv2.scores 589824 +layer3.5.conv3.scores 262144 +layer4.0.conv1.scores 524288 +layer4.0.conv2.scores 2359296 +layer4.0.conv3.scores 1048576 +layer4.0.downsample.0.scores 2097152 +layer4.1.conv1.scores 1048576 +layer4.1.conv2.scores 2359296 +layer4.1.conv3.scores 1048576 +layer4.2.conv1.scores 1048576 +layer4.2.conv2.scores 2359296 +layer4.2.conv3.scores 1048576 +fc.scores 2048000 +total num_params: 25502912 +=> Getting ImageNet dataset +scheduler: use cosine learning rate decay, with max epochs 88 +Computing prune_rate for target_sparsity 5 with iter_period 5 +Setting prune_rate to 0.16156611126074005 +1 SubnetConv(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) +2 SubnetConv(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) +3 SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +4 SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +5 SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) +6 SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +7 SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +8 SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) +9 SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +10 SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +11 SubnetConv(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) +12 SubnetConv(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) +13 SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +14 SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) +15 SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +16 SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +17 SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) +18 SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +19 SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +20 SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) +21 SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +22 SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +23 SubnetConv(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +24 SubnetConv(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) +25 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +26 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +27 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +28 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +29 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +30 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +31 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +32 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +33 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +34 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +35 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +36 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +37 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +38 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +39 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +40 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +41 SubnetConv(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +42 SubnetConv(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) +43 SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) +44 SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +45 SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +46 SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) +47 SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +48 SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +49 SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) +50 SubnetConv(2048, 1000, kernel_size=(1, 1), stride=(1, 1), bias=False) +STARTING TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241581137920, percent=6.2, used=13646835712, free=102222409728, active=115492798464, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +BEFORE TRAIN LOOP: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241581137920, percent=6.2, used=13646835712, free=102222409728, active=115492798464, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=241580105728, percent=6.2, used=13647867904, free=102221377536, active=115493830656, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Use GPU: 3 for training +=> Using trainer from trainers.default +=> Creating model 'ResNet50' +==> Conv Type: SubnetConv +==> BN Type: NonAffineBatchNorm +==> Building first layer +==> Setting prune rate of network to 0.5 +=> Rough estimate model params 25502912 +=> Freezing model weights +=> Getting ImageNet dataset +scheduler: use cosine learning rate decay, with max epochs 88 +Computing prune_rate for target_sparsity 5 with iter_period 5 +Setting prune_rate to 0.16156611126074005 +STARTING TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241542430720, percent=6.2, used=13685542912, free=102183702528, active=115531616256, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +BEFORE TRAIN LOOP: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241542430720, percent=6.2, used=13685542912, free=102183702528, active=115531616256, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=241541906432, percent=6.2, used=13686067200, free=102183178240, active=115531616256, inactive=36025708544, buffers=439332864, cached=141259505664, shared=89284608, slab=3021799424) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:3 | Epoch: 0 | loss=15.897546768188477 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.919440269470215 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.836463928222656 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.729747772216797 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.696290969848633 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.606315612792969 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.6333589553833 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.62976360321045 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.518330574035645 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.513955116271973 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.554726600646973 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.477972984313965 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.517154693603516 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.52071762084961 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.443882942199707 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.4263916015625 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.510221481323242 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.424978256225586 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.498486518859863 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.411236763000488 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.42553424835205 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.444916725158691 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.400117874145508 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.4298734664917 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.383928298950195 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.374913215637207 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.328813552856445 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.382004737854004 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.38232135772705 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.383122444152832 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.332008361816406 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.33960247039795 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.334253311157227 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.377694129943848 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.37434196472168 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.35669231414795 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.315393447875977 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.36469554901123 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.327455520629883 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.318841934204102 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.358086585998535 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.28050422668457 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.351791381835938 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.331706047058105 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.34184455871582 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.329983711242676 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.334991455078125 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.335689544677734 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.329496383666992 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.253458023071289 | Batch Time=1.171875 +GPU:3 | Epoch: 0 | loss=15.306257247924805 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.324240684509277 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.328380584716797 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.321075439453125 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.262478828430176 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.320180892944336 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.275586128234863 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.31775951385498 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.31445026397705 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.187150955200195 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.248161315917969 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.288976669311523 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.323262214660645 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.288165092468262 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.297030448913574 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.278400421142578 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.24661922454834 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.292609214782715 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.240499496459961 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.282447814941406 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.261531829833984 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.222585678100586 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.246706008911133 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.224384307861328 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.254654884338379 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.245268821716309 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.188647270202637 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.22134017944336 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.23762035369873 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.190470695495605 | Batch Time=1.953125 +GPU:3 | Epoch: 0 | loss=15.185077667236328 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.200868606567383 | Batch Time=1.171875 +GPU:3 | Epoch: 0 | loss=15.217936515808105 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.187634468078613 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.170722007751465 | Batch Time=1.171875 +GPU:3 | Epoch: 0 | loss=15.176645278930664 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.206673622131348 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.194429397583008 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.169129371643066 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.185288429260254 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.17661190032959 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.188385963439941 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.18398380279541 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.200703620910645 | Batch Time=1.171875 +GPU:3 | Epoch: 0 | loss=15.15052318572998 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.234118461608887 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.189876556396484 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.208829879760742 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.174973487854004 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.112994194030762 | Batch Time=1.171875 +GPU:3 | Epoch: 0 | loss=15.160467147827148 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.137565612792969 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.158589363098145 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.169411659240723 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.143414497375488 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.167740821838379 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.173796653747559 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.181897163391113 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.132911682128906 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.144314765930176 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.088141441345215 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.17871379852295 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.204256057739258 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.172012329101562 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.076581954956055 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.182543754577637 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.097877502441406 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.095534324645996 | Batch Time=1.171875 +GPU:3 | Epoch: 0 | loss=15.122121810913086 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.150240898132324 | Batch Time=0.0 +GPU:3 | Epoch: 0 | loss=15.126382827758789 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.148660659790039 | Batch Time=0.390625 +GPU:3 | Epoch: 0 | loss=15.074715614318848 | Batch Time=1.953125 +GPU:3 | Epoch: 0 | loss=15.080741882324219 | Batch Time=0.78125 +GPU:3 | Epoch: 0 | loss=15.089303970336914 | Batch Time=0.390625 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531812352, percent=9.7, used=18358841344, free=79642345472, active=150668443648, inactive=23187869696, buffers=439427072, cached=159127470080, shared=4426604544, slab=3125215232) +GPU:0 | Epoch: 0 | loss=15.875452041625977 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=16.029930114746094 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.966346740722656 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.749553680419922 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.59929084777832 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.699508666992188 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.667157173156738 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.586520195007324 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.53142261505127 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.565163612365723 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.595322608947754 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.54776668548584 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.440326690673828 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.39150619506836 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.456883430480957 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.484664916992188 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.495832443237305 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.350217819213867 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.487988471984863 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.371960639953613 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.371502876281738 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.452332496643066 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.406343460083008 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.375597953796387 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.417543411254883 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.31042194366455 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.483270645141602 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.395722389221191 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.431797981262207 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.358860969543457 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.369850158691406 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.370631217956543 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.346208572387695 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.372246742248535 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.365903854370117 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.383349418640137 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.342401504516602 | Batch Time=1.171875 +GPU:0 | Epoch: 0 | loss=15.341639518737793 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.341562271118164 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.309816360473633 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.28261661529541 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.311784744262695 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.361534118652344 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.29181957244873 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.314676284790039 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.30272388458252 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.318851470947266 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.327495574951172 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.346715927124023 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.280786514282227 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.291654586791992 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.292410850524902 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.334064483642578 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.309150695800781 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.340420722961426 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.308164596557617 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.264630317687988 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.247462272644043 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.295666694641113 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.278497695922852 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.249229431152344 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.223821640014648 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.285649299621582 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.246478080749512 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.290392875671387 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.282527923583984 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.218832969665527 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.297263145446777 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.328550338745117 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.204277038574219 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.19637680053711 | Batch Time=1.171875 +GPU:0 | Epoch: 0 | loss=15.279718399047852 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.238176345825195 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.22940444946289 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.244393348693848 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.209547996520996 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.23505687713623 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.184600830078125 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.210124015808105 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.233229637145996 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.175464630126953 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.228754043579102 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.189833641052246 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.169835090637207 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.140521049499512 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.164957046508789 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.197190284729004 | Batch Time=1.5625 +GPU:0 | Epoch: 0 | loss=15.149049758911133 | Batch Time=1.171875 +GPU:0 | Epoch: 0 | loss=15.212403297424316 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.214238166809082 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.166741371154785 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.155224800109863 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.213112831115723 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.212265968322754 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.16162395477295 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.157748222351074 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.168827056884766 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.200597763061523 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.133904457092285 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.179919242858887 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.185186386108398 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.155967712402344 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.123127937316895 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.163369178771973 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.197270393371582 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.169251441955566 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.16641902923584 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.137585639953613 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.137435913085938 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.133084297180176 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.136950492858887 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.150256156921387 | Batch Time=1.171875 +GPU:0 | Epoch: 0 | loss=15.124330520629883 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.194564819335938 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.16102409362793 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.159998893737793 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.148841857910156 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.12576961517334 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.094636917114258 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.147875785827637 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.104372024536133 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=15.100549697875977 | Batch Time=0.78125 +GPU:0 | Epoch: 0 | loss=15.07564640045166 | Batch Time=1.953125 +GPU:0 | Epoch: 0 | loss=15.144508361816406 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=15.149232864379883 | Batch Time=0.78125 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531812352, percent=9.7, used=18358841344, free=79642345472, active=150668443648, inactive=23187869696, buffers=439427072, cached=159127470080, shared=4426604544, slab=3125215232) +AFTER TRAIN LOOP: Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +STARTING TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +BEFORE TRAIN LOOP: Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +GPU:1 | Epoch: 0 | loss=15.859315872192383 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.90543270111084 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.783332824707031 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.74038314819336 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.76042366027832 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.66534423828125 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.67040729522705 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.5798978805542 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.55772876739502 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.575509071350098 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.43709659576416 | Batch Time=1.171875 +GPU:1 | Epoch: 0 | loss=15.511435508728027 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.48270034790039 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.463436126708984 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.421101570129395 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.42022705078125 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.471860885620117 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.432523727416992 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.44616413116455 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.43248462677002 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.473980903625488 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.404603004455566 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.39671516418457 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.408312797546387 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.463579177856445 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.40599536895752 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.391946792602539 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.390534400939941 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.363057136535645 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.352131843566895 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.353958129882812 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.3266019821167 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.343446731567383 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.377800941467285 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.367307662963867 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.339652061462402 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.359979629516602 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.345911979675293 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.326959609985352 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.377702713012695 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.324707984924316 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.324098587036133 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.340476989746094 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.285761833190918 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.298776626586914 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.354405403137207 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.314979553222656 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.294666290283203 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.26826286315918 | Batch Time=1.171875 +GPU:1 | Epoch: 0 | loss=15.303499221801758 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.330648422241211 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.292914390563965 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.263774871826172 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.32562255859375 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.26659870147705 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.292997360229492 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.295880317687988 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.263148307800293 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.252881050109863 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.250894546508789 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.311912536621094 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.236227035522461 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.245427131652832 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.277972221374512 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.2912015914917 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.292980194091797 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.281882286071777 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.271285057067871 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.218107223510742 | Batch Time=1.171875 +GPU:1 | Epoch: 0 | loss=15.246223449707031 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.240673065185547 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.247129440307617 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.24498176574707 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.26511001586914 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.187508583068848 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.23160457611084 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.16513729095459 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.271217346191406 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.229273796081543 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.18826961517334 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.241695404052734 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.213541030883789 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.270995140075684 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.186894416809082 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.227580070495605 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.18266487121582 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.221077919006348 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.177766799926758 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.214722633361816 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.195530891418457 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.159287452697754 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.225937843322754 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.17080020904541 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.143815040588379 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.160609245300293 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.153918266296387 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.139354705810547 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.178693771362305 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.196175575256348 | Batch Time=1.5625 +GPU:1 | Epoch: 0 | loss=15.184626579284668 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.160177230834961 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.13348388671875 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.149731636047363 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.16103458404541 | Batch Time=1.953125 +GPU:1 | Epoch: 0 | loss=15.119145393371582 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.159981727600098 | Batch Time=1.171875 +GPU:1 | Epoch: 0 | loss=15.168898582458496 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.15697193145752 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.124431610107422 | Batch Time=1.171875 +GPU:1 | Epoch: 0 | loss=15.153225898742676 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.067656517028809 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.120928764343262 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.15904426574707 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.094581604003906 | Batch Time=1.5625 +GPU:1 | Epoch: 0 | loss=15.158750534057617 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.059893608093262 | Batch Time=1.171875 +GPU:1 | Epoch: 0 | loss=15.154415130615234 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.125021934509277 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.087713241577148 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.122942924499512 | Batch Time=0.0 +GPU:1 | Epoch: 0 | loss=15.104188919067383 | Batch Time=1.5625 +GPU:1 | Epoch: 0 | loss=15.088518142700195 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.09291934967041 | Batch Time=0.390625 +GPU:1 | Epoch: 0 | loss=15.059127807617188 | Batch Time=0.78125 +GPU:1 | Epoch: 0 | loss=15.136857986450195 | Batch Time=0.0 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531812352, percent=9.7, used=18358841344, free=79642345472, active=150668443648, inactive=23187869696, buffers=439427072, cached=159127470080, shared=4426604544, slab=3125215232) +GPU:2 | Epoch: 0 | loss=15.978952407836914 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=16.03191375732422 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.871635437011719 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.770336151123047 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.72551155090332 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.719039916992188 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.61669635772705 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.53376293182373 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.61394214630127 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.444878578186035 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.478028297424316 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.500662803649902 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.42654800415039 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.438701629638672 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.497227668762207 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.447402954101562 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.501310348510742 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.428190231323242 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.40893268585205 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.45092487335205 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.429120063781738 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.435563087463379 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.367677688598633 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.360499382019043 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.386079788208008 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.354390144348145 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.375699996948242 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.344941139221191 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.417271614074707 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.36364459991455 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.373863220214844 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.368739128112793 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.284219741821289 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.360116004943848 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.414106369018555 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.346407890319824 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.343942642211914 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.293538093566895 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.317445755004883 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.317033767700195 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.351754188537598 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.339677810668945 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.3172607421875 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.337855339050293 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.337060928344727 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.31786060333252 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.368911743164062 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.29434585571289 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.286855697631836 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.25987434387207 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.308530807495117 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.288023948669434 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.325321197509766 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.264755249023438 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.31670093536377 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.346296310424805 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.316815376281738 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.249690055847168 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.3096284866333 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.28166389465332 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.286354064941406 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.324621200561523 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.271031379699707 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.230380058288574 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.264948844909668 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.24166488647461 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.206839561462402 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.21530818939209 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.239538192749023 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.259452819824219 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.256206512451172 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.29932975769043 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.245656967163086 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.243289947509766 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.241791725158691 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.228774070739746 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.302767753601074 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.203117370605469 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.192523002624512 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.193160057067871 | Batch Time=1.5625 +GPU:2 | Epoch: 0 | loss=15.208507537841797 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.215982437133789 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.190917015075684 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.290318489074707 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.218897819519043 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.224481582641602 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.203293800354004 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.195291519165039 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.163437843322754 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.182343482971191 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.21694278717041 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.147828102111816 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.188149452209473 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.15407657623291 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.163348197937012 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.24646282196045 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.141109466552734 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.132665634155273 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.145424842834473 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.184374809265137 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.126150131225586 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.156257629394531 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.152558326721191 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.214982032775879 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.146000862121582 | Batch Time=1.171875 +GPU:2 | Epoch: 0 | loss=15.152726173400879 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.164645195007324 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.115628242492676 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.181282043457031 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.13073444366455 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.156033515930176 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.172503471374512 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.103181838989258 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.11358642578125 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.126913070678711 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.089953422546387 | Batch Time=1.5625 +GPU:2 | Epoch: 0 | loss=15.158012390136719 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.164572715759277 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.130620956420898 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.104998588562012 | Batch Time=0.0 +GPU:2 | Epoch: 0 | loss=15.125429153442383 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.077054977416992 | Batch Time=1.5625 +GPU:2 | Epoch: 0 | loss=15.142067909240723 | Batch Time=0.78125 +GPU:2 | Epoch: 0 | loss=15.111156463623047 | Batch Time=0.390625 +GPU:2 | Epoch: 0 | loss=15.094274520874023 | Batch Time=0.78125 +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531812352, percent=9.7, used=18358841344, free=79642345472, active=150668443648, inactive=23187869696, buffers=439427072, cached=159127470080, shared=4426604544, slab=3125215232) +AFTER TRAIN LOOP: Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +BEFORE VAL LOOP: GPU: 0 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +AFTER TRAIN LOOP: Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +STARTING TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +BEFORE TRAIN LOOP: Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +AFTER TRAIN LOOP: Epoch 0 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +STARTING TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +BEFORE TRAIN LOOP: Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=232531836928, percent=9.7, used=18358980608, free=79642370048, active=150668345344, inactive=23187804160, buffers=439427072, cached=159127306240, shared=4426440704, slab=3125190656) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 0 | loss=6.3017120361328125 | Batch Time=13.671875 +GPU:0 | Epoch: 0 | loss=6.833003997802734 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.730043888092041 | Batch Time=1.5625 +GPU:0 | Epoch: 0 | loss=6.917082786560059 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.825292110443115 | Batch Time=1.171875 +GPU:0 | Epoch: 0 | loss=6.880558967590332 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.668310642242432 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=6.812735080718994 | Batch Time=1.5625 +GPU:0 | Epoch: 0 | loss=6.894185543060303 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.839200496673584 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.750614166259766 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=6.868747711181641 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.8743577003479 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.827610492706299 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.667627811431885 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.916213512420654 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.848036289215088 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.842231750488281 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.515531063079834 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.482290744781494 | Batch Time=3.515625 +Model top1 Accuracy: 0.698 +Acc before rounding: 0.698 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 0 | loss=6.3017120361328125 | Batch Time=13.671875 +GPU:0 | Epoch: 0 | loss=6.833003997802734 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.730043888092041 | Batch Time=1.5625 +GPU:0 | Epoch: 0 | loss=6.917082786560059 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.825292110443115 | Batch Time=1.171875 +GPU:0 | Epoch: 0 | loss=6.880558967590332 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.668310642242432 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=6.812735080718994 | Batch Time=1.5625 +GPU:0 | Epoch: 0 | loss=6.894185543060303 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.839200496673584 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.750614166259766 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=6.868747711181641 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.8743577003479 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.827610492706299 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.667627811431885 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.916213512420654 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.848036289215088 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.842231750488281 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.515531063079834 | Batch Time=0.0 +GPU:0 | Epoch: 0 | loss=6.482290744781494 | Batch Time=3.515625 +Model top1 Accuracy: 0.698 +Acc after rounding: 0.698 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 0 | loss=6.8107829093933105 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=6.8668742179870605 | Batch Time=0.390625 +GPU:0 | Epoch: 0 | loss=6.7270121574401855 | Batch Time=1.171875 +GPU:0 | Epoch: 0 | loss=6.804818153381348 | Batch Time=0.390625 +Model top1 Accuracy: 0.69 +Validation Acc after rounding: 0.69 +AFTER VAL LOOP: GPU: 0 | Epoch 0 | Memory Usage: svmem(total=257568083968, available=213588922368, percent=17.1, used=36376682496, free=60612296704, active=168853209088, inactive=23910060032, buffers=439492608, cached=160139612160, shared=5351645184, slab=3135545344) +Rounding model with scheme: naive +Model avg sparsity: 49.490072497529695 +GPU:0 | Epoch: 0 | Acc=0.698 | Epoch Time=15.971008133888244 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=213588922368, percent=17.1, used=36376682496, free=60612296704, active=168853233664, inactive=23910060032, buffers=439492608, cached=160139612160, shared=5351645184, slab=3135545344) +STARTING TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=213588922368, percent=17.1, used=36376682496, free=60612296704, active=168853233664, inactive=23910060032, buffers=439492608, cached=160139612160, shared=5351645184, slab=3135545344) +BEFORE TRAIN LOOP: Epoch 1 | Memory Usage: svmem(total=257568083968, available=213588922368, percent=17.1, used=36376682496, free=60612296704, active=168853233664, inactive=23910060032, buffers=439492608, cached=160139612160, shared=5351645184, slab=3135545344) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=213588922368, percent=17.1, used=36376682496, free=60612296704, active=168853233664, inactive=23910060032, buffers=439492608, cached=160139612160, shared=5351645184, slab=3135545344) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:2 | Epoch: 1 | loss=15.093867301940918 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.070402145385742 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=15.070540428161621 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=15.125298500061035 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=15.116060256958008 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=15.078313827514648 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=15.067536354064941 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=15.080958366394043 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=15.126660346984863 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=15.06307315826416 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.10631275177002 | Batch Time=2.34375 +GPU:2 | Epoch: 1 | loss=15.111413955688477 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.04477596282959 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.01540470123291 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=15.089763641357422 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.951763153076172 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.07502269744873 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.034714698791504 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=15.0758056640625 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.05726432800293 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.067991256713867 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.981465339660645 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=15.020472526550293 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=15.052714347839355 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.965456008911133 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=15.040568351745605 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.991595268249512 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.037510871887207 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.049826622009277 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=15.011921882629395 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.044942855834961 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=15.05056095123291 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.982044219970703 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=15.037298202514648 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.996283531188965 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.979519844055176 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=15.063475608825684 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=15.022594451904297 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=15.049293518066406 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.00964641571045 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.016804695129395 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.991076469421387 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=15.03668212890625 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.959492683410645 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.99902057647705 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.997835159301758 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.948801040649414 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.992835998535156 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.926039695739746 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.881315231323242 | Batch Time=2.34375 +GPU:2 | Epoch: 1 | loss=14.991345405578613 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.89550495147705 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.965048789978027 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=14.989463806152344 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.937935829162598 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.961893081665039 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.902926445007324 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.92707633972168 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.952034950256348 | Batch Time=2.34375 +GPU:2 | Epoch: 1 | loss=14.98582649230957 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.939634323120117 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.973427772521973 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.923855781555176 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=14.938552856445312 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.962641716003418 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.926801681518555 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.969759941101074 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.949271202087402 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=14.883304595947266 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.887078285217285 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.90653133392334 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=14.917530059814453 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.91254711151123 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.943475723266602 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.936639785766602 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.846909523010254 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.924643516540527 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=14.802830696105957 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.852710723876953 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.845592498779297 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.95946216583252 | Batch Time=0.0 +GPU:2 | Epoch: 1 | loss=14.778487205505371 | Batch Time=2.34375 +GPU:2 | Epoch: 1 | loss=14.864562034606934 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.715370178222656 | Batch Time=3.125 +GPU:2 | Epoch: 1 | loss=14.892215728759766 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.811043739318848 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.827335357666016 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.842696189880371 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.922219276428223 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.840599060058594 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.826430320739746 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.88990306854248 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.88584041595459 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.822970390319824 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.800702095031738 | Batch Time=3.515625 +GPU:2 | Epoch: 1 | loss=14.820371627807617 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.865859031677246 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.795246124267578 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.872209548950195 | Batch Time=2.34375 +GPU:2 | Epoch: 1 | loss=14.782492637634277 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.795391082763672 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.769426345825195 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.825729370117188 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.796317100524902 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.842642784118652 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.760281562805176 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.7151460647583 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.743810653686523 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.776683807373047 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.744169235229492 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.766522407531738 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.765667915344238 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.751333236694336 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.725442886352539 | Batch Time=3.125 +GPU:2 | Epoch: 1 | loss=14.707108497619629 | Batch Time=2.34375 +GPU:2 | Epoch: 1 | loss=14.668182373046875 | Batch Time=2.34375 +GPU:2 | Epoch: 1 | loss=14.738358497619629 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.790416717529297 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.80024528503418 | Batch Time=0.78125 +GPU:2 | Epoch: 1 | loss=14.770212173461914 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.714460372924805 | Batch Time=1.171875 +GPU:2 | Epoch: 1 | loss=14.729912757873535 | Batch Time=1.953125 +GPU:2 | Epoch: 1 | loss=14.75271224975586 | Batch Time=1.5625 +GPU:2 | Epoch: 1 | loss=14.659371376037598 | Batch Time=0.390625 +GPU:2 | Epoch: 1 | loss=14.648908615112305 | Batch Time=2.734375 +GPU:1 | Epoch: 1 | loss=15.088641166687012 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.108747482299805 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=15.101592063903809 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=15.079834938049316 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=15.118471145629883 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=15.159978866577148 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=15.092652320861816 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=15.08276653289795 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=15.038174629211426 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=15.069596290588379 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=15.03161334991455 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=15.034624099731445 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=15.049948692321777 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=15.10307788848877 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=15.085323333740234 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=15.098400115966797 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=15.0755033493042 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.994011878967285 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.098304748535156 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.080198287963867 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.042829513549805 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=15.062176704406738 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.001666069030762 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.080309867858887 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.110551834106445 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=15.045451164245605 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=15.069659233093262 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=15.04588794708252 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.056937217712402 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=14.949570655822754 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.04304313659668 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.991097450256348 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.982135772705078 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=15.026296615600586 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.012541770935059 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.986981391906738 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=15.014708518981934 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.956348419189453 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=15.047767639160156 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.986811637878418 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.978384971618652 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=15.01220989227295 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=14.981712341308594 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=14.94872760772705 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.981823921203613 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=15.014459609985352 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.97044563293457 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.951766967773438 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.939406394958496 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.979642868041992 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.972119331359863 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.931706428527832 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.939574241638184 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.9171142578125 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.880913734436035 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.934686660766602 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.991724967956543 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.958662033081055 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.973034858703613 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.904590606689453 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.935419082641602 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.936070442199707 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.936436653137207 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=14.955482482910156 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.898043632507324 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.897235870361328 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.925719261169434 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.911616325378418 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.911701202392578 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.940140724182129 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.883734703063965 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.952518463134766 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.80102825164795 | Batch Time=1.953125 +GPU:1 | Epoch: 1 | loss=14.903383255004883 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=14.901620864868164 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.930520057678223 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.832884788513184 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.852696418762207 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.892765045166016 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.842365264892578 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.817090034484863 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.925436973571777 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.842413902282715 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.903640747070312 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.814743041992188 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.84472370147705 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.846092224121094 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.817793846130371 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.899781227111816 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.869983673095703 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.794825553894043 | Batch Time=2.34375 +GPU:1 | Epoch: 1 | loss=14.897212028503418 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.866408348083496 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.778315544128418 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=14.785550117492676 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.801782608032227 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.89041805267334 | Batch Time=0.0 +GPU:1 | Epoch: 1 | loss=14.74007797241211 | Batch Time=2.34375 +GPU:1 | Epoch: 1 | loss=14.798646926879883 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.824736595153809 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.88509750366211 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.680337905883789 | Batch Time=2.734375 +GPU:1 | Epoch: 1 | loss=14.747550964355469 | Batch Time=1.953125 +GPU:1 | Epoch: 1 | loss=14.695273399353027 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.741049766540527 | Batch Time=2.734375 +GPU:1 | Epoch: 1 | loss=14.758213996887207 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.824246406555176 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.725595474243164 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.738712310791016 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.783758163452148 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.77788257598877 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.7709321975708 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.773666381835938 | Batch Time=2.34375 +GPU:1 | Epoch: 1 | loss=14.682867050170898 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.764725685119629 | Batch Time=3.125 +GPU:1 | Epoch: 1 | loss=14.772270202636719 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.715889930725098 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.65700912475586 | Batch Time=1.5625 +GPU:1 | Epoch: 1 | loss=14.799848556518555 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.635037422180176 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.766035079956055 | Batch Time=0.390625 +GPU:1 | Epoch: 1 | loss=14.651825904846191 | Batch Time=1.171875 +GPU:1 | Epoch: 1 | loss=14.698997497558594 | Batch Time=0.78125 +GPU:1 | Epoch: 1 | loss=14.722506523132324 | Batch Time=1.953125 +GPU:1 | Epoch: 1 | loss=14.767782211303711 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.08228588104248 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=15.161382675170898 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=15.036627769470215 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=15.096505165100098 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.075822830200195 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=15.130987167358398 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=15.067734718322754 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=15.148921012878418 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.118000984191895 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=15.096695899963379 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=15.142842292785645 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.055421829223633 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=15.031645774841309 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.081814765930176 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.046253204345703 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.060222625732422 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.977198600769043 | Batch Time=2.34375 +GPU:0 | Epoch: 1 | loss=15.04934024810791 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=15.083473205566406 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=15.078489303588867 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=15.054159164428711 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=15.074475288391113 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=15.019122123718262 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.014178276062012 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=15.015565872192383 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=15.007647514343262 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=15.0093412399292 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.982991218566895 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.02735424041748 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.009453773498535 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=15.008100509643555 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=15.022568702697754 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.976253509521484 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.993803024291992 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.983702659606934 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.979413032531738 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.959761619567871 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.934810638427734 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.991943359375 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.994250297546387 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.031279563903809 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.008326530456543 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=14.991893768310547 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.975838661193848 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.055684089660645 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.977991104125977 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=15.011285781860352 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.977951049804688 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.953665733337402 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=15.002012252807617 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.939417839050293 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.962529182434082 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.973097801208496 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.998634338378906 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.910233497619629 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.919191360473633 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.83277416229248 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.960420608520508 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=14.934678077697754 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.943058013916016 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.966428756713867 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.903786659240723 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.911839485168457 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.887924194335938 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.984667778015137 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.841058731079102 | Batch Time=3.515625 +GPU:0 | Epoch: 1 | loss=14.874842643737793 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.861296653747559 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.893978118896484 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.966153144836426 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.889464378356934 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=14.900264739990234 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.862137794494629 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=14.86735725402832 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=14.869318008422852 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.863839149475098 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.84671688079834 | Batch Time=2.34375 +GPU:0 | Epoch: 1 | loss=14.873005867004395 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.906108856201172 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=14.84360122680664 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.90837574005127 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.901381492614746 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.89939022064209 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.806350708007812 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=14.77020263671875 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.808506965637207 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.800395965576172 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.788161277770996 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.859482765197754 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.847400665283203 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.857657432556152 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.802824974060059 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.868453025817871 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=14.93310832977295 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.860188484191895 | Batch Time=2.34375 +GPU:0 | Epoch: 1 | loss=14.773660659790039 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.734179496765137 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.809650421142578 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.819147109985352 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.800993919372559 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.82236099243164 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.825311660766602 | Batch Time=2.734375 +GPU:0 | Epoch: 1 | loss=14.787261962890625 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=14.791312217712402 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.81943416595459 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.7902193069458 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.824124336242676 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.73750114440918 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=14.8443603515625 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.796270370483398 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.7700777053833 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.748356819152832 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.718608856201172 | Batch Time=2.34375 +GPU:0 | Epoch: 1 | loss=14.835638046264648 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=14.850869178771973 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.772483825683594 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.802689552307129 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.758838653564453 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.777143478393555 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=14.743265151977539 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=14.782140731811523 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.719536781311035 | Batch Time=1.5625 +GPU:0 | Epoch: 1 | loss=14.679027557373047 | Batch Time=2.34375 +GPU:0 | Epoch: 1 | loss=14.76796817779541 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=14.670522689819336 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=15.127711296081543 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.084409713745117 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=15.052977561950684 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=15.105957984924316 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.041040420532227 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=15.149900436401367 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.093010902404785 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.086482048034668 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=15.05883502960205 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.071427345275879 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.092671394348145 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=15.121362686157227 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=15.046378135681152 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=15.03930377960205 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.046371459960938 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=15.037357330322266 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.092944145202637 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.03182315826416 | Batch Time=1.953125 +GPU:3 | Epoch: 1 | loss=15.007820129394531 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=15.055784225463867 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.109060287475586 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.070866584777832 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.029986381530762 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=15.022677421569824 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.034929275512695 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=15.025073051452637 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.052302360534668 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=15.070202827453613 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.062155723571777 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.9622163772583 | Batch Time=2.34375 +GPU:3 | Epoch: 1 | loss=15.062284469604492 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=15.025010108947754 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=15.00741195678711 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=15.008962631225586 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=15.030470848083496 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.99511432647705 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=15.078902244567871 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=15.06943130493164 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.910715103149414 | Batch Time=2.734375 +GPU:3 | Epoch: 1 | loss=14.969141960144043 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.991401672363281 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=15.07702922821045 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.950363159179688 | Batch Time=1.953125 +GPU:3 | Epoch: 1 | loss=15.01732349395752 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.959813117980957 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.959993362426758 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.921106338500977 | Batch Time=1.953125 +GPU:3 | Epoch: 1 | loss=14.934883117675781 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.995573997497559 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.961660385131836 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.942280769348145 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.98168659210205 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.988875389099121 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.951812744140625 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.920548439025879 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.98051643371582 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.957018852233887 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.996137619018555 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.950303077697754 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.94308090209961 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.955503463745117 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.913132667541504 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.910077095031738 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.929695129394531 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.932208061218262 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.916990280151367 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.947115898132324 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.945197105407715 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.944507598876953 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.898758888244629 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.933436393737793 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.90108871459961 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.834336280822754 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.842248916625977 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.904741287231445 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.833809852600098 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.885420799255371 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.965336799621582 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.848094940185547 | Batch Time=2.34375 +GPU:3 | Epoch: 1 | loss=14.988826751708984 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.85693073272705 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.930136680603027 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.837325096130371 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.86916732788086 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.871017456054688 | Batch Time=1.953125 +GPU:3 | Epoch: 1 | loss=14.858887672424316 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.828548431396484 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.795531272888184 | Batch Time=1.953125 +GPU:3 | Epoch: 1 | loss=14.876397132873535 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.862140655517578 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.842474937438965 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.817675590515137 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.888518333435059 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.811373710632324 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.839398384094238 | Batch Time=1.953125 +GPU:3 | Epoch: 1 | loss=14.765810012817383 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.766890525817871 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.824932098388672 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.84691047668457 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.71994686126709 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.723186492919922 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.846796035766602 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.784904479980469 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.967597007751465 | Batch Time=0.0 +GPU:3 | Epoch: 1 | loss=14.82669734954834 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.732518196105957 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.77238941192627 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.80933952331543 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.720378875732422 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.79066276550293 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.736451148986816 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.732735633850098 | Batch Time=2.34375 +GPU:3 | Epoch: 1 | loss=14.678878784179688 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.713537216186523 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.777047157287598 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.754142761230469 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.749787330627441 | Batch Time=1.171875 +GPU:3 | Epoch: 1 | loss=14.701427459716797 | Batch Time=0.390625 +GPU:3 | Epoch: 1 | loss=14.718652725219727 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.741495132446289 | Batch Time=2.34375 +GPU:3 | Epoch: 1 | loss=14.733762741088867 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.705857276916504 | Batch Time=2.734375 +GPU:3 | Epoch: 1 | loss=14.754993438720703 | Batch Time=1.5625 +GPU:3 | Epoch: 1 | loss=14.76656436920166 | Batch Time=0.78125 +GPU:3 | Epoch: 1 | loss=14.659116744995117 | Batch Time=1.5625 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=224182878208, percent=13.0, used=26091098112, free=66892275712, active=172983369728, inactive=13496295424, buffers=443375616, cached=164141334528, shared=5043273728, slab=3167752192) +AFTER TRAIN LOOP: Epoch 1 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +BEFORE VAL LOOP: GPU: 0 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=224182878208, percent=13.0, used=26091098112, free=66892275712, active=172983369728, inactive=13496295424, buffers=443375616, cached=164141334528, shared=5043273728, slab=3167752192) +AFTER TRAIN LOOP: Epoch 1 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +STARTING TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +BEFORE TRAIN LOOP: Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=224182878208, percent=13.0, used=26091098112, free=66892275712, active=172983369728, inactive=13496295424, buffers=443375616, cached=164141334528, shared=5043273728, slab=3167752192) +AFTER TRAIN LOOP: Epoch 1 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +STARTING TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +BEFORE TRAIN LOOP: Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=224182878208, percent=13.0, used=26091098112, free=66892275712, active=172983369728, inactive=13496295424, buffers=443375616, cached=164141334528, shared=5043273728, slab=3167752192) +AFTER TRAIN LOOP: Epoch 1 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +STARTING TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +BEFORE TRAIN LOOP: Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=224183160832, percent=13.0, used=26090962944, free=66892558336, active=172983291904, inactive=13496229888, buffers=443375616, cached=164141187072, shared=5043109888, slab=3167735808) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 1 | loss=5.2059550285339355 | Batch Time=8.984375 +GPU:0 | Epoch: 1 | loss=6.651482105255127 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.347459316253662 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=6.823805809020996 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=6.70438289642334 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=6.652111530303955 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.504240989685059 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.655132293701172 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.709140300750732 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.772781848907471 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.657020092010498 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.726808071136475 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=6.733271598815918 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=6.525564670562744 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.4666218757629395 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.9120683670043945 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.490158557891846 | Batch Time=3.125 +GPU:0 | Epoch: 1 | loss=6.478692054748535 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.335063457489014 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=5.8076958656311035 | Batch Time=5.859375 +Model top1 Accuracy: 1.438 +Acc before rounding: 1.438 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 1 | loss=5.2059550285339355 | Batch Time=8.984375 +GPU:0 | Epoch: 1 | loss=6.651482105255127 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.347459316253662 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=6.823805809020996 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=6.70438289642334 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=6.652111530303955 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.504240989685059 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.655132293701172 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.709140300750732 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.772781848907471 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.657020092010498 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.726808071136475 | Batch Time=0.78125 +GPU:0 | Epoch: 1 | loss=6.733271598815918 | Batch Time=0.390625 +GPU:0 | Epoch: 1 | loss=6.525564670562744 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.4666218757629395 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.9120683670043945 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.490158557891846 | Batch Time=3.125 +GPU:0 | Epoch: 1 | loss=6.478692054748535 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=6.335063457489014 | Batch Time=0.0 +GPU:0 | Epoch: 1 | loss=5.8076958656311035 | Batch Time=5.859375 +Model top1 Accuracy: 1.438 +Acc after rounding: 1.438 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 1 | loss=6.506982326507568 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=6.588831901550293 | Batch Time=1.171875 +GPU:0 | Epoch: 1 | loss=6.500247001647949 | Batch Time=1.953125 +GPU:0 | Epoch: 1 | loss=6.567180633544922 | Batch Time=2.734375 +Model top1 Accuracy: 1.58 +Validation Acc after rounding: 1.58 +AFTER VAL LOOP: GPU: 0 | Epoch 1 | Memory Usage: svmem(total=257568083968, available=205907619840, percent=20.1, used=43904544768, free=48603848704, active=190740963328, inactive=13909274624, buffers=443420672, cached=164616269824, shared=5505867776, slab=3174273024) +Rounding model with scheme: naive +Model avg sparsity: 48.98678262564784 +GPU:0 | Epoch: 1 | Acc=1.438 | Epoch Time=15.672431111335754 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=205907619840, percent=20.1, used=43904626688, free=48603848704, active=190740905984, inactive=13909241856, buffers=443420672, cached=164616187904, shared=5505785856, slab=3174273024) +STARTING TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=205907619840, percent=20.1, used=43904626688, free=48603848704, active=190740905984, inactive=13909241856, buffers=443420672, cached=164616187904, shared=5505785856, slab=3174273024) +BEFORE TRAIN LOOP: Epoch 2 | Memory Usage: svmem(total=257568083968, available=205907619840, percent=20.1, used=43904626688, free=48603848704, active=190740905984, inactive=13909241856, buffers=443420672, cached=164616187904, shared=5505785856, slab=3174273024) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=205907619840, percent=20.1, used=43904626688, free=48603848704, active=190740905984, inactive=13909241856, buffers=443420672, cached=164616187904, shared=5505785856, slab=3174273024) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:2 | Epoch: 2 | loss=14.751173973083496 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.712718963623047 | Batch Time=0.0 +GPU:2 | Epoch: 2 | loss=14.656946182250977 | Batch Time=3.515625 +GPU:2 | Epoch: 2 | loss=14.675252914428711 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.80139446258545 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.680705070495605 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.686196327209473 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.606810569763184 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.72276782989502 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.68307113647461 | Batch Time=0.0 +GPU:2 | Epoch: 2 | loss=14.700458526611328 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.615466117858887 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.721668243408203 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.656010627746582 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.699899673461914 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.668949127197266 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.63957691192627 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.618094444274902 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.604341506958008 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.781841278076172 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.676241874694824 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.623039245605469 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.628437042236328 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.689300537109375 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.627938270568848 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.525596618652344 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.559694290161133 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.656168937683105 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.638367652893066 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.591344833374023 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.72667121887207 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.52538013458252 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.561362266540527 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.579587936401367 | Batch Time=0.390625 +GPU:2 | Epoch: 2 | loss=14.568314552307129 | Batch Time=2.734375 +GPU:2 | Epoch: 2 | loss=14.534703254699707 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.55168628692627 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.555484771728516 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.473254203796387 | Batch Time=3.125 +GPU:2 | Epoch: 2 | loss=14.576175689697266 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.578457832336426 | Batch Time=2.734375 +GPU:2 | Epoch: 2 | loss=14.590747833251953 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.564998626708984 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.524410247802734 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.577710151672363 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.635455131530762 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.466235160827637 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.482995986938477 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.493769645690918 | Batch Time=3.125 +GPU:2 | Epoch: 2 | loss=14.48732852935791 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.472969055175781 | Batch Time=3.515625 +GPU:2 | Epoch: 2 | loss=14.505767822265625 | Batch Time=2.734375 +GPU:2 | Epoch: 2 | loss=14.42221450805664 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.561834335327148 | Batch Time=2.734375 +GPU:2 | Epoch: 2 | loss=14.568087577819824 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.567214012145996 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.51826000213623 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.434945106506348 | Batch Time=2.734375 +GPU:2 | Epoch: 2 | loss=14.514753341674805 | Batch Time=0.0 +GPU:2 | Epoch: 2 | loss=14.69329833984375 | Batch Time=0.390625 +GPU:2 | Epoch: 2 | loss=14.486377716064453 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.454753875732422 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.527535438537598 | Batch Time=3.125 +GPU:2 | Epoch: 2 | loss=14.601702690124512 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.452178955078125 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.42899227142334 | Batch Time=0.0 +GPU:2 | Epoch: 2 | loss=14.542608261108398 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.404537200927734 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.510563850402832 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.39432144165039 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.507952690124512 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.523805618286133 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.397721290588379 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.450912475585938 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.500650405883789 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.486178398132324 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.444750785827637 | Batch Time=2.734375 +GPU:2 | Epoch: 2 | loss=14.519930839538574 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.532139778137207 | Batch Time=0.390625 +GPU:2 | Epoch: 2 | loss=14.555639266967773 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.489555358886719 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.53456974029541 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.356143951416016 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.345998764038086 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.290833473205566 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.339056015014648 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.255904197692871 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.46950912475586 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.42618465423584 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.401057243347168 | Batch Time=0.390625 +GPU:2 | Epoch: 2 | loss=14.525991439819336 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.370135307312012 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.255712509155273 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.38021469116211 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.300315856933594 | Batch Time=0.390625 +GPU:2 | Epoch: 2 | loss=14.457633018493652 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.364474296569824 | Batch Time=3.125 +GPU:2 | Epoch: 2 | loss=14.456756591796875 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.334202766418457 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.319735527038574 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.376556396484375 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.387648582458496 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.493178367614746 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.323759078979492 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.454656600952148 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.382298469543457 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.293517112731934 | Batch Time=3.515625 +GPU:2 | Epoch: 2 | loss=14.213709831237793 | Batch Time=3.125 +GPU:2 | Epoch: 2 | loss=14.392212867736816 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.333467483520508 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.3186616897583 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.29651927947998 | Batch Time=2.34375 +GPU:2 | Epoch: 2 | loss=14.374448776245117 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.341588973999023 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.325514793395996 | Batch Time=1.5625 +GPU:2 | Epoch: 2 | loss=14.296746253967285 | Batch Time=3.125 +GPU:2 | Epoch: 2 | loss=14.255680084228516 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.303399085998535 | Batch Time=2.734375 +GPU:2 | Epoch: 2 | loss=14.363363265991211 | Batch Time=1.171875 +GPU:2 | Epoch: 2 | loss=14.435111999511719 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.257801055908203 | Batch Time=1.953125 +GPU:2 | Epoch: 2 | loss=14.369926452636719 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.294153213500977 | Batch Time=0.78125 +GPU:2 | Epoch: 2 | loss=14.242451667785645 | Batch Time=2.734375 +GPU:2 | Epoch: 2 | loss=14.185235977172852 | Batch Time=2.734375 +GPU:1 | Epoch: 2 | loss=14.768118858337402 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.704883575439453 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.581626892089844 | Batch Time=2.734375 +GPU:1 | Epoch: 2 | loss=14.677087783813477 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.825854301452637 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.626299858093262 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.64023208618164 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.69315242767334 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.712010383605957 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.69021224975586 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.70650863647461 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.656817436218262 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.73455810546875 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.814610481262207 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.681293487548828 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.727222442626953 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.600178718566895 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.654417991638184 | Batch Time=0.390625 +GPU:1 | Epoch: 2 | loss=14.692041397094727 | Batch Time=0.390625 +GPU:1 | Epoch: 2 | loss=14.625274658203125 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.667441368103027 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.666305541992188 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.603435516357422 | Batch Time=3.125 +GPU:1 | Epoch: 2 | loss=14.601005554199219 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.6788969039917 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.638534545898438 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.678167343139648 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.704501152038574 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.653008460998535 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.639219284057617 | Batch Time=0.390625 +GPU:1 | Epoch: 2 | loss=14.560495376586914 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.564911842346191 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.570067405700684 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.562215805053711 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.616070747375488 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.583630561828613 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.700783729553223 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.608772277832031 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.57052993774414 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.522743225097656 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.6487398147583 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.541454315185547 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.658344268798828 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.603229522705078 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.433865547180176 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.563776969909668 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.528647422790527 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.567934036254883 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.480778694152832 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.552044868469238 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.572715759277344 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.46170425415039 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.494709014892578 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.467618942260742 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.56555461883545 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.578940391540527 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.50251293182373 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.523072242736816 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.443906784057617 | Batch Time=3.125 +GPU:1 | Epoch: 2 | loss=14.452293395996094 | Batch Time=2.734375 +GPU:1 | Epoch: 2 | loss=14.426025390625 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.491966247558594 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.391987800598145 | Batch Time=3.90625 +GPU:1 | Epoch: 2 | loss=14.464264869689941 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.415279388427734 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.48352336883545 | Batch Time=2.734375 +GPU:1 | Epoch: 2 | loss=14.539052963256836 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.43116283416748 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.484501838684082 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.418529510498047 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.488299369812012 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.558481216430664 | Batch Time=0.390625 +GPU:1 | Epoch: 2 | loss=14.477036476135254 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.452415466308594 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.458078384399414 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.411456108093262 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.509203910827637 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.404833793640137 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.453648567199707 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.464773178100586 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.439964294433594 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.368447303771973 | Batch Time=2.734375 +GPU:1 | Epoch: 2 | loss=14.43045425415039 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.439210891723633 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.387506484985352 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.46127700805664 | Batch Time=0.390625 +GPU:1 | Epoch: 2 | loss=14.398917198181152 | Batch Time=0.390625 +GPU:1 | Epoch: 2 | loss=14.481014251708984 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.39985179901123 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.391093254089355 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.424528121948242 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.3757963180542 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.496488571166992 | Batch Time=2.734375 +GPU:1 | Epoch: 2 | loss=14.348283767700195 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.40924072265625 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.458563804626465 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.385340690612793 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.288246154785156 | Batch Time=3.125 +GPU:1 | Epoch: 2 | loss=14.41460132598877 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.4402494430542 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.330619812011719 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.323695182800293 | Batch Time=3.125 +GPU:1 | Epoch: 2 | loss=14.440081596374512 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.411611557006836 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.338094711303711 | Batch Time=0.390625 +GPU:1 | Epoch: 2 | loss=14.326146125793457 | Batch Time=3.90625 +GPU:1 | Epoch: 2 | loss=14.439604759216309 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.316637992858887 | Batch Time=2.734375 +GPU:1 | Epoch: 2 | loss=14.337944984436035 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.23744010925293 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.32275104522705 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.279765129089355 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.306425094604492 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.317037582397461 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.236227989196777 | Batch Time=3.515625 +GPU:1 | Epoch: 2 | loss=14.229195594787598 | Batch Time=1.171875 +GPU:1 | Epoch: 2 | loss=14.299625396728516 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.300881385803223 | Batch Time=3.125 +GPU:1 | Epoch: 2 | loss=14.234228134155273 | Batch Time=2.34375 +GPU:1 | Epoch: 2 | loss=14.340457916259766 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.254947662353516 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.415153503417969 | Batch Time=1.5625 +GPU:1 | Epoch: 2 | loss=14.306047439575195 | Batch Time=1.953125 +GPU:1 | Epoch: 2 | loss=14.268651008605957 | Batch Time=0.78125 +GPU:1 | Epoch: 2 | loss=14.280031204223633 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.669933319091797 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.774822235107422 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.661828994750977 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.700624465942383 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.691277503967285 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.594225883483887 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.641816139221191 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.689845085144043 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.68779468536377 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.77468490600586 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.718852996826172 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.748560905456543 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.684734344482422 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=14.640660285949707 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.716182708740234 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.62203598022461 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.74181842803955 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=14.628203392028809 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.663576126098633 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.65548324584961 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.619444847106934 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.585647583007812 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.590625762939453 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.660530090332031 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=14.587313652038574 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.566890716552734 | Batch Time=3.125 +GPU:0 | Epoch: 2 | loss=14.647176742553711 | Batch Time=3.125 +GPU:0 | Epoch: 2 | loss=14.632708549499512 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.590996742248535 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.573995590209961 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.622034072875977 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.60457706451416 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.641470909118652 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=14.595762252807617 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.580883979797363 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.534741401672363 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.563931465148926 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.590572357177734 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.566822052001953 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.53067398071289 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.462048530578613 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.573719024658203 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.503948211669922 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.551822662353516 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.52521800994873 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.489546775817871 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.645712852478027 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.510316848754883 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.556292533874512 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.65959644317627 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=14.524467468261719 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.61172103881836 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.55343246459961 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.640005111694336 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.468653678894043 | Batch Time=4.296875 +GPU:0 | Epoch: 2 | loss=14.552062034606934 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.420466423034668 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.640259742736816 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=14.467031478881836 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.480888366699219 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.43954086303711 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=14.577117919921875 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.477673530578613 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.4198637008667 | Batch Time=3.125 +GPU:0 | Epoch: 2 | loss=14.473388671875 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.52640438079834 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.491861343383789 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.40969467163086 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.486056327819824 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.459896087646484 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.520682334899902 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.705415725708008 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.424946784973145 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.418144226074219 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.559274673461914 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.54664134979248 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=14.389422416687012 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.50888729095459 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.378998756408691 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.365903854370117 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.433670043945312 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.373414039611816 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.411375999450684 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.349889755249023 | Batch Time=3.90625 +GPU:0 | Epoch: 2 | loss=14.418633460998535 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.47360610961914 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.411200523376465 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.400588989257812 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.320818901062012 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.326739311218262 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.355764389038086 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.518275260925293 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.402853012084961 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.475032806396484 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.434036254882812 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.430808067321777 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.408892631530762 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.380294799804688 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.344014167785645 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.408289909362793 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.424030303955078 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.31473445892334 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.319708824157715 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.334356307983398 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.349203109741211 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.271622657775879 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.307852745056152 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=14.219420433044434 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.447949409484863 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.239015579223633 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.360718727111816 | Batch Time=3.90625 +GPU:0 | Epoch: 2 | loss=14.381396293640137 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.385538101196289 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.253515243530273 | Batch Time=5.46875 +GPU:0 | Epoch: 2 | loss=14.328474998474121 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.246649742126465 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.316715240478516 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.208069801330566 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.406576156616211 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=14.318778991699219 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=14.27066421508789 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.300445556640625 | Batch Time=2.34375 +GPU:0 | Epoch: 2 | loss=14.35753059387207 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=14.403508186340332 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=14.215736389160156 | Batch Time=3.515625 +GPU:3 | Epoch: 2 | loss=14.72086238861084 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.68117904663086 | Batch Time=3.125 +GPU:3 | Epoch: 2 | loss=14.701990127563477 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.722875595092773 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.755785942077637 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.651755332946777 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.635489463806152 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.713347434997559 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.674103736877441 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.60422134399414 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.628498077392578 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.670748710632324 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.594036102294922 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.7053804397583 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.645450592041016 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.67397689819336 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.6686372756958 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.661368370056152 | Batch Time=0.390625 +GPU:3 | Epoch: 2 | loss=14.621484756469727 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.67953872680664 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.645102500915527 | Batch Time=0.390625 +GPU:3 | Epoch: 2 | loss=14.635883331298828 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.691631317138672 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.562881469726562 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.592890739440918 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.593063354492188 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.705633163452148 | Batch Time=0.0 +GPU:3 | Epoch: 2 | loss=14.613215446472168 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.577820777893066 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.591169357299805 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.662790298461914 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.527634620666504 | Batch Time=0.390625 +GPU:3 | Epoch: 2 | loss=14.654982566833496 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.592550277709961 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.629406929016113 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.54675006866455 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.671841621398926 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.71091079711914 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.572664260864258 | Batch Time=2.734375 +GPU:3 | Epoch: 2 | loss=14.59359359741211 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.544873237609863 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.549243927001953 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.578510284423828 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.591452598571777 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.549511909484863 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.611159324645996 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.56525707244873 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.64097785949707 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.492980003356934 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.575871467590332 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.553215026855469 | Batch Time=0.390625 +GPU:3 | Epoch: 2 | loss=14.550251007080078 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.607677459716797 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.566308975219727 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.609759330749512 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.528380393981934 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.595503807067871 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.459664344787598 | Batch Time=3.515625 +GPU:3 | Epoch: 2 | loss=14.538694381713867 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.563148498535156 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.415111541748047 | Batch Time=3.90625 +GPU:3 | Epoch: 2 | loss=14.463569641113281 | Batch Time=3.125 +GPU:3 | Epoch: 2 | loss=14.503193855285645 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.462260246276855 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.519012451171875 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.475035667419434 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.526067733764648 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.497631072998047 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.48037052154541 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.443058013916016 | Batch Time=2.734375 +GPU:3 | Epoch: 2 | loss=14.473605155944824 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.539674758911133 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.417359352111816 | Batch Time=3.125 +GPU:3 | Epoch: 2 | loss=14.505569458007812 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.530054092407227 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.4788236618042 | Batch Time=3.515625 +GPU:3 | Epoch: 2 | loss=14.543368339538574 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.453951835632324 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.414311408996582 | Batch Time=3.125 +GPU:3 | Epoch: 2 | loss=14.351682662963867 | Batch Time=2.734375 +GPU:3 | Epoch: 2 | loss=14.477523803710938 | Batch Time=0.390625 +GPU:3 | Epoch: 2 | loss=14.381211280822754 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.376567840576172 | Batch Time=3.515625 +GPU:3 | Epoch: 2 | loss=14.466646194458008 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.398667335510254 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.420345306396484 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.452086448669434 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.452091217041016 | Batch Time=2.734375 +GPU:3 | Epoch: 2 | loss=14.368086814880371 | Batch Time=2.734375 +GPU:3 | Epoch: 2 | loss=14.409411430358887 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.372419357299805 | Batch Time=4.296875 +GPU:3 | Epoch: 2 | loss=14.360560417175293 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.415700912475586 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.397380828857422 | Batch Time=3.90625 +GPU:3 | Epoch: 2 | loss=14.458160400390625 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.386187553405762 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.345438957214355 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.393943786621094 | Batch Time=0.390625 +GPU:3 | Epoch: 2 | loss=14.405911445617676 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.43408489227295 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.365913391113281 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.288931846618652 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.283438682556152 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.41413688659668 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.31751823425293 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.313019752502441 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.361662864685059 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.334105491638184 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.37648868560791 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.287008285522461 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.400372505187988 | Batch Time=2.34375 +GPU:3 | Epoch: 2 | loss=14.387930870056152 | Batch Time=0.78125 +GPU:3 | Epoch: 2 | loss=14.279863357543945 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.297285079956055 | Batch Time=3.515625 +GPU:3 | Epoch: 2 | loss=14.307066917419434 | Batch Time=3.515625 +GPU:3 | Epoch: 2 | loss=14.332489967346191 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.264968872070312 | Batch Time=2.734375 +GPU:3 | Epoch: 2 | loss=14.331459999084473 | Batch Time=1.5625 +GPU:3 | Epoch: 2 | loss=14.306638717651367 | Batch Time=1.171875 +GPU:3 | Epoch: 2 | loss=14.371753692626953 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.294963836669922 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.255470275878906 | Batch Time=3.125 +GPU:3 | Epoch: 2 | loss=14.229146957397461 | Batch Time=1.953125 +GPU:3 | Epoch: 2 | loss=14.270207405090332 | Batch Time=2.734375 +GPU:3 | Epoch: 2 | loss=14.308446884155273 | Batch Time=2.734375 +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159117312, percent=16.1, used=33653084160, free=57459687424, active=186574761984, inactive=9235767296, buffers=443490304, cached=166011822080, shared=5505818624, slab=3188801536) +AFTER TRAIN LOOP: Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +STARTING TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +BEFORE TRAIN LOOP: Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159117312, percent=16.1, used=33653084160, free=57459687424, active=186574761984, inactive=9235767296, buffers=443490304, cached=166011822080, shared=5505818624, slab=3188801536) +AFTER TRAIN LOOP: Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +STARTING TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +BEFORE TRAIN LOOP: Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159125504, percent=16.1, used=33653075968, free=57459695616, active=186574761984, inactive=9235767296, buffers=443490304, cached=166011822080, shared=5505818624, slab=3188801536) +AFTER TRAIN LOOP: Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +BEFORE VAL LOOP: GPU: 0 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159125504, percent=16.1, used=33653075968, free=57459695616, active=186574761984, inactive=9235767296, buffers=443490304, cached=166011822080, shared=5505818624, slab=3188801536) +AFTER TRAIN LOOP: Epoch 2 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +STARTING TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +BEFORE TRAIN LOOP: Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=216159408128, percent=16.1, used=33652867072, free=57459978240, active=186574442496, inactive=9235734528, buffers=443490304, cached=166011748352, shared=5505736704, slab=3188776960) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 2 | loss=4.802340030670166 | Batch Time=12.890625 +GPU:0 | Epoch: 2 | loss=6.330104827880859 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=5.9845476150512695 | Batch Time=3.90625 +GPU:0 | Epoch: 2 | loss=6.8840155601501465 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.74654483795166 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=6.421809196472168 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=6.178232192993164 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=6.385862827301025 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=6.640466213226318 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.80457878112793 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.8190178871154785 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.610541820526123 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.707800388336182 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=6.494615077972412 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=6.086212158203125 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.563351631164551 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.384646892547607 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=6.0130414962768555 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=5.93527889251709 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=5.262838840484619 | Batch Time=7.8125 +Model top1 Accuracy: 2.062 +Acc before rounding: 2.062 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 2 | loss=4.802340030670166 | Batch Time=12.890625 +GPU:0 | Epoch: 2 | loss=6.330104827880859 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=5.9845476150512695 | Batch Time=3.90625 +GPU:0 | Epoch: 2 | loss=6.8840155601501465 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.74654483795166 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=6.421809196472168 | Batch Time=1.5625 +GPU:0 | Epoch: 2 | loss=6.178232192993164 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=6.385862827301025 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=6.640466213226318 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.80457878112793 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.8190178871154785 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.610541820526123 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.707800388336182 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=6.494615077972412 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=6.086212158203125 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.563351631164551 | Batch Time=0.0 +GPU:0 | Epoch: 2 | loss=6.384646892547607 | Batch Time=0.390625 +GPU:0 | Epoch: 2 | loss=6.0130414962768555 | Batch Time=1.171875 +GPU:0 | Epoch: 2 | loss=5.93527889251709 | Batch Time=0.78125 +GPU:0 | Epoch: 2 | loss=5.262838840484619 | Batch Time=7.8125 +Model top1 Accuracy: 2.062 +Acc after rounding: 2.062 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 2 | loss=6.38822603225708 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=6.429160118103027 | Batch Time=2.734375 +GPU:0 | Epoch: 2 | loss=6.17044734954834 | Batch Time=1.953125 +GPU:0 | Epoch: 2 | loss=6.377553939819336 | Batch Time=3.515625 +Model top1 Accuracy: 2.14 +Validation Acc after rounding: 2.14 +AFTER VAL LOOP: GPU: 0 | Epoch 2 | Memory Usage: svmem(total=257568083968, available=197877305344, percent=23.2, used=51473240064, free=39173054464, active=204297539584, inactive=9685458944, buffers=443555840, cached=166478233600, shared=5967478784, slab=3195842560) +Rounding model with scheme: naive +Model avg sparsity: 48.47499007654433 +GPU:0 | Epoch: 2 | Acc=2.062 | Epoch Time=15.633248841762542 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=197877305344, percent=23.2, used=51473240064, free=39173054464, active=204297539584, inactive=9685458944, buffers=443555840, cached=166478233600, shared=5967478784, slab=3195842560) +STARTING TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=197877305344, percent=23.2, used=51473240064, free=39173054464, active=204297539584, inactive=9685458944, buffers=443555840, cached=166478233600, shared=5967478784, slab=3195842560) +BEFORE TRAIN LOOP: Epoch 3 | Memory Usage: svmem(total=257568083968, available=197877305344, percent=23.2, used=51473240064, free=39173054464, active=204297539584, inactive=9685458944, buffers=443555840, cached=166478233600, shared=5967478784, slab=3195842560) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=197877305344, percent=23.2, used=51473240064, free=39173054464, active=204297539584, inactive=9685458944, buffers=443555840, cached=166478233600, shared=5967478784, slab=3195842560) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:3 | Epoch: 3 | loss=14.347550392150879 | Batch Time=0.78125 +GPU:3 | Epoch: 3 | loss=14.235444068908691 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.240433692932129 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.245765686035156 | Batch Time=4.296875 +GPU:3 | Epoch: 3 | loss=14.198010444641113 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.339103698730469 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.120718002319336 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.255309104919434 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.243712425231934 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.213302612304688 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=14.187005996704102 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.32297134399414 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.253973007202148 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.198020935058594 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.116206169128418 | Batch Time=3.515625 +GPU:3 | Epoch: 3 | loss=14.141897201538086 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.266260147094727 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=14.205609321594238 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.210052490234375 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=14.126005172729492 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.270978927612305 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.27762222290039 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.224386215209961 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.187736511230469 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.203896522521973 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.266220092773438 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.136439323425293 | Batch Time=3.515625 +GPU:3 | Epoch: 3 | loss=14.089600563049316 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.224991798400879 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.141353607177734 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.021322250366211 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.15432357788086 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.203347206115723 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.089515686035156 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.125922203063965 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.192544937133789 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.06283187866211 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=14.128637313842773 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.13436508178711 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=14.228182792663574 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.211200714111328 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.212703704833984 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=14.143024444580078 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=14.197742462158203 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.104825973510742 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=14.135650634765625 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=14.104511260986328 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=14.111331939697266 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.13923454284668 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.139738082885742 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.218856811523438 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.044465065002441 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.02618408203125 | Batch Time=4.296875 +GPU:3 | Epoch: 3 | loss=14.082809448242188 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=14.182100296020508 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=13.976232528686523 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.179155349731445 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.072784423828125 | Batch Time=3.90625 +GPU:3 | Epoch: 3 | loss=14.083154678344727 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.078483581542969 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=14.085596084594727 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=13.97293472290039 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.03408432006836 | Batch Time=3.90625 +GPU:3 | Epoch: 3 | loss=14.087257385253906 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=14.07250690460205 | Batch Time=0.78125 +GPU:3 | Epoch: 3 | loss=14.063787460327148 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=14.035684585571289 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=14.1231689453125 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.00621223449707 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=14.061015129089355 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=14.024141311645508 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.093814849853516 | Batch Time=3.515625 +GPU:3 | Epoch: 3 | loss=14.046455383300781 | Batch Time=3.515625 +GPU:3 | Epoch: 3 | loss=14.201011657714844 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=13.983133316040039 | Batch Time=3.515625 +GPU:3 | Epoch: 3 | loss=14.107845306396484 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.008590698242188 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=13.95022201538086 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=13.970460891723633 | Batch Time=4.296875 +GPU:3 | Epoch: 3 | loss=13.991188049316406 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=14.039215087890625 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=14.008277893066406 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=13.932798385620117 | Batch Time=4.296875 +GPU:3 | Epoch: 3 | loss=13.918251991271973 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=13.918219566345215 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=13.997426986694336 | Batch Time=3.515625 +GPU:3 | Epoch: 3 | loss=14.01103401184082 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=13.94361686706543 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=13.931703567504883 | Batch Time=3.90625 +GPU:3 | Epoch: 3 | loss=13.933650970458984 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=13.979515075683594 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=13.990962028503418 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=13.99374008178711 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=13.990779876708984 | Batch Time=3.90625 +GPU:3 | Epoch: 3 | loss=14.061248779296875 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=14.000131607055664 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=13.94154167175293 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=13.862600326538086 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=13.974294662475586 | Batch Time=4.296875 +GPU:3 | Epoch: 3 | loss=13.865217208862305 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=13.957239151000977 | Batch Time=1.5625 +GPU:3 | Epoch: 3 | loss=13.860976219177246 | Batch Time=4.296875 +GPU:3 | Epoch: 3 | loss=13.87786865234375 | Batch Time=4.6875 +GPU:3 | Epoch: 3 | loss=13.949014663696289 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=13.993672370910645 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=13.943014144897461 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=13.900392532348633 | Batch Time=3.90625 +GPU:3 | Epoch: 3 | loss=13.991819381713867 | Batch Time=1.171875 +GPU:3 | Epoch: 3 | loss=13.8278169631958 | Batch Time=3.90625 +GPU:3 | Epoch: 3 | loss=13.937047004699707 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=13.83509635925293 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=13.929433822631836 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=13.836240768432617 | Batch Time=4.6875 +GPU:3 | Epoch: 3 | loss=13.966588020324707 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=13.840560913085938 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=13.876137733459473 | Batch Time=2.734375 +GPU:3 | Epoch: 3 | loss=13.92904281616211 | Batch Time=3.90625 +GPU:3 | Epoch: 3 | loss=13.895118713378906 | Batch Time=5.46875 +GPU:3 | Epoch: 3 | loss=13.787619590759277 | Batch Time=3.515625 +GPU:3 | Epoch: 3 | loss=13.947088241577148 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=13.931215286254883 | Batch Time=2.34375 +GPU:3 | Epoch: 3 | loss=13.971063613891602 | Batch Time=1.953125 +GPU:3 | Epoch: 3 | loss=13.770084381103516 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=13.781180381774902 | Batch Time=3.125 +GPU:3 | Epoch: 3 | loss=13.99548625946045 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.271138191223145 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.252320289611816 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=14.224541664123535 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.31552505493164 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.258172035217285 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=14.212570190429688 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.22453498840332 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.364691734313965 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=14.25574779510498 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.237579345703125 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.179349899291992 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=14.258659362792969 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.295289993286133 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.228836059570312 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.086424827575684 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.136329650878906 | Batch Time=4.6875 +GPU:0 | Epoch: 3 | loss=14.25413703918457 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.201382637023926 | Batch Time=0.390625 +GPU:0 | Epoch: 3 | loss=14.218940734863281 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.191274642944336 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.23307991027832 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.179618835449219 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.25383186340332 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.33798599243164 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=14.241646766662598 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.230030059814453 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=14.0829496383667 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.101693153381348 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.229726791381836 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.240497589111328 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.073280334472656 | Batch Time=4.6875 +GPU:0 | Epoch: 3 | loss=14.3035888671875 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.148537635803223 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.113513946533203 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.15088939666748 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.17161750793457 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.202533721923828 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.162969589233398 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.116901397705078 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.17178726196289 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.090045928955078 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.31951904296875 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.26938247680664 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.160579681396484 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.097902297973633 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.085693359375 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.04062271118164 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.204708099365234 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.124135971069336 | Batch Time=0.390625 +GPU:0 | Epoch: 3 | loss=14.099029541015625 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.080558776855469 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.083664894104004 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=14.103572845458984 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=14.168563842773438 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=14.144735336303711 | Batch Time=5.078125 +GPU:0 | Epoch: 3 | loss=13.970315933227539 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.140268325805664 | Batch Time=3.90625 +GPU:0 | Epoch: 3 | loss=14.140829086303711 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.065694808959961 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.119476318359375 | Batch Time=4.6875 +GPU:0 | Epoch: 3 | loss=14.060785293579102 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.120296478271484 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.05215835571289 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=13.988121032714844 | Batch Time=4.6875 +GPU:0 | Epoch: 3 | loss=14.0281343460083 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=13.997678756713867 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.05029296875 | Batch Time=4.296875 +GPU:0 | Epoch: 3 | loss=14.036056518554688 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.063375473022461 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=13.947352409362793 | Batch Time=3.90625 +GPU:0 | Epoch: 3 | loss=14.044580459594727 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=14.071670532226562 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.041698455810547 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.126789093017578 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.105792999267578 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=14.06155776977539 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.08001708984375 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=14.15567398071289 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=14.046754837036133 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=13.966949462890625 | Batch Time=3.90625 +GPU:0 | Epoch: 3 | loss=14.163909912109375 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=13.93735122680664 | Batch Time=4.296875 +GPU:0 | Epoch: 3 | loss=13.989938735961914 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.046008110046387 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=13.982691764831543 | Batch Time=4.296875 +GPU:0 | Epoch: 3 | loss=14.117818832397461 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=14.039873123168945 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=13.982559204101562 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=14.017724990844727 | Batch Time=0.390625 +GPU:0 | Epoch: 3 | loss=13.981449127197266 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=13.924484252929688 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=13.859844207763672 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=13.922996520996094 | Batch Time=4.296875 +GPU:0 | Epoch: 3 | loss=14.066173553466797 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=13.931198120117188 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=13.866982460021973 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=13.986154556274414 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.06049919128418 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=13.988229751586914 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.002313613891602 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.023191452026367 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=14.006484031677246 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=13.967342376708984 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=13.870405197143555 | Batch Time=3.90625 +GPU:0 | Epoch: 3 | loss=13.989792823791504 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=13.954687118530273 | Batch Time=4.296875 +GPU:0 | Epoch: 3 | loss=13.870378494262695 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=13.900197982788086 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=13.962864875793457 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=14.006756782531738 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=14.02583122253418 | Batch Time=5.46875 +GPU:0 | Epoch: 3 | loss=14.006107330322266 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=13.960103988647461 | Batch Time=4.296875 +GPU:0 | Epoch: 3 | loss=13.89860725402832 | Batch Time=4.6875 +GPU:0 | Epoch: 3 | loss=13.87725830078125 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=13.960587501525879 | Batch Time=1.5625 +GPU:0 | Epoch: 3 | loss=13.991466522216797 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=13.92746353149414 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=13.903136253356934 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=13.879255294799805 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=14.007593154907227 | Batch Time=1.953125 +GPU:0 | Epoch: 3 | loss=13.855142593383789 | Batch Time=3.125 +GPU:0 | Epoch: 3 | loss=13.891551971435547 | Batch Time=2.734375 +GPU:0 | Epoch: 3 | loss=13.88131046295166 | Batch Time=3.515625 +GPU:0 | Epoch: 3 | loss=13.888550758361816 | Batch Time=4.296875 +GPU:1 | Epoch: 3 | loss=14.343733787536621 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=14.273482322692871 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.27901554107666 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.276848793029785 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=14.221375465393066 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=14.115535736083984 | Batch Time=5.078125 +GPU:1 | Epoch: 3 | loss=14.162817001342773 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.349761009216309 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.309393882751465 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.207199096679688 | Batch Time=4.296875 +GPU:1 | Epoch: 3 | loss=14.28318977355957 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.239055633544922 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.247457504272461 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.322738647460938 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.201899528503418 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=14.177804946899414 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=14.289484024047852 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.139935493469238 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.324226379394531 | Batch Time=1.171875 +GPU:1 | Epoch: 3 | loss=14.248998641967773 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=14.209371566772461 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=14.186820983886719 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.220760345458984 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.193439483642578 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.222321510314941 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=14.225467681884766 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.102984428405762 | Batch Time=4.296875 +GPU:1 | Epoch: 3 | loss=14.25175952911377 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.186628341674805 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.099430084228516 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.186059951782227 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.082012176513672 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.096089363098145 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.120689392089844 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.155314445495605 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.165952682495117 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.138618469238281 | Batch Time=0.78125 +GPU:1 | Epoch: 3 | loss=14.026872634887695 | Batch Time=4.296875 +GPU:1 | Epoch: 3 | loss=14.15523910522461 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=14.092474937438965 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.175647735595703 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.162984848022461 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.064517974853516 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.013547897338867 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=14.117422103881836 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.059913635253906 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=14.169452667236328 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=14.041034698486328 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.201879501342773 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.255781173706055 | Batch Time=1.171875 +GPU:1 | Epoch: 3 | loss=14.105567932128906 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=14.066765785217285 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.106060028076172 | Batch Time=6.25 +GPU:1 | Epoch: 3 | loss=14.092025756835938 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.062410354614258 | Batch Time=0.78125 +GPU:1 | Epoch: 3 | loss=14.168706893920898 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.206525802612305 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.127035140991211 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.137552261352539 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.100513458251953 | Batch Time=4.296875 +GPU:1 | Epoch: 3 | loss=14.121648788452148 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.084993362426758 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.111047744750977 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=13.97637939453125 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.146428108215332 | Batch Time=0.390625 +GPU:1 | Epoch: 3 | loss=14.119539260864258 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.101444244384766 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.103523254394531 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=14.081121444702148 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=13.940024375915527 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=14.042531967163086 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=14.034687042236328 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=13.963508605957031 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=13.961029052734375 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.036931991577148 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.948047637939453 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=14.015872955322266 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=14.092754364013672 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.055875778198242 | Batch Time=4.296875 +GPU:1 | Epoch: 3 | loss=14.004280090332031 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.031963348388672 | Batch Time=1.171875 +GPU:1 | Epoch: 3 | loss=13.945777893066406 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.927820205688477 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=13.929316520690918 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=13.927680015563965 | Batch Time=4.6875 +GPU:1 | Epoch: 3 | loss=13.847444534301758 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=14.168863296508789 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=13.994237899780273 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=13.822793960571289 | Batch Time=4.6875 +GPU:1 | Epoch: 3 | loss=13.762462615966797 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=13.88241958618164 | Batch Time=4.6875 +GPU:1 | Epoch: 3 | loss=14.057239532470703 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=13.986984252929688 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.026718139648438 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=13.970478057861328 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.886331558227539 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=14.136465072631836 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=13.954671859741211 | Batch Time=1.171875 +GPU:1 | Epoch: 3 | loss=14.136472702026367 | Batch Time=3.515625 +GPU:1 | Epoch: 3 | loss=13.845722198486328 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=14.083246231079102 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=13.979787826538086 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.984947204589844 | Batch Time=5.078125 +GPU:1 | Epoch: 3 | loss=13.934331893920898 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.958104133605957 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=14.055517196655273 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=14.05404281616211 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.886026382446289 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=13.861950874328613 | Batch Time=5.078125 +GPU:1 | Epoch: 3 | loss=13.929352760314941 | Batch Time=1.5625 +GPU:1 | Epoch: 3 | loss=13.931798934936523 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=14.007146835327148 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=13.837003707885742 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.99874210357666 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=13.875259399414062 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.87804889678955 | Batch Time=4.296875 +GPU:1 | Epoch: 3 | loss=13.893375396728516 | Batch Time=4.6875 +GPU:1 | Epoch: 3 | loss=13.796632766723633 | Batch Time=4.296875 +GPU:1 | Epoch: 3 | loss=13.831673622131348 | Batch Time=4.6875 +GPU:1 | Epoch: 3 | loss=13.872018814086914 | Batch Time=3.90625 +GPU:1 | Epoch: 3 | loss=13.826555252075195 | Batch Time=3.125 +GPU:1 | Epoch: 3 | loss=13.934976577758789 | Batch Time=1.953125 +GPU:1 | Epoch: 3 | loss=13.941902160644531 | Batch Time=2.34375 +GPU:1 | Epoch: 3 | loss=13.75007152557373 | Batch Time=2.734375 +GPU:1 | Epoch: 3 | loss=13.814749717712402 | Batch Time=5.46875 +GPU:2 | Epoch: 3 | loss=14.33461856842041 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.21157169342041 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.307827949523926 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.235774993896484 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.254883766174316 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=14.325376510620117 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.25031852722168 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.278105735778809 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.179360389709473 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.211799621582031 | Batch Time=3.90625 +GPU:2 | Epoch: 3 | loss=14.1854829788208 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=14.253402709960938 | Batch Time=1.171875 +GPU:2 | Epoch: 3 | loss=14.094816207885742 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.256484985351562 | Batch Time=1.171875 +GPU:2 | Epoch: 3 | loss=14.259081840515137 | Batch Time=0.78125 +GPU:2 | Epoch: 3 | loss=14.147089004516602 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=14.190301895141602 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=14.252629280090332 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=14.310653686523438 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.289751052856445 | Batch Time=0.78125 +GPU:2 | Epoch: 3 | loss=14.236530303955078 | Batch Time=1.171875 +GPU:2 | Epoch: 3 | loss=14.28823471069336 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.292802810668945 | Batch Time=1.171875 +GPU:2 | Epoch: 3 | loss=14.250320434570312 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.024788856506348 | Batch Time=4.6875 +GPU:2 | Epoch: 3 | loss=14.182624816894531 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.245810508728027 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.25594425201416 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.259397506713867 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.208362579345703 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.222169876098633 | Batch Time=1.171875 +GPU:2 | Epoch: 3 | loss=14.18661880493164 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.25407886505127 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.184879302978516 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.00004768371582 | Batch Time=4.296875 +GPU:2 | Epoch: 3 | loss=14.153032302856445 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.22235107421875 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=14.125555038452148 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.278316497802734 | Batch Time=0.390625 +GPU:2 | Epoch: 3 | loss=14.1868314743042 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.05987548828125 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=14.190044403076172 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.036800384521484 | Batch Time=4.6875 +GPU:2 | Epoch: 3 | loss=14.095550537109375 | Batch Time=3.90625 +GPU:2 | Epoch: 3 | loss=14.131780624389648 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.181503295898438 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.122055053710938 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.089908599853516 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.162538528442383 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.139646530151367 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.107376098632812 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.144379615783691 | Batch Time=3.90625 +GPU:2 | Epoch: 3 | loss=14.152408599853516 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.098217010498047 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.082597732543945 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.08773422241211 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=13.961196899414062 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.03951644897461 | Batch Time=3.90625 +GPU:2 | Epoch: 3 | loss=14.060997009277344 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.167621612548828 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.024358749389648 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=13.986438751220703 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=14.151012420654297 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.099449157714844 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.07359790802002 | Batch Time=1.171875 +GPU:2 | Epoch: 3 | loss=14.079439163208008 | Batch Time=4.296875 +GPU:2 | Epoch: 3 | loss=14.168548583984375 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.143203735351562 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.135053634643555 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.074435234069824 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.112848281860352 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.029037475585938 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=13.973487854003906 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=14.086139678955078 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=14.056730270385742 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.083904266357422 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.042316436767578 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=13.890632629394531 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=14.021726608276367 | Batch Time=4.296875 +GPU:2 | Epoch: 3 | loss=14.021621704101562 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.011955261230469 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=13.867231369018555 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=14.079584121704102 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=14.087191581726074 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.051440238952637 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=13.988256454467773 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=13.999094009399414 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=13.892683029174805 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=13.887449264526367 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=14.056804656982422 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=13.986778259277344 | Batch Time=0.78125 +GPU:2 | Epoch: 3 | loss=14.088115692138672 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=13.892868041992188 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=13.931217193603516 | Batch Time=3.90625 +GPU:2 | Epoch: 3 | loss=14.018352508544922 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.033342361450195 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=13.947103500366211 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=13.961374282836914 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.031927108764648 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=14.050085067749023 | Batch Time=3.90625 +GPU:2 | Epoch: 3 | loss=13.888425827026367 | Batch Time=4.296875 +GPU:2 | Epoch: 3 | loss=13.870070457458496 | Batch Time=3.90625 +GPU:2 | Epoch: 3 | loss=13.953479766845703 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=13.940980911254883 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=14.007336616516113 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=13.913454055786133 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=13.86079216003418 | Batch Time=2.34375 +GPU:2 | Epoch: 3 | loss=13.964265823364258 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=14.032628059387207 | Batch Time=1.5625 +GPU:2 | Epoch: 3 | loss=13.84581470489502 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=13.88215446472168 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=13.927034378051758 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=13.820943832397461 | Batch Time=4.296875 +GPU:2 | Epoch: 3 | loss=13.93161392211914 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=13.910823822021484 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=13.902953147888184 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=13.936525344848633 | Batch Time=4.296875 +GPU:2 | Epoch: 3 | loss=13.892147064208984 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=13.848299980163574 | Batch Time=3.90625 +GPU:2 | Epoch: 3 | loss=13.983499526977539 | Batch Time=2.734375 +GPU:2 | Epoch: 3 | loss=13.871702194213867 | Batch Time=3.125 +GPU:2 | Epoch: 3 | loss=13.887590408325195 | Batch Time=3.515625 +GPU:2 | Epoch: 3 | loss=13.83504867553711 | Batch Time=4.296875 +GPU:2 | Epoch: 3 | loss=14.03644847869873 | Batch Time=1.953125 +GPU:2 | Epoch: 3 | loss=13.827692985534668 | Batch Time=2.34375 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=208459493376, percent=19.1, used=41198653440, free=49318428672, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +AFTER TRAIN LOOP: Epoch 3 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +STARTING TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +BEFORE TRAIN LOOP: Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=208459493376, percent=19.1, used=41198653440, free=49318428672, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +AFTER TRAIN LOOP: Epoch 3 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +BEFORE VAL LOOP: GPU: 0 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=208459493376, percent=19.1, used=41198653440, free=49318428672, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +AFTER TRAIN LOOP: Epoch 3 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +STARTING TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +BEFORE TRAIN LOOP: Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=208459493376, percent=19.1, used=41198653440, free=49318428672, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +AFTER TRAIN LOOP: Epoch 3 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +STARTING TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +BEFORE TRAIN LOOP: Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=208460009472, percent=19.1, used=41198137344, free=49318944768, active=196475002880, inactive=7391842304, buffers=443596800, cached=166607405056, shared=5659885568, slab=3199426560) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 3 | loss=4.790079593658447 | Batch Time=7.8125 +GPU:0 | Epoch: 3 | loss=5.950860977172852 | Batch Time=5.078125 +GPU:0 | Epoch: 3 | loss=5.525110721588135 | Batch Time=8.203125 +GPU:0 | Epoch: 3 | loss=6.395721912384033 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=6.389284133911133 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=6.156222820281982 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=6.175640106201172 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.076026439666748 | Batch Time=0.390625 +GPU:0 | Epoch: 3 | loss=6.379429340362549 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.634638786315918 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.6111884117126465 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.27398157119751 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=6.453568458557129 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=6.201510906219482 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=6.034187316894531 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.257366180419922 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.130952835083008 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=5.625624179840088 | Batch Time=5.859375 +GPU:0 | Epoch: 3 | loss=6.047201633453369 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=4.882997035980225 | Batch Time=13.28125 +Model top1 Accuracy: 3.166 +Acc before rounding: 3.166 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 3 | loss=4.790079593658447 | Batch Time=7.8125 +GPU:0 | Epoch: 3 | loss=5.950860977172852 | Batch Time=5.078125 +GPU:0 | Epoch: 3 | loss=5.525110721588135 | Batch Time=8.203125 +GPU:0 | Epoch: 3 | loss=6.395721912384033 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=6.389284133911133 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=6.156222820281982 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=6.175640106201172 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.076026439666748 | Batch Time=0.390625 +GPU:0 | Epoch: 3 | loss=6.379429340362549 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.634638786315918 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.6111884117126465 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.27398157119751 | Batch Time=0.78125 +GPU:0 | Epoch: 3 | loss=6.453568458557129 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=6.201510906219482 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=6.034187316894531 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.257366180419922 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=6.130952835083008 | Batch Time=1.171875 +GPU:0 | Epoch: 3 | loss=5.625624179840088 | Batch Time=5.859375 +GPU:0 | Epoch: 3 | loss=6.047201633453369 | Batch Time=0.0 +GPU:0 | Epoch: 3 | loss=4.882997035980225 | Batch Time=13.28125 +Model top1 Accuracy: 3.166 +Acc after rounding: 3.166 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 3 | loss=6.072900295257568 | Batch Time=2.34375 +GPU:0 | Epoch: 3 | loss=6.0949859619140625 | Batch Time=4.6875 +GPU:0 | Epoch: 3 | loss=5.978786945343018 | Batch Time=4.296875 +GPU:0 | Epoch: 3 | loss=6.148578643798828 | Batch Time=3.515625 +Model top1 Accuracy: 2.89 +Validation Acc after rounding: 2.89 +AFTER VAL LOOP: GPU: 0 | Epoch 3 | Memory Usage: svmem(total=257568083968, available=190660890624, percent=26.0, used=58996928512, free=31518580736, active=214167519232, inactive=7385231360, buffers=443654144, cached=166608920576, shared=5660213248, slab=3206225920) +Rounding model with scheme: naive +Model avg sparsity: 47.959665375444445 +GPU:0 | Epoch: 3 | Acc=3.166 | Epoch Time=15.601132929325104 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=190661148672, percent=26.0, used=58996752384, free=31518838784, active=214167453696, inactive=7385198592, buffers=443654144, cached=166608838656, shared=5660131328, slab=3206225920) +STARTING TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=190661148672, percent=26.0, used=58996752384, free=31518838784, active=214167453696, inactive=7385198592, buffers=443654144, cached=166608838656, shared=5660131328, slab=3206225920) +BEFORE TRAIN LOOP: Epoch 4 | Memory Usage: svmem(total=257568083968, available=190661148672, percent=26.0, used=58996752384, free=31518838784, active=214167453696, inactive=7385198592, buffers=443654144, cached=166608838656, shared=5660131328, slab=3206225920) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=190661148672, percent=26.0, used=58996752384, free=31518838784, active=214167453696, inactive=7385198592, buffers=443654144, cached=166608838656, shared=5660131328, slab=3206225920) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 4 | loss=13.90697193145752 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.908058166503906 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.790559768676758 | Batch Time=5.46875 +GPU:1 | Epoch: 4 | loss=13.860480308532715 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.844315528869629 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.800969123840332 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.819793701171875 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.770742416381836 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.945213317871094 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.851531982421875 | Batch Time=1.953125 +GPU:1 | Epoch: 4 | loss=13.754876136779785 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.707952499389648 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.809647560119629 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.802077293395996 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.991453170776367 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.805707931518555 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.919696807861328 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.80220890045166 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.879364013671875 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.836978912353516 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.888463973999023 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.859134674072266 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.861624717712402 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.9362154006958 | Batch Time=1.953125 +GPU:1 | Epoch: 4 | loss=13.84445571899414 | Batch Time=5.46875 +GPU:1 | Epoch: 4 | loss=13.77203369140625 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.830297470092773 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.741174697875977 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.788095474243164 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.792259216308594 | Batch Time=5.078125 +GPU:1 | Epoch: 4 | loss=13.730990409851074 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.841309547424316 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.682976722717285 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.807353973388672 | Batch Time=1.5625 +GPU:1 | Epoch: 4 | loss=13.695180892944336 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.658578872680664 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.810863494873047 | Batch Time=4.6875 +GPU:1 | Epoch: 4 | loss=13.891550064086914 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.805961608886719 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.97218132019043 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.667365074157715 | Batch Time=5.46875 +GPU:1 | Epoch: 4 | loss=13.84245491027832 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.852181434631348 | Batch Time=1.5625 +GPU:1 | Epoch: 4 | loss=13.853116989135742 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.732080459594727 | Batch Time=1.5625 +GPU:1 | Epoch: 4 | loss=13.788694381713867 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.76193618774414 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.759696960449219 | Batch Time=5.078125 +GPU:1 | Epoch: 4 | loss=13.776473999023438 | Batch Time=1.5625 +GPU:1 | Epoch: 4 | loss=13.685771942138672 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.794879913330078 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.7792329788208 | Batch Time=5.859375 +GPU:1 | Epoch: 4 | loss=13.72359848022461 | Batch Time=5.46875 +GPU:1 | Epoch: 4 | loss=13.712028503417969 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.780481338500977 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.696949005126953 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.764167785644531 | Batch Time=5.46875 +GPU:1 | Epoch: 4 | loss=13.843581199645996 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.823309898376465 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.6795654296875 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.763450622558594 | Batch Time=5.46875 +GPU:1 | Epoch: 4 | loss=13.770156860351562 | Batch Time=1.953125 +GPU:1 | Epoch: 4 | loss=13.802717208862305 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.773246765136719 | Batch Time=1.171875 +GPU:1 | Epoch: 4 | loss=13.826096534729004 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.790693283081055 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.742115020751953 | Batch Time=5.46875 +GPU:1 | Epoch: 4 | loss=13.751092910766602 | Batch Time=1.171875 +GPU:1 | Epoch: 4 | loss=13.69062328338623 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.689998626708984 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.694414138793945 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.819189071655273 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.657466888427734 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.702653884887695 | Batch Time=5.078125 +GPU:1 | Epoch: 4 | loss=13.731459617614746 | Batch Time=4.6875 +GPU:1 | Epoch: 4 | loss=13.738758087158203 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.658060073852539 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.886554718017578 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.7108154296875 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.698824882507324 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.580404281616211 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.597675323486328 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.716646194458008 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.675515174865723 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.501440048217773 | Batch Time=4.6875 +GPU:1 | Epoch: 4 | loss=13.628555297851562 | Batch Time=6.25 +GPU:1 | Epoch: 4 | loss=13.551015853881836 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.620320320129395 | Batch Time=5.46875 +GPU:1 | Epoch: 4 | loss=13.754806518554688 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.738761901855469 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.606931686401367 | Batch Time=5.859375 +GPU:1 | Epoch: 4 | loss=13.6361083984375 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.607658386230469 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.784066200256348 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.577499389648438 | Batch Time=4.6875 +GPU:1 | Epoch: 4 | loss=13.598379135131836 | Batch Time=1.171875 +GPU:1 | Epoch: 4 | loss=13.581354141235352 | Batch Time=5.078125 +GPU:1 | Epoch: 4 | loss=13.588993072509766 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.65390396118164 | Batch Time=2.734375 +GPU:1 | Epoch: 4 | loss=13.684672355651855 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.614025115966797 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.621940612792969 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.573871612548828 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.562459945678711 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.728092193603516 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.693204879760742 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.572596549987793 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.572463035583496 | Batch Time=6.25 +GPU:1 | Epoch: 4 | loss=13.527463912963867 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.45816707611084 | Batch Time=5.859375 +GPU:1 | Epoch: 4 | loss=13.534473419189453 | Batch Time=3.125 +GPU:1 | Epoch: 4 | loss=13.61555290222168 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.613626480102539 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.52982234954834 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.670089721679688 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.568280220031738 | Batch Time=1.171875 +GPU:1 | Epoch: 4 | loss=13.44667911529541 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.539392471313477 | Batch Time=5.078125 +GPU:1 | Epoch: 4 | loss=13.650419235229492 | Batch Time=3.515625 +GPU:1 | Epoch: 4 | loss=13.642560005187988 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.448877334594727 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.671197891235352 | Batch Time=3.90625 +GPU:1 | Epoch: 4 | loss=13.568748474121094 | Batch Time=4.296875 +GPU:1 | Epoch: 4 | loss=13.515649795532227 | Batch Time=2.34375 +GPU:1 | Epoch: 4 | loss=13.456255912780762 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.799015998840332 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.853595733642578 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.8982572555542 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.751921653747559 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.810670852661133 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.815569877624512 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.875009536743164 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.863824844360352 | Batch Time=5.078125 +GPU:2 | Epoch: 4 | loss=13.944801330566406 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.755634307861328 | Batch Time=5.46875 +GPU:2 | Epoch: 4 | loss=13.898957252502441 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.829275131225586 | Batch Time=1.5625 +GPU:2 | Epoch: 4 | loss=13.834307670593262 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.749823570251465 | Batch Time=0.78125 +GPU:2 | Epoch: 4 | loss=13.876577377319336 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.789972305297852 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.971183776855469 | Batch Time=1.5625 +GPU:2 | Epoch: 4 | loss=13.886467933654785 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.912593841552734 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.73930835723877 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.81004524230957 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.724849700927734 | Batch Time=3.90625 +GPU:2 | Epoch: 4 | loss=13.961943626403809 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.830998420715332 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.904741287231445 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.856432914733887 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.772394180297852 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.803451538085938 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.88521671295166 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.825193405151367 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.744288444519043 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.76378345489502 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.817845344543457 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.755237579345703 | Batch Time=5.46875 +GPU:2 | Epoch: 4 | loss=13.729703903198242 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.732980728149414 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.679195404052734 | Batch Time=5.078125 +GPU:2 | Epoch: 4 | loss=13.734748840332031 | Batch Time=4.6875 +GPU:2 | Epoch: 4 | loss=13.806549072265625 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.80549430847168 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.681599617004395 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.776453018188477 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.693364143371582 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.844930648803711 | Batch Time=1.171875 +GPU:2 | Epoch: 4 | loss=13.74400520324707 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.75092887878418 | Batch Time=5.46875 +GPU:2 | Epoch: 4 | loss=13.662158966064453 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.842876434326172 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.854961395263672 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.76620101928711 | Batch Time=1.5625 +GPU:2 | Epoch: 4 | loss=13.864994049072266 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.685574531555176 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.739912986755371 | Batch Time=5.859375 +GPU:2 | Epoch: 4 | loss=13.814956665039062 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.691217422485352 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.664514541625977 | Batch Time=6.640625 +GPU:2 | Epoch: 4 | loss=13.750503540039062 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.622417449951172 | Batch Time=5.078125 +GPU:2 | Epoch: 4 | loss=13.587298393249512 | Batch Time=4.6875 +GPU:2 | Epoch: 4 | loss=13.781822204589844 | Batch Time=3.90625 +GPU:2 | Epoch: 4 | loss=13.70169448852539 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.75094985961914 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.67201042175293 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.613231658935547 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.676709175109863 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.68223762512207 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.874639511108398 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.681574821472168 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.67325496673584 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.791534423828125 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.723833084106445 | Batch Time=4.6875 +GPU:2 | Epoch: 4 | loss=13.64341926574707 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.660911560058594 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.632650375366211 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.727057456970215 | Batch Time=0.78125 +GPU:2 | Epoch: 4 | loss=13.602956771850586 | Batch Time=5.078125 +GPU:2 | Epoch: 4 | loss=13.733701705932617 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.638336181640625 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.666778564453125 | Batch Time=5.078125 +GPU:2 | Epoch: 4 | loss=13.735526084899902 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.647764205932617 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.83719253540039 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.503950119018555 | Batch Time=4.6875 +GPU:2 | Epoch: 4 | loss=13.622995376586914 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.85407829284668 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.625938415527344 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.707769393920898 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.591416358947754 | Batch Time=5.078125 +GPU:2 | Epoch: 4 | loss=13.705320358276367 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.742633819580078 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.563611030578613 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.577824592590332 | Batch Time=5.859375 +GPU:2 | Epoch: 4 | loss=13.70050048828125 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.59060001373291 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.667652130126953 | Batch Time=4.6875 +GPU:2 | Epoch: 4 | loss=13.559228897094727 | Batch Time=3.90625 +GPU:2 | Epoch: 4 | loss=13.451395034790039 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.633525848388672 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.58730697631836 | Batch Time=5.078125 +GPU:2 | Epoch: 4 | loss=13.75842571258545 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.627716064453125 | Batch Time=3.90625 +GPU:2 | Epoch: 4 | loss=13.72359848022461 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.648456573486328 | Batch Time=3.90625 +GPU:2 | Epoch: 4 | loss=13.580459594726562 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.751270294189453 | Batch Time=5.078125 +GPU:2 | Epoch: 4 | loss=13.671854019165039 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.629656791687012 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.600303649902344 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.458944320678711 | Batch Time=5.859375 +GPU:2 | Epoch: 4 | loss=13.5258207321167 | Batch Time=4.296875 +GPU:2 | Epoch: 4 | loss=13.65614128112793 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.711771011352539 | Batch Time=3.125 +GPU:2 | Epoch: 4 | loss=13.581872940063477 | Batch Time=5.859375 +GPU:2 | Epoch: 4 | loss=13.471102714538574 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.524669647216797 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.687310218811035 | Batch Time=3.90625 +GPU:2 | Epoch: 4 | loss=13.647477149963379 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.631784439086914 | Batch Time=2.34375 +GPU:2 | Epoch: 4 | loss=13.535440444946289 | Batch Time=5.859375 +GPU:2 | Epoch: 4 | loss=13.731614112854004 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.480627059936523 | Batch Time=1.953125 +GPU:2 | Epoch: 4 | loss=13.506290435791016 | Batch Time=2.734375 +GPU:2 | Epoch: 4 | loss=13.472885131835938 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.512899398803711 | Batch Time=3.515625 +GPU:2 | Epoch: 4 | loss=13.5387601852417 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.922383308410645 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.894172668457031 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.869345664978027 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.830153465270996 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.902822494506836 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.860941886901855 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.746532440185547 | Batch Time=5.078125 +GPU:3 | Epoch: 4 | loss=13.959135055541992 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.810184478759766 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.855751037597656 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.929886817932129 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.961828231811523 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.809691429138184 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.850703239440918 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.801904678344727 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.832517623901367 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.879539489746094 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.69680118560791 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.798347473144531 | Batch Time=1.171875 +GPU:3 | Epoch: 4 | loss=13.868267059326172 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.947614669799805 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.900348663330078 | Batch Time=2.34375 +GPU:3 | Epoch: 4 | loss=13.838519096374512 | Batch Time=5.078125 +GPU:3 | Epoch: 4 | loss=13.753138542175293 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.713823318481445 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.754656791687012 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.892118453979492 | Batch Time=1.5625 +GPU:3 | Epoch: 4 | loss=13.735591888427734 | Batch Time=5.46875 +GPU:3 | Epoch: 4 | loss=13.825398445129395 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.80859375 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.786843299865723 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.806992530822754 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.697564125061035 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.73232650756836 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.759206771850586 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.710412979125977 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.722908020019531 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.662363052368164 | Batch Time=6.25 +GPU:3 | Epoch: 4 | loss=13.734336853027344 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.74567985534668 | Batch Time=1.5625 +GPU:3 | Epoch: 4 | loss=13.710225105285645 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.857316970825195 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.885769844055176 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.819467544555664 | Batch Time=2.34375 +GPU:3 | Epoch: 4 | loss=13.75758171081543 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.797005653381348 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.80301284790039 | Batch Time=2.34375 +GPU:3 | Epoch: 4 | loss=13.743480682373047 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.735879898071289 | Batch Time=1.171875 +GPU:3 | Epoch: 4 | loss=13.77059555053711 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.566871643066406 | Batch Time=1.5625 +GPU:3 | Epoch: 4 | loss=13.636208534240723 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.65518856048584 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.73592758178711 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.771120071411133 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.722709655761719 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.64706039428711 | Batch Time=2.34375 +GPU:3 | Epoch: 4 | loss=13.714556694030762 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.756743431091309 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.658599853515625 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.84567642211914 | Batch Time=2.34375 +GPU:3 | Epoch: 4 | loss=13.743228912353516 | Batch Time=2.34375 +GPU:3 | Epoch: 4 | loss=13.772497177124023 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.763383865356445 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.723942756652832 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.744028091430664 | Batch Time=1.5625 +GPU:3 | Epoch: 4 | loss=13.730978012084961 | Batch Time=1.171875 +GPU:3 | Epoch: 4 | loss=13.775545120239258 | Batch Time=5.859375 +GPU:3 | Epoch: 4 | loss=13.818526268005371 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.684989929199219 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.706422805786133 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.809789657592773 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.667871475219727 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.635557174682617 | Batch Time=2.34375 +GPU:3 | Epoch: 4 | loss=13.594538688659668 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.694698333740234 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.784502029418945 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.647838592529297 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.802505493164062 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.769179344177246 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.656644821166992 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.782085418701172 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.750730514526367 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.61562442779541 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.552302360534668 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.605560302734375 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.657747268676758 | Batch Time=1.5625 +GPU:3 | Epoch: 4 | loss=13.644631385803223 | Batch Time=5.078125 +GPU:3 | Epoch: 4 | loss=13.662849426269531 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.575080871582031 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.583878517150879 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.578792572021484 | Batch Time=6.25 +GPU:3 | Epoch: 4 | loss=13.602071762084961 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.700623512268066 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.714305877685547 | Batch Time=2.34375 +GPU:3 | Epoch: 4 | loss=13.59526252746582 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.710210800170898 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.637271881103516 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.66628646850586 | Batch Time=3.90625 +GPU:3 | Epoch: 4 | loss=13.554522514343262 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.645902633666992 | Batch Time=5.46875 +GPU:3 | Epoch: 4 | loss=13.658782958984375 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.621330261230469 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.578243255615234 | Batch Time=3.515625 +GPU:3 | Epoch: 4 | loss=13.654876708984375 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.625566482543945 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.568221092224121 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.477752685546875 | Batch Time=5.46875 +GPU:3 | Epoch: 4 | loss=13.525613784790039 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.56238842010498 | Batch Time=5.46875 +GPU:3 | Epoch: 4 | loss=13.513193130493164 | Batch Time=5.859375 +GPU:3 | Epoch: 4 | loss=13.672975540161133 | Batch Time=1.5625 +GPU:3 | Epoch: 4 | loss=13.588762283325195 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.644213676452637 | Batch Time=1.953125 +GPU:3 | Epoch: 4 | loss=13.606132507324219 | Batch Time=3.125 +GPU:3 | Epoch: 4 | loss=13.554207801818848 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.514824867248535 | Batch Time=4.6875 +GPU:3 | Epoch: 4 | loss=13.432241439819336 | Batch Time=6.640625 +GPU:3 | Epoch: 4 | loss=13.55021858215332 | Batch Time=2.734375 +GPU:3 | Epoch: 4 | loss=13.588822364807129 | Batch Time=5.078125 +GPU:3 | Epoch: 4 | loss=13.601997375488281 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.70328140258789 | Batch Time=4.296875 +GPU:3 | Epoch: 4 | loss=13.439682006835938 | Batch Time=5.46875 +GPU:3 | Epoch: 4 | loss=13.439493179321289 | Batch Time=5.46875 +GPU:3 | Epoch: 4 | loss=13.717658042907715 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=13.842133522033691 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.895484924316406 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.931599617004395 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.84995174407959 | Batch Time=7.03125 +GPU:0 | Epoch: 4 | loss=13.96507453918457 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=13.769156455993652 | Batch Time=5.46875 +GPU:0 | Epoch: 4 | loss=13.815513610839844 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.89381217956543 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.768491744995117 | Batch Time=1.5625 +GPU:0 | Epoch: 4 | loss=13.827850341796875 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=13.914517402648926 | Batch Time=1.953125 +GPU:0 | Epoch: 4 | loss=13.79014778137207 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.733302116394043 | Batch Time=5.46875 +GPU:0 | Epoch: 4 | loss=13.935568809509277 | Batch Time=4.6875 +GPU:0 | Epoch: 4 | loss=13.860433578491211 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.747720718383789 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.86767578125 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.811558723449707 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.873470306396484 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.658245086669922 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.892950057983398 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.821207046508789 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.887118339538574 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.873478889465332 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.852775573730469 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=13.871552467346191 | Batch Time=4.6875 +GPU:0 | Epoch: 4 | loss=13.718412399291992 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.896017074584961 | Batch Time=1.953125 +GPU:0 | Epoch: 4 | loss=13.816251754760742 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.82513427734375 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.882418632507324 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.841752052307129 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.747086524963379 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.839881896972656 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.773828506469727 | Batch Time=5.46875 +GPU:0 | Epoch: 4 | loss=13.694908142089844 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.675254821777344 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.75822639465332 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.73769760131836 | Batch Time=4.6875 +GPU:0 | Epoch: 4 | loss=13.647356986999512 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.728045463562012 | Batch Time=6.25 +GPU:0 | Epoch: 4 | loss=13.726423263549805 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.842747688293457 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.699335098266602 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.860185623168945 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.757104873657227 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.750045776367188 | Batch Time=1.953125 +GPU:0 | Epoch: 4 | loss=13.874408721923828 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.75859260559082 | Batch Time=1.953125 +GPU:0 | Epoch: 4 | loss=13.791248321533203 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.791391372680664 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.734429359436035 | Batch Time=5.078125 +GPU:0 | Epoch: 4 | loss=13.654276847839355 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.614158630371094 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.737520217895508 | Batch Time=5.078125 +GPU:0 | Epoch: 4 | loss=13.686328887939453 | Batch Time=1.953125 +GPU:0 | Epoch: 4 | loss=13.743412017822266 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.57193660736084 | Batch Time=5.859375 +GPU:0 | Epoch: 4 | loss=13.669108390808105 | Batch Time=4.6875 +GPU:0 | Epoch: 4 | loss=13.731590270996094 | Batch Time=1.953125 +GPU:0 | Epoch: 4 | loss=13.614055633544922 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.785396575927734 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=13.728933334350586 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.774093627929688 | Batch Time=1.5625 +GPU:0 | Epoch: 4 | loss=13.8052339553833 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.634825706481934 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.864072799682617 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.719106674194336 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.524886131286621 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.776470184326172 | Batch Time=1.5625 +GPU:0 | Epoch: 4 | loss=13.845975875854492 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.533353805541992 | Batch Time=5.46875 +GPU:0 | Epoch: 4 | loss=13.720508575439453 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.563261032104492 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.708369255065918 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.803234100341797 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.641866683959961 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.868236541748047 | Batch Time=1.953125 +GPU:0 | Epoch: 4 | loss=13.639232635498047 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.609755516052246 | Batch Time=4.6875 +GPU:0 | Epoch: 4 | loss=13.663690567016602 | Batch Time=5.46875 +GPU:0 | Epoch: 4 | loss=13.725399017333984 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.56916618347168 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.753889083862305 | Batch Time=5.078125 +GPU:0 | Epoch: 4 | loss=13.711626052856445 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.542834281921387 | Batch Time=4.6875 +GPU:0 | Epoch: 4 | loss=13.705137252807617 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.54816722869873 | Batch Time=5.46875 +GPU:0 | Epoch: 4 | loss=13.720876693725586 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.679359436035156 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.633400917053223 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.584424018859863 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.765434265136719 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.701722145080566 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.522250175476074 | Batch Time=4.6875 +GPU:0 | Epoch: 4 | loss=13.672301292419434 | Batch Time=5.46875 +GPU:0 | Epoch: 4 | loss=13.673868179321289 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.70676040649414 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.597038269042969 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.69831371307373 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=13.605194091796875 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.748844146728516 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.690444946289062 | Batch Time=5.078125 +GPU:0 | Epoch: 4 | loss=13.598438262939453 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.654396057128906 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.619148254394531 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.677336692810059 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.563488006591797 | Batch Time=4.6875 +GPU:0 | Epoch: 4 | loss=13.618120193481445 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.399558067321777 | Batch Time=5.859375 +GPU:0 | Epoch: 4 | loss=13.598237991333008 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.721624374389648 | Batch Time=1.5625 +GPU:0 | Epoch: 4 | loss=13.627725601196289 | Batch Time=5.078125 +GPU:0 | Epoch: 4 | loss=13.664904594421387 | Batch Time=5.078125 +GPU:0 | Epoch: 4 | loss=13.589556694030762 | Batch Time=5.46875 +GPU:0 | Epoch: 4 | loss=13.45900821685791 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.471861839294434 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.62009048461914 | Batch Time=2.34375 +GPU:0 | Epoch: 4 | loss=13.594339370727539 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=13.578408241271973 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=13.40452766418457 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.565013885498047 | Batch Time=3.125 +GPU:0 | Epoch: 4 | loss=13.493953704833984 | Batch Time=1.953125 +GPU:0 | Epoch: 4 | loss=13.447526931762695 | Batch Time=4.296875 +GPU:0 | Epoch: 4 | loss=13.487116813659668 | Batch Time=3.90625 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +AFTER TRAIN LOOP: Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +STARTING TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +BEFORE TRAIN LOOP: Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +AFTER TRAIN LOOP: Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +STARTING TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +BEFORE TRAIN LOOP: Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +AFTER TRAIN LOOP: Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +STARTING TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +BEFORE TRAIN LOOP: Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +AFTER TRAIN LOOP: Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +BEFORE VAL LOOP: GPU: 0 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=200914784256, percent=22.0, used=48743067648, free=41632817152, active=204876042240, inactive=6597844992, buffers=443699200, cached=166748499968, shared=5660180480, slab=3207544832) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 4 | loss=4.579524040222168 | Batch Time=8.203125 +GPU:0 | Epoch: 4 | loss=5.764533042907715 | Batch Time=1.5625 +GPU:0 | Epoch: 4 | loss=5.569868564605713 | Batch Time=6.25 +GPU:0 | Epoch: 4 | loss=6.203168869018555 | Batch Time=1.5625 +GPU:0 | Epoch: 4 | loss=6.1963210105896 | Batch Time=0.0 +GPU:0 | Epoch: 4 | loss=5.686137676239014 | Batch Time=6.640625 +GPU:0 | Epoch: 4 | loss=5.706753730773926 | Batch Time=0.390625 +GPU:0 | Epoch: 4 | loss=6.089773654937744 | Batch Time=0.390625 +GPU:0 | Epoch: 4 | loss=6.310647487640381 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=6.684844493865967 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=6.517068862915039 | Batch Time=0.78125 +GPU:0 | Epoch: 4 | loss=6.0237812995910645 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=6.394435405731201 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=5.994657039642334 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=5.940245628356934 | Batch Time=0.390625 +GPU:0 | Epoch: 4 | loss=6.057304382324219 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=6.212105751037598 | Batch Time=0.390625 +GPU:0 | Epoch: 4 | loss=5.511551380157471 | Batch Time=5.859375 +GPU:0 | Epoch: 4 | loss=5.155518531799316 | Batch Time=7.8125 +GPU:0 | Epoch: 4 | loss=4.581152439117432 | Batch Time=17.1875 +Model top1 Accuracy: 3.952 +Acc before rounding: 3.952 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 4 | loss=4.579524040222168 | Batch Time=8.203125 +GPU:0 | Epoch: 4 | loss=5.764533042907715 | Batch Time=1.5625 +GPU:0 | Epoch: 4 | loss=5.569868564605713 | Batch Time=6.25 +GPU:0 | Epoch: 4 | loss=6.203168869018555 | Batch Time=1.5625 +GPU:0 | Epoch: 4 | loss=6.1963210105896 | Batch Time=0.0 +GPU:0 | Epoch: 4 | loss=5.686137676239014 | Batch Time=6.640625 +GPU:0 | Epoch: 4 | loss=5.706753730773926 | Batch Time=0.390625 +GPU:0 | Epoch: 4 | loss=6.089773654937744 | Batch Time=0.390625 +GPU:0 | Epoch: 4 | loss=6.310647487640381 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=6.684844493865967 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=6.517068862915039 | Batch Time=0.78125 +GPU:0 | Epoch: 4 | loss=6.0237812995910645 | Batch Time=1.171875 +GPU:0 | Epoch: 4 | loss=6.394435405731201 | Batch Time=3.90625 +GPU:0 | Epoch: 4 | loss=5.994657039642334 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=5.940245628356934 | Batch Time=0.390625 +GPU:0 | Epoch: 4 | loss=6.057304382324219 | Batch Time=2.734375 +GPU:0 | Epoch: 4 | loss=6.212105751037598 | Batch Time=0.390625 +GPU:0 | Epoch: 4 | loss=5.511551380157471 | Batch Time=5.859375 +GPU:0 | Epoch: 4 | loss=5.155518531799316 | Batch Time=7.8125 +GPU:0 | Epoch: 4 | loss=4.581152439117432 | Batch Time=17.1875 +Model top1 Accuracy: 3.952 +Acc after rounding: 3.952 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 4 | loss=5.955047607421875 | Batch Time=3.515625 +GPU:0 | Epoch: 4 | loss=5.893013000488281 | Batch Time=7.03125 +GPU:0 | Epoch: 4 | loss=5.685566425323486 | Batch Time=5.859375 +GPU:0 | Epoch: 4 | loss=5.970531940460205 | Batch Time=3.125 +Model top1 Accuracy: 3.45 +Validation Acc after rounding: 3.45 +AFTER VAL LOOP: GPU: 0 | Epoch 4 | Memory Usage: svmem(total=257568083968, available=183122964480, percent=28.9, used=66534739968, free=23840182272, active=222553821184, inactive=6596780032, buffers=443748352, cached=166749413376, shared=5660327936, slab=3213815808) +Rounding model with scheme: naive +Model avg sparsity: 47.43790538372103 +GPU:0 | Epoch: 4 | Acc=3.952 | Epoch Time=15.57726065715154 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=183122964480, percent=28.9, used=66534739968, free=23840182272, active=222553817088, inactive=6596780032, buffers=443748352, cached=166749413376, shared=5660327936, slab=3213815808) +STARTING TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=183122964480, percent=28.9, used=66534739968, free=23840182272, active=222553817088, inactive=6596780032, buffers=443748352, cached=166749413376, shared=5660327936, slab=3213815808) +BEFORE TRAIN LOOP: Epoch 5 | Memory Usage: svmem(total=257568083968, available=183122964480, percent=28.9, used=66534739968, free=23840182272, active=222553817088, inactive=6596780032, buffers=443748352, cached=166749413376, shared=5660327936, slab=3213815808) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=183122964480, percent=28.9, used=66534739968, free=23840182272, active=222553817088, inactive=6596780032, buffers=443748352, cached=166749413376, shared=5660327936, slab=3213815808) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:3 | Epoch: 5 | loss=13.496786117553711 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.693229675292969 | Batch Time=2.734375 +GPU:3 | Epoch: 5 | loss=13.43934440612793 | Batch Time=2.34375 +GPU:3 | Epoch: 5 | loss=13.440479278564453 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.472558975219727 | Batch Time=5.859375 +GPU:3 | Epoch: 5 | loss=13.552519798278809 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.562211990356445 | Batch Time=2.734375 +GPU:3 | Epoch: 5 | loss=13.50802993774414 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.464916229248047 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.49167251586914 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.48759651184082 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.50787353515625 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.527925491333008 | Batch Time=1.953125 +GPU:3 | Epoch: 5 | loss=13.59560775756836 | Batch Time=2.34375 +GPU:3 | Epoch: 5 | loss=13.449324607849121 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.369592666625977 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.414520263671875 | Batch Time=6.25 +GPU:3 | Epoch: 5 | loss=13.508453369140625 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.493365287780762 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.466476440429688 | Batch Time=5.859375 +GPU:3 | Epoch: 5 | loss=13.585304260253906 | Batch Time=5.078125 +GPU:3 | Epoch: 5 | loss=13.546152114868164 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.574638366699219 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.409189224243164 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.538808822631836 | Batch Time=2.734375 +GPU:3 | Epoch: 5 | loss=13.421760559082031 | Batch Time=6.640625 +GPU:3 | Epoch: 5 | loss=13.45599365234375 | Batch Time=2.734375 +GPU:3 | Epoch: 5 | loss=13.446826934814453 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.476051330566406 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.360755920410156 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.47433853149414 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.404022216796875 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.45478343963623 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.541274070739746 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.400184631347656 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.480196952819824 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.4204740524292 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.589637756347656 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.568376541137695 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.485626220703125 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.486278533935547 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.478585243225098 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.576982498168945 | Batch Time=3.125 +GPU:3 | Epoch: 5 | loss=13.55225658416748 | Batch Time=2.734375 +GPU:3 | Epoch: 5 | loss=13.46477222442627 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.52513313293457 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.522757530212402 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.409311294555664 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.502464294433594 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.486055374145508 | Batch Time=1.953125 +GPU:3 | Epoch: 5 | loss=13.444316864013672 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.334739685058594 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.382970809936523 | Batch Time=5.859375 +GPU:3 | Epoch: 5 | loss=13.534051895141602 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.464183807373047 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.465847969055176 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.466712951660156 | Batch Time=5.078125 +GPU:3 | Epoch: 5 | loss=13.419940948486328 | Batch Time=5.078125 +GPU:3 | Epoch: 5 | loss=13.41780948638916 | Batch Time=1.171875 +GPU:3 | Epoch: 5 | loss=13.301586151123047 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.152779579162598 | Batch Time=5.859375 +GPU:3 | Epoch: 5 | loss=13.46322250366211 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.469841957092285 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.323408126831055 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.425811767578125 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.486409187316895 | Batch Time=3.125 +GPU:3 | Epoch: 5 | loss=13.38633918762207 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.321122169494629 | Batch Time=2.734375 +GPU:3 | Epoch: 5 | loss=13.356372833251953 | Batch Time=3.125 +GPU:3 | Epoch: 5 | loss=13.235223770141602 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.529956817626953 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.44272232055664 | Batch Time=5.078125 +GPU:3 | Epoch: 5 | loss=13.358722686767578 | Batch Time=6.640625 +GPU:3 | Epoch: 5 | loss=13.25950813293457 | Batch Time=6.640625 +GPU:3 | Epoch: 5 | loss=13.321224212646484 | Batch Time=3.125 +GPU:3 | Epoch: 5 | loss=13.37610149383545 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.231559753417969 | Batch Time=5.859375 +GPU:3 | Epoch: 5 | loss=13.327577590942383 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.249837875366211 | Batch Time=2.734375 +GPU:3 | Epoch: 5 | loss=13.216536521911621 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.370994567871094 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.34820556640625 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.349723815917969 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.336311340332031 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.371238708496094 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.286165237426758 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.353321075439453 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.239028930664062 | Batch Time=7.421875 +GPU:3 | Epoch: 5 | loss=13.300212860107422 | Batch Time=5.078125 +GPU:3 | Epoch: 5 | loss=13.200640678405762 | Batch Time=5.078125 +GPU:3 | Epoch: 5 | loss=13.32490062713623 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.322580337524414 | Batch Time=7.03125 +GPU:3 | Epoch: 5 | loss=13.422718048095703 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.24979019165039 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.414383888244629 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.200095176696777 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.319015502929688 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.298097610473633 | Batch Time=3.125 +GPU:3 | Epoch: 5 | loss=13.178869247436523 | Batch Time=7.8125 +GPU:3 | Epoch: 5 | loss=13.344548225402832 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.284934043884277 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.27829360961914 | Batch Time=3.125 +GPU:3 | Epoch: 5 | loss=13.138100624084473 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.351980209350586 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.26873779296875 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.241352081298828 | Batch Time=5.859375 +GPU:3 | Epoch: 5 | loss=13.423730850219727 | Batch Time=7.03125 +GPU:3 | Epoch: 5 | loss=13.074897766113281 | Batch Time=6.640625 +GPU:3 | Epoch: 5 | loss=13.26747989654541 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.365707397460938 | Batch Time=2.34375 +GPU:3 | Epoch: 5 | loss=13.215764999389648 | Batch Time=5.46875 +GPU:3 | Epoch: 5 | loss=13.232154846191406 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.219135284423828 | Batch Time=5.859375 +GPU:3 | Epoch: 5 | loss=13.311481475830078 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.421056747436523 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.123714447021484 | Batch Time=5.859375 +GPU:3 | Epoch: 5 | loss=13.295123100280762 | Batch Time=6.640625 +GPU:3 | Epoch: 5 | loss=13.181129455566406 | Batch Time=4.6875 +GPU:3 | Epoch: 5 | loss=13.382670402526855 | Batch Time=4.296875 +GPU:3 | Epoch: 5 | loss=13.33762264251709 | Batch Time=3.90625 +GPU:3 | Epoch: 5 | loss=13.207301139831543 | Batch Time=3.515625 +GPU:3 | Epoch: 5 | loss=13.2240571975708 | Batch Time=6.25 +GPU:3 | Epoch: 5 | loss=13.213454246520996 | Batch Time=6.640625 +GPU:3 | Epoch: 5 | loss=13.251079559326172 | Batch Time=6.25 +GPU:3 | Epoch: 5 | loss=13.17658519744873 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.674943923950195 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.683441162109375 | Batch Time=1.5625 +GPU:2 | Epoch: 5 | loss=13.677021026611328 | Batch Time=1.953125 +GPU:2 | Epoch: 5 | loss=13.542215347290039 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.475866317749023 | Batch Time=6.25 +GPU:2 | Epoch: 5 | loss=13.509516716003418 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.626604080200195 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.556114196777344 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.618953704833984 | Batch Time=2.34375 +GPU:2 | Epoch: 5 | loss=13.578231811523438 | Batch Time=5.46875 +GPU:2 | Epoch: 5 | loss=13.479175567626953 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.480537414550781 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.500133514404297 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.464057922363281 | Batch Time=5.078125 +GPU:2 | Epoch: 5 | loss=13.470988273620605 | Batch Time=1.5625 +GPU:2 | Epoch: 5 | loss=13.349462509155273 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.443552017211914 | Batch Time=5.078125 +GPU:2 | Epoch: 5 | loss=13.39634895324707 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.428256034851074 | Batch Time=6.640625 +GPU:2 | Epoch: 5 | loss=13.47520637512207 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.574241638183594 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.641626358032227 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.430418014526367 | Batch Time=5.46875 +GPU:2 | Epoch: 5 | loss=13.624490737915039 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.412446022033691 | Batch Time=5.46875 +GPU:2 | Epoch: 5 | loss=13.539527893066406 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.501426696777344 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.54281997680664 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.443937301635742 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.496322631835938 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.484516143798828 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.445415496826172 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.53979778289795 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.381558418273926 | Batch Time=5.46875 +GPU:2 | Epoch: 5 | loss=13.521507263183594 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.452498435974121 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.39807415008545 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.592021942138672 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.556798934936523 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.433011054992676 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.45880126953125 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.358967781066895 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.482202529907227 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.481038093566895 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.393078804016113 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.361763000488281 | Batch Time=2.34375 +GPU:2 | Epoch: 5 | loss=13.422684669494629 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.440738677978516 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.490631103515625 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.418352127075195 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.490806579589844 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.399862289428711 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.511367797851562 | Batch Time=5.078125 +GPU:2 | Epoch: 5 | loss=13.42094612121582 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.440296173095703 | Batch Time=5.078125 +GPU:2 | Epoch: 5 | loss=13.519633293151855 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.464942932128906 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.380308151245117 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.412812232971191 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.43472671508789 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.504122734069824 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.407020568847656 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.425810813903809 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.546087265014648 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.443957328796387 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.493729591369629 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.557882308959961 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.470917701721191 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.290245056152344 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.465076446533203 | Batch Time=1.953125 +GPU:2 | Epoch: 5 | loss=13.432353973388672 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.427787780761719 | Batch Time=2.34375 +GPU:2 | Epoch: 5 | loss=13.344635009765625 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.341255187988281 | Batch Time=5.46875 +GPU:2 | Epoch: 5 | loss=13.320565223693848 | Batch Time=1.171875 +GPU:2 | Epoch: 5 | loss=13.458914756774902 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.41876220703125 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.448297500610352 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.332778930664062 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.47809886932373 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.340198516845703 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.55389404296875 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.411828994750977 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.561254501342773 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.381868362426758 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.455842971801758 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.258790016174316 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.31661605834961 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.221389770507812 | Batch Time=6.640625 +GPU:2 | Epoch: 5 | loss=13.293890953063965 | Batch Time=1.953125 +GPU:2 | Epoch: 5 | loss=13.398818016052246 | Batch Time=2.34375 +GPU:2 | Epoch: 5 | loss=13.341909408569336 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.293903350830078 | Batch Time=3.90625 +GPU:2 | Epoch: 5 | loss=13.465421676635742 | Batch Time=5.078125 +GPU:2 | Epoch: 5 | loss=13.38919448852539 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.485291481018066 | Batch Time=1.953125 +GPU:2 | Epoch: 5 | loss=13.274770736694336 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.276437759399414 | Batch Time=5.46875 +GPU:2 | Epoch: 5 | loss=13.186330795288086 | Batch Time=5.46875 +GPU:2 | Epoch: 5 | loss=13.41039752960205 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.424269676208496 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.216779708862305 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.401658058166504 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.394376754760742 | Batch Time=6.25 +GPU:2 | Epoch: 5 | loss=13.218803405761719 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.31375503540039 | Batch Time=7.03125 +GPU:2 | Epoch: 5 | loss=13.514299392700195 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.460189819335938 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.343338966369629 | Batch Time=3.515625 +GPU:2 | Epoch: 5 | loss=13.28303337097168 | Batch Time=5.078125 +GPU:2 | Epoch: 5 | loss=13.332220077514648 | Batch Time=6.25 +GPU:2 | Epoch: 5 | loss=13.243894577026367 | Batch Time=4.6875 +GPU:2 | Epoch: 5 | loss=13.41141128540039 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.262653350830078 | Batch Time=5.078125 +GPU:2 | Epoch: 5 | loss=13.321569442749023 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.219696044921875 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.238821983337402 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.346729278564453 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.243163108825684 | Batch Time=5.078125 +GPU:2 | Epoch: 5 | loss=13.36575984954834 | Batch Time=2.734375 +GPU:2 | Epoch: 5 | loss=13.267924308776855 | Batch Time=2.34375 +GPU:2 | Epoch: 5 | loss=13.379167556762695 | Batch Time=4.296875 +GPU:2 | Epoch: 5 | loss=13.300270080566406 | Batch Time=5.859375 +GPU:2 | Epoch: 5 | loss=13.273963928222656 | Batch Time=3.125 +GPU:2 | Epoch: 5 | loss=13.184222221374512 | Batch Time=5.46875 +GPU:0 | Epoch: 5 | loss=13.531206130981445 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.437248229980469 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.490570068359375 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.530965805053711 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.525117874145508 | Batch Time=2.734375 +GPU:0 | Epoch: 5 | loss=13.578375816345215 | Batch Time=2.734375 +GPU:0 | Epoch: 5 | loss=13.634454727172852 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.623470306396484 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=13.452657699584961 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.650733947753906 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.460956573486328 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.664688110351562 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.472126007080078 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.559741973876953 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.574481010437012 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=13.519680976867676 | Batch Time=2.734375 +GPU:0 | Epoch: 5 | loss=13.499919891357422 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.628829956054688 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.455798149108887 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.567934036254883 | Batch Time=2.734375 +GPU:0 | Epoch: 5 | loss=13.532760620117188 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.39506721496582 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.517980575561523 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.514820098876953 | Batch Time=1.953125 +GPU:0 | Epoch: 5 | loss=13.545331954956055 | Batch Time=0.78125 +GPU:0 | Epoch: 5 | loss=13.508556365966797 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.550201416015625 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=13.399417877197266 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.386892318725586 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.4249267578125 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.549226760864258 | Batch Time=5.46875 +GPU:0 | Epoch: 5 | loss=13.464712142944336 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.519793510437012 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.46911907196045 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.324031829833984 | Batch Time=6.640625 +GPU:0 | Epoch: 5 | loss=13.480250358581543 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.474396705627441 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.509284973144531 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=13.35827350616455 | Batch Time=5.46875 +GPU:0 | Epoch: 5 | loss=13.311259269714355 | Batch Time=5.859375 +GPU:0 | Epoch: 5 | loss=13.496734619140625 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.499726295471191 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.46540641784668 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.52049732208252 | Batch Time=2.734375 +GPU:0 | Epoch: 5 | loss=13.347687721252441 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.50790786743164 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.588128089904785 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.62675666809082 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=13.458694458007812 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.46000862121582 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=13.472921371459961 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.489896774291992 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.362205505371094 | Batch Time=5.859375 +GPU:0 | Epoch: 5 | loss=13.448068618774414 | Batch Time=2.734375 +GPU:0 | Epoch: 5 | loss=13.346309661865234 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.421853065490723 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.310583114624023 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.330938339233398 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.30492877960205 | Batch Time=7.03125 +GPU:0 | Epoch: 5 | loss=13.445930480957031 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.450966835021973 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.319679260253906 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.4530611038208 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.519636154174805 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.413932800292969 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.359051704406738 | Batch Time=8.203125 +GPU:0 | Epoch: 5 | loss=13.461481094360352 | Batch Time=5.859375 +GPU:0 | Epoch: 5 | loss=13.373805046081543 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.32156753540039 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.382720947265625 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.4888916015625 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.373476028442383 | Batch Time=5.859375 +GPU:0 | Epoch: 5 | loss=13.376527786254883 | Batch Time=7.421875 +GPU:0 | Epoch: 5 | loss=13.32925796508789 | Batch Time=5.859375 +GPU:0 | Epoch: 5 | loss=13.28697681427002 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.198662757873535 | Batch Time=5.46875 +GPU:0 | Epoch: 5 | loss=13.301658630371094 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.559846878051758 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=13.256168365478516 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.29959774017334 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.45309066772461 | Batch Time=1.5625 +GPU:0 | Epoch: 5 | loss=13.484031677246094 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.413299560546875 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=13.24410629272461 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.214591979980469 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.301725387573242 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.313377380371094 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.230230331420898 | Batch Time=5.46875 +GPU:0 | Epoch: 5 | loss=13.468894958496094 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.393662452697754 | Batch Time=5.46875 +GPU:0 | Epoch: 5 | loss=13.375865936279297 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.332128524780273 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.241246223449707 | Batch Time=6.640625 +GPU:0 | Epoch: 5 | loss=13.27285385131836 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.218022346496582 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.301791191101074 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.307069778442383 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.234792709350586 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.342262268066406 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.299071311950684 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.308095932006836 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.33975601196289 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.413472175598145 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.421936988830566 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.460247039794922 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.382888793945312 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.197370529174805 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=13.199508666992188 | Batch Time=6.25 +GPU:0 | Epoch: 5 | loss=13.499531745910645 | Batch Time=1.953125 +GPU:0 | Epoch: 5 | loss=13.34130859375 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.290246963500977 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.1361083984375 | Batch Time=6.25 +GPU:0 | Epoch: 5 | loss=13.063011169433594 | Batch Time=5.46875 +GPU:0 | Epoch: 5 | loss=13.316032409667969 | Batch Time=5.078125 +GPU:0 | Epoch: 5 | loss=13.546010971069336 | Batch Time=3.515625 +GPU:0 | Epoch: 5 | loss=13.37588119506836 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.290953636169434 | Batch Time=6.25 +GPU:0 | Epoch: 5 | loss=13.159614562988281 | Batch Time=6.25 +GPU:0 | Epoch: 5 | loss=13.279494285583496 | Batch Time=5.859375 +GPU:0 | Epoch: 5 | loss=13.332453727722168 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=13.256253242492676 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.138166427612305 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.300938606262207 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=13.141975402832031 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=13.234892845153809 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.551248550415039 | Batch Time=6.25 +GPU:1 | Epoch: 5 | loss=13.423690795898438 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.47930908203125 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.53398323059082 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.638494491577148 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.483668327331543 | Batch Time=5.078125 +GPU:1 | Epoch: 5 | loss=13.673906326293945 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.520912170410156 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.571317672729492 | Batch Time=1.953125 +GPU:1 | Epoch: 5 | loss=13.44088363647461 | Batch Time=7.03125 +GPU:1 | Epoch: 5 | loss=13.526851654052734 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.413116455078125 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.43136978149414 | Batch Time=5.078125 +GPU:1 | Epoch: 5 | loss=13.480358123779297 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.50665283203125 | Batch Time=5.078125 +GPU:1 | Epoch: 5 | loss=13.63766860961914 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.52386474609375 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.499797821044922 | Batch Time=5.859375 +GPU:1 | Epoch: 5 | loss=13.640429496765137 | Batch Time=2.734375 +GPU:1 | Epoch: 5 | loss=13.403444290161133 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.521976470947266 | Batch Time=2.734375 +GPU:1 | Epoch: 5 | loss=13.348043441772461 | Batch Time=6.640625 +GPU:1 | Epoch: 5 | loss=13.442102432250977 | Batch Time=6.640625 +GPU:1 | Epoch: 5 | loss=13.464311599731445 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.440591812133789 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.50284194946289 | Batch Time=5.859375 +GPU:1 | Epoch: 5 | loss=13.65560531616211 | Batch Time=2.34375 +GPU:1 | Epoch: 5 | loss=13.340240478515625 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.436225891113281 | Batch Time=5.078125 +GPU:1 | Epoch: 5 | loss=13.349143981933594 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.467144012451172 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.434556007385254 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.448565483093262 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.464077949523926 | Batch Time=6.640625 +GPU:1 | Epoch: 5 | loss=13.44837760925293 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.43074893951416 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.543585777282715 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.485492706298828 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.559258460998535 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.400574684143066 | Batch Time=6.25 +GPU:1 | Epoch: 5 | loss=13.45132827758789 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.483433723449707 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.469537734985352 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.387579917907715 | Batch Time=2.734375 +GPU:1 | Epoch: 5 | loss=13.433201789855957 | Batch Time=5.078125 +GPU:1 | Epoch: 5 | loss=13.47335433959961 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.498661994934082 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.427270889282227 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.284355163574219 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.588369369506836 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.363349914550781 | Batch Time=5.859375 +GPU:1 | Epoch: 5 | loss=13.452167510986328 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.489404678344727 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.355676651000977 | Batch Time=6.25 +GPU:1 | Epoch: 5 | loss=13.463916778564453 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.510621070861816 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.28017807006836 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.553564071655273 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.345843315124512 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.358089447021484 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.514843940734863 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.389137268066406 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.42211627960205 | Batch Time=2.734375 +GPU:1 | Epoch: 5 | loss=13.342016220092773 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.426568031311035 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.425007820129395 | Batch Time=2.734375 +GPU:1 | Epoch: 5 | loss=13.47252082824707 | Batch Time=2.34375 +GPU:1 | Epoch: 5 | loss=13.4423189163208 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.25546646118164 | Batch Time=5.859375 +GPU:1 | Epoch: 5 | loss=13.404632568359375 | Batch Time=2.734375 +GPU:1 | Epoch: 5 | loss=13.391498565673828 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.471771240234375 | Batch Time=0.78125 +GPU:1 | Epoch: 5 | loss=13.43050765991211 | Batch Time=1.953125 +GPU:1 | Epoch: 5 | loss=13.528007507324219 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.395684242248535 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.44025707244873 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.339191436767578 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.39445686340332 | Batch Time=5.078125 +GPU:1 | Epoch: 5 | loss=13.377206802368164 | Batch Time=6.25 +GPU:1 | Epoch: 5 | loss=13.370993614196777 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.345195770263672 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.339958190917969 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.28411865234375 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.37677001953125 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.338760375976562 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.40251350402832 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.287757873535156 | Batch Time=5.078125 +GPU:1 | Epoch: 5 | loss=13.338443756103516 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.302074432373047 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.489008903503418 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.24438762664795 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.263082504272461 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.21744155883789 | Batch Time=6.25 +GPU:1 | Epoch: 5 | loss=13.387351036071777 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.439183235168457 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.219943046569824 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.384119033813477 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.318941116333008 | Batch Time=6.25 +GPU:1 | Epoch: 5 | loss=13.399164199829102 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.289307594299316 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.39093017578125 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.293855667114258 | Batch Time=5.859375 +GPU:1 | Epoch: 5 | loss=13.511483192443848 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.275674819946289 | Batch Time=5.859375 +GPU:1 | Epoch: 5 | loss=13.297853469848633 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.281852722167969 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.263853073120117 | Batch Time=6.25 +GPU:1 | Epoch: 5 | loss=13.4366455078125 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.23869800567627 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.365863800048828 | Batch Time=3.515625 +GPU:1 | Epoch: 5 | loss=13.16978645324707 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.249786376953125 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.174087524414062 | Batch Time=5.859375 +GPU:1 | Epoch: 5 | loss=13.27444839477539 | Batch Time=5.078125 +GPU:1 | Epoch: 5 | loss=13.173482894897461 | Batch Time=3.125 +GPU:1 | Epoch: 5 | loss=13.251583099365234 | Batch Time=4.296875 +GPU:1 | Epoch: 5 | loss=13.2017240524292 | Batch Time=6.25 +GPU:1 | Epoch: 5 | loss=13.292217254638672 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.238692283630371 | Batch Time=3.90625 +GPU:1 | Epoch: 5 | loss=13.266962051391602 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.148990631103516 | Batch Time=4.6875 +GPU:1 | Epoch: 5 | loss=13.26240348815918 | Batch Time=5.859375 +GPU:1 | Epoch: 5 | loss=13.302247047424316 | Batch Time=2.734375 +GPU:1 | Epoch: 5 | loss=13.425178527832031 | Batch Time=5.46875 +GPU:1 | Epoch: 5 | loss=13.437438011169434 | Batch Time=5.46875 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=193368576000, percent=24.9, used=56289161216, free=34030993408, active=212723564544, inactive=6273437696, buffers=443801600, cached=166804127744, shared=5660295168, slab=3213516800) +AFTER TRAIN LOOP: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369939968, percent=24.9, used=56287952896, free=34032349184, active=212723150848, inactive=6273380352, buffers=443801600, cached=166803980288, shared=5660131328, slab=3213484032) +BEFORE VAL LOOP: GPU: 0 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369939968, percent=24.9, used=56287952896, free=34032349184, active=212723150848, inactive=6273380352, buffers=443801600, cached=166803980288, shared=5660131328, slab=3213484032) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=193368543232, percent=24.9, used=56289193984, free=34030960640, active=212723564544, inactive=6273458176, buffers=443801600, cached=166804127744, shared=5660295168, slab=3213524992) +AFTER TRAIN LOOP: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193368576000, percent=24.9, used=56289161216, free=34030993408, active=212723564544, inactive=6273437696, buffers=443801600, cached=166804127744, shared=5660295168, slab=3213516800) +BEFORE PRUNE: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193368576000, percent=24.9, used=56289161216, free=34030993408, active=212723564544, inactive=6273437696, buffers=443801600, cached=166804127744, shared=5660295168, slab=3213516800) +Pruning Model: +AFTER PRUNE: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +STARTING TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +BEFORE TRAIN LOOP: Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=193368576000, percent=24.9, used=56289161216, free=34030993408, active=212723564544, inactive=6273437696, buffers=443801600, cached=166804127744, shared=5660295168, slab=3213516800) +AFTER TRAIN LOOP: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369939968, percent=24.9, used=56287952896, free=34032349184, active=212723150848, inactive=6273380352, buffers=443801600, cached=166803980288, shared=5660131328, slab=3213484032) +BEFORE PRUNE: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369939968, percent=24.9, used=56287952896, free=34032349184, active=212723150848, inactive=6273380352, buffers=443801600, cached=166803980288, shared=5660131328, slab=3213484032) +Pruning Model: +AFTER PRUNE: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +STARTING TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +BEFORE TRAIN LOOP: Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=193368576000, percent=24.9, used=56289161216, free=34030993408, active=212723564544, inactive=6273437696, buffers=443801600, cached=166804127744, shared=5660295168, slab=3213516800) +AFTER TRAIN LOOP: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369939968, percent=24.9, used=56287952896, free=34032349184, active=212723150848, inactive=6273380352, buffers=443801600, cached=166803980288, shared=5660131328, slab=3213484032) +BEFORE PRUNE: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369939968, percent=24.9, used=56287952896, free=34032349184, active=212723150848, inactive=6273380352, buffers=443801600, cached=166803980288, shared=5660131328, slab=3213484032) +Pruning Model: +AFTER PRUNE: Epoch 5 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +STARTING TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +BEFORE TRAIN LOOP: Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=193369567232, percent=24.9, used=56288325632, free=34030833664, active=212723150848, inactive=6274490368, buffers=443801600, cached=166805123072, shared=5660049408, slab=3213443072) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 5 | loss=4.543580532073975 | Batch Time=6.640625 +GPU:0 | Epoch: 5 | loss=5.752195358276367 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=5.065025329589844 | Batch Time=10.9375 +GPU:0 | Epoch: 5 | loss=5.715673446655273 | Batch Time=1.171875 +GPU:0 | Epoch: 5 | loss=5.755899906158447 | Batch Time=0.0 +GPU:0 | Epoch: 5 | loss=5.283850193023682 | Batch Time=6.25 +GPU:0 | Epoch: 5 | loss=5.920328140258789 | Batch Time=0.78125 +GPU:0 | Epoch: 5 | loss=5.71626091003418 | Batch Time=1.171875 +GPU:0 | Epoch: 5 | loss=6.231677055358887 | Batch Time=0.390625 +GPU:0 | Epoch: 5 | loss=6.55417537689209 | Batch Time=0.0 +GPU:0 | Epoch: 5 | loss=6.257343292236328 | Batch Time=1.953125 +GPU:0 | Epoch: 5 | loss=5.728674411773682 | Batch Time=2.734375 +GPU:0 | Epoch: 5 | loss=6.265360355377197 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=5.7964396476745605 | Batch Time=7.421875 +GPU:0 | Epoch: 5 | loss=5.865978717803955 | Batch Time=1.5625 +GPU:0 | Epoch: 5 | loss=6.006193161010742 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=5.8852434158325195 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=5.1945109367370605 | Batch Time=7.8125 +GPU:0 | Epoch: 5 | loss=5.222732067108154 | Batch Time=6.640625 +GPU:0 | Epoch: 5 | loss=4.358987808227539 | Batch Time=24.609375 +Model top1 Accuracy: 4.98 +Acc before rounding: 4.98 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 5 | loss=4.543580532073975 | Batch Time=6.640625 +GPU:0 | Epoch: 5 | loss=5.752195358276367 | Batch Time=3.125 +GPU:0 | Epoch: 5 | loss=5.065025329589844 | Batch Time=10.9375 +GPU:0 | Epoch: 5 | loss=5.715673446655273 | Batch Time=1.171875 +GPU:0 | Epoch: 5 | loss=5.755899906158447 | Batch Time=0.0 +GPU:0 | Epoch: 5 | loss=5.283850193023682 | Batch Time=6.25 +GPU:0 | Epoch: 5 | loss=5.920328140258789 | Batch Time=0.78125 +GPU:0 | Epoch: 5 | loss=5.71626091003418 | Batch Time=1.171875 +GPU:0 | Epoch: 5 | loss=6.231677055358887 | Batch Time=0.390625 +GPU:0 | Epoch: 5 | loss=6.55417537689209 | Batch Time=0.0 +GPU:0 | Epoch: 5 | loss=6.257343292236328 | Batch Time=1.953125 +GPU:0 | Epoch: 5 | loss=5.728674411773682 | Batch Time=2.734375 +GPU:0 | Epoch: 5 | loss=6.265360355377197 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=5.7964396476745605 | Batch Time=7.421875 +GPU:0 | Epoch: 5 | loss=5.865978717803955 | Batch Time=1.5625 +GPU:0 | Epoch: 5 | loss=6.006193161010742 | Batch Time=3.90625 +GPU:0 | Epoch: 5 | loss=5.8852434158325195 | Batch Time=2.34375 +GPU:0 | Epoch: 5 | loss=5.1945109367370605 | Batch Time=7.8125 +GPU:0 | Epoch: 5 | loss=5.222732067108154 | Batch Time=6.640625 +GPU:0 | Epoch: 5 | loss=4.358987808227539 | Batch Time=24.609375 +Model top1 Accuracy: 4.98 +Acc after rounding: 4.98 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 5 | loss=5.638490676879883 | Batch Time=4.6875 +GPU:0 | Epoch: 5 | loss=5.773346424102783 | Batch Time=6.25 +GPU:0 | Epoch: 5 | loss=5.5746941566467285 | Batch Time=4.296875 +GPU:0 | Epoch: 5 | loss=5.771341323852539 | Batch Time=5.46875 +Model top1 Accuracy: 4.31 +Validation Acc after rounding: 4.31 +AFTER VAL LOOP: GPU: 0 | Epoch 5 | Memory Usage: svmem(total=257568083968, available=175564505088, percent=31.8, used=74093076480, free=16223891456, active=230416076800, inactive=6273908736, buffers=443854848, cached=166807261184, shared=5660442624, slab=3221876736) +BEFORE PRUNE: Epoch 5 | Memory Usage: svmem(total=257568083968, available=175564505088, percent=31.8, used=74093076480, free=16223891456, active=230416076800, inactive=6273908736, buffers=443854848, cached=166807261184, shared=5660442624, slab=3221876736) +Pruning Model: +AFTER PRUNE: Epoch 5 | Memory Usage: svmem(total=257568083968, available=175564537856, percent=31.8, used=74093043712, free=16223924224, active=230416076800, inactive=6273908736, buffers=443854848, cached=166807261184, shared=5660442624, slab=3221843968) +Rounding model with scheme: naive +Model avg sparsity: 46.91265282825525 +GPU:0 | Epoch: 5 | Acc=4.98 | Epoch Time=15.569342863559722 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=175564537856, percent=31.8, used=74093043712, free=16223924224, active=230416076800, inactive=6273908736, buffers=443854848, cached=166807261184, shared=5660442624, slab=3221843968) +STARTING TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=175564537856, percent=31.8, used=74093043712, free=16223924224, active=230416076800, inactive=6273908736, buffers=443854848, cached=166807261184, shared=5660442624, slab=3221843968) +BEFORE TRAIN LOOP: Epoch 6 | Memory Usage: svmem(total=257568083968, available=175564537856, percent=31.8, used=74093043712, free=16223924224, active=230416076800, inactive=6273908736, buffers=443854848, cached=166807261184, shared=5660442624, slab=3221843968) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=175564537856, percent=31.8, used=74093043712, free=16223924224, active=230416076800, inactive=6273908736, buffers=443854848, cached=166807261184, shared=5660442624, slab=3221843968) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 6 | loss=13.23299789428711 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=13.218084335327148 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.294320106506348 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=13.21348762512207 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=13.247294425964355 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=13.249215126037598 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.337087631225586 | Batch Time=3.515625 +GPU:1 | Epoch: 6 | loss=13.217734336853027 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=13.371139526367188 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=13.14659309387207 | Batch Time=7.421875 +GPU:1 | Epoch: 6 | loss=13.10950756072998 | Batch Time=6.25 +GPU:1 | Epoch: 6 | loss=13.28329849243164 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=13.14305305480957 | Batch Time=7.8125 +GPU:1 | Epoch: 6 | loss=13.256153106689453 | Batch Time=7.03125 +GPU:1 | Epoch: 6 | loss=13.210054397583008 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=13.137022018432617 | Batch Time=8.59375 +GPU:1 | Epoch: 6 | loss=13.203484535217285 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.230241775512695 | Batch Time=3.515625 +GPU:1 | Epoch: 6 | loss=13.228443145751953 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.319612503051758 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=13.188680648803711 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=13.264263153076172 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=13.113000869750977 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=13.170670509338379 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.191137313842773 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=13.11917495727539 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=13.128570556640625 | Batch Time=6.640625 +GPU:1 | Epoch: 6 | loss=13.27911376953125 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.104716300964355 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.036502838134766 | Batch Time=8.59375 +GPU:1 | Epoch: 6 | loss=13.130931854248047 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.18940544128418 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=13.102439880371094 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.248613357543945 | Batch Time=1.5625 +GPU:1 | Epoch: 6 | loss=13.22521686553955 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=13.207941055297852 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.246475219726562 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.192400932312012 | Batch Time=6.640625 +GPU:1 | Epoch: 6 | loss=13.171967506408691 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=12.994386672973633 | Batch Time=7.03125 +GPU:1 | Epoch: 6 | loss=13.109033584594727 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.100630760192871 | Batch Time=8.203125 +GPU:1 | Epoch: 6 | loss=13.200389862060547 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.097493171691895 | Batch Time=3.515625 +GPU:1 | Epoch: 6 | loss=13.249223709106445 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.159049034118652 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.158149719238281 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=13.04238510131836 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=12.991144180297852 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.139238357543945 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=12.98096752166748 | Batch Time=7.03125 +GPU:1 | Epoch: 6 | loss=13.159891128540039 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=12.98992919921875 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=13.143413543701172 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=13.161559104919434 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.043776512145996 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.129863739013672 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=13.103872299194336 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=13.137598037719727 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.097601890563965 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.135082244873047 | Batch Time=1.5625 +GPU:1 | Epoch: 6 | loss=13.2105712890625 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.104381561279297 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.172237396240234 | Batch Time=7.03125 +GPU:1 | Epoch: 6 | loss=12.999610900878906 | Batch Time=9.765625 +GPU:1 | Epoch: 6 | loss=13.09189224243164 | Batch Time=3.515625 +GPU:1 | Epoch: 6 | loss=13.18537712097168 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=13.188531875610352 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=12.99423599243164 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.175403594970703 | Batch Time=3.515625 +GPU:1 | Epoch: 6 | loss=12.979562759399414 | Batch Time=7.421875 +GPU:1 | Epoch: 6 | loss=13.150375366210938 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.214048385620117 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=13.188142776489258 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.288806915283203 | Batch Time=2.34375 +GPU:1 | Epoch: 6 | loss=12.98112678527832 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.072850227355957 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.049520492553711 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=12.980111122131348 | Batch Time=8.984375 +GPU:1 | Epoch: 6 | loss=13.141469955444336 | Batch Time=3.125 +GPU:1 | Epoch: 6 | loss=13.012845039367676 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.135635375976562 | Batch Time=7.03125 +GPU:1 | Epoch: 6 | loss=13.08877944946289 | Batch Time=6.640625 +GPU:1 | Epoch: 6 | loss=13.031688690185547 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=13.120535850524902 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=12.9979248046875 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.1668062210083 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.042914390563965 | Batch Time=7.03125 +GPU:1 | Epoch: 6 | loss=12.926612854003906 | Batch Time=6.25 +GPU:1 | Epoch: 6 | loss=13.171114921569824 | Batch Time=3.515625 +GPU:1 | Epoch: 6 | loss=12.997615814208984 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=12.994991302490234 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=12.978348731994629 | Batch Time=6.640625 +GPU:1 | Epoch: 6 | loss=12.981218338012695 | Batch Time=6.25 +GPU:1 | Epoch: 6 | loss=13.12893009185791 | Batch Time=2.734375 +GPU:1 | Epoch: 6 | loss=13.14332389831543 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=12.970382690429688 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=13.062814712524414 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.109926223754883 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=13.137139320373535 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.141439437866211 | Batch Time=2.734375 +GPU:1 | Epoch: 6 | loss=12.938520431518555 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=12.935408592224121 | Batch Time=7.421875 +GPU:1 | Epoch: 6 | loss=13.010173797607422 | Batch Time=6.25 +GPU:1 | Epoch: 6 | loss=12.799068450927734 | Batch Time=7.421875 +GPU:1 | Epoch: 6 | loss=13.072246551513672 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=12.990110397338867 | Batch Time=6.640625 +GPU:1 | Epoch: 6 | loss=12.976485252380371 | Batch Time=8.203125 +GPU:1 | Epoch: 6 | loss=13.055925369262695 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=12.880941390991211 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=13.12844181060791 | Batch Time=2.734375 +GPU:1 | Epoch: 6 | loss=12.936023712158203 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=12.929814338684082 | Batch Time=6.640625 +GPU:1 | Epoch: 6 | loss=13.096654891967773 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=12.851707458496094 | Batch Time=6.25 +GPU:1 | Epoch: 6 | loss=12.890241622924805 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=13.141385078430176 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=12.93453598022461 | Batch Time=3.90625 +GPU:1 | Epoch: 6 | loss=12.927494049072266 | Batch Time=5.859375 +GPU:1 | Epoch: 6 | loss=12.972003936767578 | Batch Time=5.46875 +GPU:1 | Epoch: 6 | loss=12.980978965759277 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=13.035491943359375 | Batch Time=4.6875 +GPU:1 | Epoch: 6 | loss=12.986120223999023 | Batch Time=5.078125 +GPU:1 | Epoch: 6 | loss=12.93771743774414 | Batch Time=4.296875 +GPU:1 | Epoch: 6 | loss=12.940055847167969 | Batch Time=7.03125 +GPU:3 | Epoch: 6 | loss=13.241506576538086 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.329334259033203 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.154812812805176 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.211183547973633 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.169245719909668 | Batch Time=6.640625 +GPU:3 | Epoch: 6 | loss=13.211258888244629 | Batch Time=7.8125 +GPU:3 | Epoch: 6 | loss=13.200115203857422 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.285265922546387 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.210119247436523 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=13.382383346557617 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.257418632507324 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.213348388671875 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.218021392822266 | Batch Time=5.859375 +GPU:3 | Epoch: 6 | loss=13.175310134887695 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=13.303853988647461 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=13.20289421081543 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.173415184020996 | Batch Time=5.859375 +GPU:3 | Epoch: 6 | loss=13.309457778930664 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.286809921264648 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.166496276855469 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.152478218078613 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.187488555908203 | Batch Time=5.859375 +GPU:3 | Epoch: 6 | loss=13.179910659790039 | Batch Time=3.90625 +GPU:3 | Epoch: 6 | loss=13.375615119934082 | Batch Time=1.171875 +GPU:3 | Epoch: 6 | loss=13.301164627075195 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.242120742797852 | Batch Time=5.859375 +GPU:3 | Epoch: 6 | loss=13.136941909790039 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.21290397644043 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.230441093444824 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.20065975189209 | Batch Time=3.125 +GPU:3 | Epoch: 6 | loss=13.187812805175781 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=13.231643676757812 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.187032699584961 | Batch Time=3.90625 +GPU:3 | Epoch: 6 | loss=13.008180618286133 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.121304512023926 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.098569869995117 | Batch Time=7.8125 +GPU:3 | Epoch: 6 | loss=13.12674331665039 | Batch Time=3.90625 +GPU:3 | Epoch: 6 | loss=13.2641019821167 | Batch Time=3.90625 +GPU:3 | Epoch: 6 | loss=12.996460914611816 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.145444869995117 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.053747177124023 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.106795310974121 | Batch Time=7.8125 +GPU:3 | Epoch: 6 | loss=13.141464233398438 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.092933654785156 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.054708480834961 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.254124641418457 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.08822250366211 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.113334655761719 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.284233093261719 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.076887130737305 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=13.21657657623291 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.201719284057617 | Batch Time=7.03125 +GPU:3 | Epoch: 6 | loss=13.030925750732422 | Batch Time=6.640625 +GPU:3 | Epoch: 6 | loss=13.178998947143555 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=12.986895561218262 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.230345726013184 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.008983612060547 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=12.823762893676758 | Batch Time=9.765625 +GPU:3 | Epoch: 6 | loss=13.013620376586914 | Batch Time=5.859375 +GPU:3 | Epoch: 6 | loss=13.121684074401855 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.281070709228516 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=13.132107734680176 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=12.986310958862305 | Batch Time=7.8125 +GPU:3 | Epoch: 6 | loss=13.180514335632324 | Batch Time=2.34375 +GPU:3 | Epoch: 6 | loss=13.089534759521484 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.16507339477539 | Batch Time=5.859375 +GPU:3 | Epoch: 6 | loss=13.102113723754883 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.07524299621582 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=12.937384605407715 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.175010681152344 | Batch Time=3.125 +GPU:3 | Epoch: 6 | loss=13.148153305053711 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.092323303222656 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.240278244018555 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.013381958007812 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.008960723876953 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.178258895874023 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.057950973510742 | Batch Time=5.859375 +GPU:3 | Epoch: 6 | loss=13.112771987915039 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=13.114888191223145 | Batch Time=3.125 +GPU:3 | Epoch: 6 | loss=13.239721298217773 | Batch Time=2.734375 +GPU:3 | Epoch: 6 | loss=13.048785209655762 | Batch Time=1.953125 +GPU:3 | Epoch: 6 | loss=12.986774444580078 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=12.882957458496094 | Batch Time=8.59375 +GPU:3 | Epoch: 6 | loss=13.054450988769531 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.101944923400879 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=13.037982940673828 | Batch Time=7.8125 +GPU:3 | Epoch: 6 | loss=13.170957565307617 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.0958833694458 | Batch Time=5.859375 +GPU:3 | Epoch: 6 | loss=13.1116943359375 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.018805503845215 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=12.961734771728516 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.011964797973633 | Batch Time=3.125 +GPU:3 | Epoch: 6 | loss=13.062691688537598 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=12.998466491699219 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=12.87038516998291 | Batch Time=7.421875 +GPU:3 | Epoch: 6 | loss=13.041633605957031 | Batch Time=3.90625 +GPU:3 | Epoch: 6 | loss=13.057428359985352 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=12.980470657348633 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.113771438598633 | Batch Time=7.03125 +GPU:3 | Epoch: 6 | loss=12.982003211975098 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.023962020874023 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=12.92059326171875 | Batch Time=7.03125 +GPU:3 | Epoch: 6 | loss=12.785954475402832 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=12.995655059814453 | Batch Time=3.515625 +GPU:3 | Epoch: 6 | loss=12.94102668762207 | Batch Time=6.25 +GPU:3 | Epoch: 6 | loss=13.088361740112305 | Batch Time=7.8125 +GPU:3 | Epoch: 6 | loss=13.065671920776367 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=13.027874946594238 | Batch Time=7.8125 +GPU:3 | Epoch: 6 | loss=12.852249145507812 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=12.746322631835938 | Batch Time=7.421875 +GPU:3 | Epoch: 6 | loss=13.100445747375488 | Batch Time=3.90625 +GPU:3 | Epoch: 6 | loss=13.116361618041992 | Batch Time=5.46875 +GPU:3 | Epoch: 6 | loss=12.885760307312012 | Batch Time=7.03125 +GPU:3 | Epoch: 6 | loss=13.099859237670898 | Batch Time=6.640625 +GPU:3 | Epoch: 6 | loss=13.005613327026367 | Batch Time=7.03125 +GPU:3 | Epoch: 6 | loss=12.962873458862305 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=12.971811294555664 | Batch Time=6.640625 +GPU:3 | Epoch: 6 | loss=13.063741683959961 | Batch Time=2.34375 +GPU:3 | Epoch: 6 | loss=12.97688102722168 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.1146240234375 | Batch Time=3.90625 +GPU:3 | Epoch: 6 | loss=13.06474781036377 | Batch Time=4.296875 +GPU:3 | Epoch: 6 | loss=12.814756393432617 | Batch Time=7.8125 +GPU:3 | Epoch: 6 | loss=13.06442928314209 | Batch Time=5.078125 +GPU:3 | Epoch: 6 | loss=13.011270523071289 | Batch Time=4.6875 +GPU:3 | Epoch: 6 | loss=13.027099609375 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.249767303466797 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.127008438110352 | Batch Time=8.59375 +GPU:2 | Epoch: 6 | loss=13.241291999816895 | Batch Time=1.171875 +GPU:2 | Epoch: 6 | loss=13.244176864624023 | Batch Time=2.34375 +GPU:2 | Epoch: 6 | loss=13.19295883178711 | Batch Time=3.90625 +GPU:2 | Epoch: 6 | loss=13.262025833129883 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.381170272827148 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.265437126159668 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=13.288301467895508 | Batch Time=3.125 +GPU:2 | Epoch: 6 | loss=13.30705451965332 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.084802627563477 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.019176483154297 | Batch Time=7.03125 +GPU:2 | Epoch: 6 | loss=13.251062393188477 | Batch Time=2.734375 +GPU:2 | Epoch: 6 | loss=13.176498413085938 | Batch Time=7.03125 +GPU:2 | Epoch: 6 | loss=13.155384063720703 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.290838241577148 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.045884132385254 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.201519012451172 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.068794250488281 | Batch Time=7.8125 +GPU:2 | Epoch: 6 | loss=13.284076690673828 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.241316795349121 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.138166427612305 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.186922073364258 | Batch Time=2.734375 +GPU:2 | Epoch: 6 | loss=13.202263832092285 | Batch Time=3.125 +GPU:2 | Epoch: 6 | loss=13.188871383666992 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=13.259784698486328 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.120311737060547 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=12.971786499023438 | Batch Time=8.59375 +GPU:2 | Epoch: 6 | loss=13.064751625061035 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.319809913635254 | Batch Time=3.125 +GPU:2 | Epoch: 6 | loss=13.144170761108398 | Batch Time=7.03125 +GPU:2 | Epoch: 6 | loss=13.190763473510742 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.149038314819336 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.090608596801758 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.095687866210938 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.146676063537598 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.1396484375 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=13.128334999084473 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.24077320098877 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=13.195432662963867 | Batch Time=3.125 +GPU:2 | Epoch: 6 | loss=13.14439582824707 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=12.990950584411621 | Batch Time=8.984375 +GPU:2 | Epoch: 6 | loss=13.072517395019531 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=13.085237503051758 | Batch Time=6.640625 +GPU:2 | Epoch: 6 | loss=13.2013521194458 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.14791202545166 | Batch Time=6.640625 +GPU:2 | Epoch: 6 | loss=13.198455810546875 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=13.081893920898438 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.371200561523438 | Batch Time=2.734375 +GPU:2 | Epoch: 6 | loss=12.956415176391602 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.075984001159668 | Batch Time=7.03125 +GPU:2 | Epoch: 6 | loss=13.263299942016602 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=13.120306968688965 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=12.975269317626953 | Batch Time=7.03125 +GPU:2 | Epoch: 6 | loss=13.214441299438477 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=13.064436912536621 | Batch Time=3.90625 +GPU:2 | Epoch: 6 | loss=13.204906463623047 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=13.118843078613281 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.205179214477539 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.100379943847656 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.212547302246094 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=13.177875518798828 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.079532623291016 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.128974914550781 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.284473419189453 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=12.98489761352539 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.063863754272461 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.136087417602539 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.017881393432617 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=12.995357513427734 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.164770126342773 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.260557174682617 | Batch Time=3.90625 +GPU:2 | Epoch: 6 | loss=12.979284286499023 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.27821159362793 | Batch Time=3.90625 +GPU:2 | Epoch: 6 | loss=13.053508758544922 | Batch Time=3.125 +GPU:2 | Epoch: 6 | loss=13.12762451171875 | Batch Time=3.90625 +GPU:2 | Epoch: 6 | loss=12.949601173400879 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=13.040975570678711 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=13.104783058166504 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=13.10877799987793 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.037089347839355 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.234054565429688 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.244499206542969 | Batch Time=3.90625 +GPU:2 | Epoch: 6 | loss=13.119415283203125 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=12.952421188354492 | Batch Time=7.421875 +GPU:2 | Epoch: 6 | loss=13.268875122070312 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.10667896270752 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=12.920973777770996 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=13.009140014648438 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.052678108215332 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.095054626464844 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=13.089033126831055 | Batch Time=3.125 +GPU:2 | Epoch: 6 | loss=13.00311279296875 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.093687057495117 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.136717796325684 | Batch Time=3.90625 +GPU:2 | Epoch: 6 | loss=12.943258285522461 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.009719848632812 | Batch Time=3.515625 +GPU:2 | Epoch: 6 | loss=13.014045715332031 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.072576522827148 | Batch Time=5.46875 +GPU:2 | Epoch: 6 | loss=13.069180488586426 | Batch Time=8.984375 +GPU:2 | Epoch: 6 | loss=12.999792098999023 | Batch Time=7.03125 +GPU:2 | Epoch: 6 | loss=12.939607620239258 | Batch Time=6.640625 +GPU:2 | Epoch: 6 | loss=13.032246589660645 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=12.746116638183594 | Batch Time=6.640625 +GPU:2 | Epoch: 6 | loss=13.09373664855957 | Batch Time=5.078125 +GPU:2 | Epoch: 6 | loss=13.062088012695312 | Batch Time=3.90625 +GPU:2 | Epoch: 6 | loss=12.890985488891602 | Batch Time=7.421875 +GPU:2 | Epoch: 6 | loss=12.997756004333496 | Batch Time=6.640625 +GPU:2 | Epoch: 6 | loss=13.177529335021973 | Batch Time=2.734375 +GPU:2 | Epoch: 6 | loss=13.07591438293457 | Batch Time=7.03125 +GPU:2 | Epoch: 6 | loss=13.176461219787598 | Batch Time=3.125 +GPU:2 | Epoch: 6 | loss=12.757963180541992 | Batch Time=7.421875 +GPU:2 | Epoch: 6 | loss=12.896007537841797 | Batch Time=8.984375 +GPU:2 | Epoch: 6 | loss=12.994882583618164 | Batch Time=6.640625 +GPU:2 | Epoch: 6 | loss=13.026613235473633 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=13.18484115600586 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=12.980850219726562 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=12.88909912109375 | Batch Time=7.421875 +GPU:2 | Epoch: 6 | loss=13.083330154418945 | Batch Time=4.296875 +GPU:2 | Epoch: 6 | loss=12.936056137084961 | Batch Time=5.859375 +GPU:2 | Epoch: 6 | loss=12.8623628616333 | Batch Time=4.6875 +GPU:2 | Epoch: 6 | loss=12.917173385620117 | Batch Time=7.421875 +GPU:2 | Epoch: 6 | loss=12.943574905395508 | Batch Time=6.25 +GPU:2 | Epoch: 6 | loss=12.930801391601562 | Batch Time=8.203125 +GPU:2 | Epoch: 6 | loss=12.878416061401367 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.372787475585938 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=13.196535110473633 | Batch Time=7.03125 +GPU:0 | Epoch: 6 | loss=13.285016059875488 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.366905212402344 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.283492088317871 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.197098731994629 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.232583999633789 | Batch Time=8.203125 +GPU:0 | Epoch: 6 | loss=13.26096248626709 | Batch Time=1.953125 +GPU:0 | Epoch: 6 | loss=13.334882736206055 | Batch Time=2.34375 +GPU:0 | Epoch: 6 | loss=13.150934219360352 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.207762718200684 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.118915557861328 | Batch Time=8.203125 +GPU:0 | Epoch: 6 | loss=13.069047927856445 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.315570831298828 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=13.16944694519043 | Batch Time=3.125 +GPU:0 | Epoch: 6 | loss=13.182861328125 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.401230812072754 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.12553596496582 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.148704528808594 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.238727569580078 | Batch Time=3.515625 +GPU:0 | Epoch: 6 | loss=13.256585121154785 | Batch Time=2.34375 +GPU:0 | Epoch: 6 | loss=13.21623420715332 | Batch Time=3.125 +GPU:0 | Epoch: 6 | loss=13.334688186645508 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.220076560974121 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.238718032836914 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.090816497802734 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.1556396484375 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.117267608642578 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.022040367126465 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.04247760772705 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.03432846069336 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.259809494018555 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=13.120576858520508 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=12.979352951049805 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.211194038391113 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.197721481323242 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.285091400146484 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.145522117614746 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.081913948059082 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.20747184753418 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=12.961902618408203 | Batch Time=7.8125 +GPU:0 | Epoch: 6 | loss=13.143942832946777 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.295337677001953 | Batch Time=2.34375 +GPU:0 | Epoch: 6 | loss=13.051300048828125 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.094158172607422 | Batch Time=5.078125 +GPU:0 | Epoch: 6 | loss=13.259175300598145 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.168697357177734 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=12.999795913696289 | Batch Time=7.421875 +GPU:0 | Epoch: 6 | loss=13.354328155517578 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.278898239135742 | Batch Time=3.515625 +GPU:0 | Epoch: 6 | loss=13.100937843322754 | Batch Time=3.515625 +GPU:0 | Epoch: 6 | loss=13.095956802368164 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.240934371948242 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.128206253051758 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.037662506103516 | Batch Time=8.203125 +GPU:0 | Epoch: 6 | loss=13.120728492736816 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.116718292236328 | Batch Time=5.078125 +GPU:0 | Epoch: 6 | loss=12.969680786132812 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.024991989135742 | Batch Time=5.078125 +GPU:0 | Epoch: 6 | loss=13.1007719039917 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.10104751586914 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.083694458007812 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=12.980830192565918 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.053061485290527 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.188329696655273 | Batch Time=5.078125 +GPU:0 | Epoch: 6 | loss=13.057205200195312 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.288022994995117 | Batch Time=3.125 +GPU:0 | Epoch: 6 | loss=13.020299911499023 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.08224105834961 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.13003921508789 | Batch Time=1.953125 +GPU:0 | Epoch: 6 | loss=13.255121231079102 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.149526596069336 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.126108169555664 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=13.101888656616211 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.032764434814453 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=12.964876174926758 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=12.961956977844238 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.107622146606445 | Batch Time=5.078125 +GPU:0 | Epoch: 6 | loss=13.068079948425293 | Batch Time=7.421875 +GPU:0 | Epoch: 6 | loss=12.999540328979492 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.148308753967285 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.024932861328125 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.044021606445312 | Batch Time=5.078125 +GPU:0 | Epoch: 6 | loss=13.351070404052734 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.091105461120605 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=13.009515762329102 | Batch Time=5.078125 +GPU:0 | Epoch: 6 | loss=13.129161834716797 | Batch Time=3.515625 +GPU:0 | Epoch: 6 | loss=13.060029029846191 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=13.185169219970703 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.027390480041504 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.106285095214844 | Batch Time=6.25 +GPU:0 | Epoch: 6 | loss=13.115102767944336 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=12.86377239227295 | Batch Time=7.8125 +GPU:0 | Epoch: 6 | loss=13.043695449829102 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.011628150939941 | Batch Time=6.25 +GPU:0 | Epoch: 6 | loss=13.04829216003418 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.031488418579102 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=12.951045989990234 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.0438232421875 | Batch Time=6.25 +GPU:0 | Epoch: 6 | loss=12.955410957336426 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.000993728637695 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=13.29830551147461 | Batch Time=3.125 +GPU:0 | Epoch: 6 | loss=13.083306312561035 | Batch Time=3.515625 +GPU:0 | Epoch: 6 | loss=12.757131576538086 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=12.934959411621094 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=13.040811538696289 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.069807052612305 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.143403053283691 | Batch Time=3.515625 +GPU:0 | Epoch: 6 | loss=13.003727912902832 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=12.975154876708984 | Batch Time=7.8125 +GPU:0 | Epoch: 6 | loss=13.053845405578613 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=12.978340148925781 | Batch Time=5.078125 +GPU:0 | Epoch: 6 | loss=12.97249698638916 | Batch Time=7.8125 +GPU:0 | Epoch: 6 | loss=13.20655632019043 | Batch Time=2.34375 +GPU:0 | Epoch: 6 | loss=13.024747848510742 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=13.070907592773438 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=12.835125923156738 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=13.004610061645508 | Batch Time=6.640625 +GPU:0 | Epoch: 6 | loss=12.954011917114258 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=12.86288070678711 | Batch Time=7.03125 +GPU:0 | Epoch: 6 | loss=12.949204444885254 | Batch Time=7.421875 +GPU:0 | Epoch: 6 | loss=13.043257713317871 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=12.984020233154297 | Batch Time=5.859375 +GPU:0 | Epoch: 6 | loss=12.932153701782227 | Batch Time=3.90625 +GPU:0 | Epoch: 6 | loss=12.958755493164062 | Batch Time=5.078125 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820012544, percent=27.9, used=63837609984, free=26464448512, active=220314480640, inactive=6166949888, buffers=443924480, cached=166822100992, shared=5660409856, slab=3223080960) +AFTER TRAIN LOOP: Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +STARTING TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +BEFORE TRAIN LOOP: Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820012544, percent=27.9, used=63837609984, free=26464448512, active=220314480640, inactive=6166949888, buffers=443924480, cached=166822100992, shared=5660409856, slab=3223080960) +AFTER TRAIN LOOP: Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +STARTING TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +BEFORE TRAIN LOOP: Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820012544, percent=27.9, used=63837609984, free=26464448512, active=220314480640, inactive=6166949888, buffers=443924480, cached=166822100992, shared=5660409856, slab=3223080960) +AFTER TRAIN LOOP: Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +STARTING TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +BEFORE TRAIN LOOP: Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820012544, percent=27.9, used=63837609984, free=26464448512, active=220314480640, inactive=6166949888, buffers=443924480, cached=166822100992, shared=5660409856, slab=3223080960) +AFTER TRAIN LOOP: Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +BEFORE VAL LOOP: GPU: 0 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=185820827648, percent=27.9, used=63837016064, free=26465263616, active=220314054656, inactive=6166851584, buffers=443924480, cached=166821879808, shared=5660164096, slab=3223040000) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 6 | loss=4.564809322357178 | Batch Time=15.625 +GPU:0 | Epoch: 6 | loss=5.828030109405518 | Batch Time=1.5625 +GPU:0 | Epoch: 6 | loss=5.501135349273682 | Batch Time=10.546875 +GPU:0 | Epoch: 6 | loss=5.741267681121826 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=6.0634284019470215 | Batch Time=0.0 +GPU:0 | Epoch: 6 | loss=5.750077724456787 | Batch Time=3.125 +GPU:0 | Epoch: 6 | loss=5.888983726501465 | Batch Time=0.78125 +GPU:0 | Epoch: 6 | loss=6.0085039138793945 | Batch Time=1.171875 +GPU:0 | Epoch: 6 | loss=5.94628381729126 | Batch Time=0.390625 +GPU:0 | Epoch: 6 | loss=6.423783779144287 | Batch Time=0.0 +GPU:0 | Epoch: 6 | loss=5.872167587280273 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=5.4282965660095215 | Batch Time=7.8125 +GPU:0 | Epoch: 6 | loss=6.084885597229004 | Batch Time=1.5625 +GPU:0 | Epoch: 6 | loss=5.677171230316162 | Batch Time=8.59375 +GPU:0 | Epoch: 6 | loss=5.470853328704834 | Batch Time=2.34375 +GPU:0 | Epoch: 6 | loss=5.690694332122803 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=5.590753078460693 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=5.06067419052124 | Batch Time=10.9375 +GPU:0 | Epoch: 6 | loss=5.005996227264404 | Batch Time=8.984375 +GPU:0 | Epoch: 6 | loss=4.764954566955566 | Batch Time=14.453125 +Model top1 Accuracy: 5.388 +Acc before rounding: 5.388 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 6 | loss=4.564809322357178 | Batch Time=15.625 +GPU:0 | Epoch: 6 | loss=5.828030109405518 | Batch Time=1.5625 +GPU:0 | Epoch: 6 | loss=5.501135349273682 | Batch Time=10.546875 +GPU:0 | Epoch: 6 | loss=5.741267681121826 | Batch Time=4.296875 +GPU:0 | Epoch: 6 | loss=6.0634284019470215 | Batch Time=0.0 +GPU:0 | Epoch: 6 | loss=5.750077724456787 | Batch Time=3.125 +GPU:0 | Epoch: 6 | loss=5.888983726501465 | Batch Time=0.78125 +GPU:0 | Epoch: 6 | loss=6.0085039138793945 | Batch Time=1.171875 +GPU:0 | Epoch: 6 | loss=5.94628381729126 | Batch Time=0.390625 +GPU:0 | Epoch: 6 | loss=6.423783779144287 | Batch Time=0.0 +GPU:0 | Epoch: 6 | loss=5.872167587280273 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=5.4282965660095215 | Batch Time=7.8125 +GPU:0 | Epoch: 6 | loss=6.084885597229004 | Batch Time=1.5625 +GPU:0 | Epoch: 6 | loss=5.677171230316162 | Batch Time=8.59375 +GPU:0 | Epoch: 6 | loss=5.470853328704834 | Batch Time=2.34375 +GPU:0 | Epoch: 6 | loss=5.690694332122803 | Batch Time=5.46875 +GPU:0 | Epoch: 6 | loss=5.590753078460693 | Batch Time=2.734375 +GPU:0 | Epoch: 6 | loss=5.06067419052124 | Batch Time=10.9375 +GPU:0 | Epoch: 6 | loss=5.005996227264404 | Batch Time=8.984375 +GPU:0 | Epoch: 6 | loss=4.764954566955566 | Batch Time=14.453125 +Model top1 Accuracy: 5.388 +Acc after rounding: 5.388 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 6 | loss=5.724629878997803 | Batch Time=4.6875 +GPU:0 | Epoch: 6 | loss=5.7333455085754395 | Batch Time=8.59375 +GPU:0 | Epoch: 6 | loss=5.670739650726318 | Batch Time=6.25 +GPU:0 | Epoch: 6 | loss=5.767742156982422 | Batch Time=4.296875 +Model top1 Accuracy: 4.59 +Validation Acc after rounding: 4.59 +AFTER VAL LOOP: GPU: 0 | Epoch 6 | Memory Usage: svmem(total=257568083968, available=167989043200, percent=34.8, used=81668423680, free=8632020992, active=238029447168, inactive=6167699456, buffers=444092416, cached=166823546880, shared=5660557312, slab=3229835264) +Rounding model with scheme: naive +Model avg sparsity: 46.38319512047497 +GPU:0 | Epoch: 6 | Acc=5.388 | Epoch Time=15.614885715643565 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=167989043200, percent=34.8, used=81668423680, free=8632020992, active=238029443072, inactive=6167699456, buffers=444092416, cached=166823546880, shared=5660557312, slab=3229835264) +STARTING TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=167989043200, percent=34.8, used=81668423680, free=8632020992, active=238029443072, inactive=6167699456, buffers=444092416, cached=166823546880, shared=5660557312, slab=3229835264) +BEFORE TRAIN LOOP: Epoch 7 | Memory Usage: svmem(total=257568083968, available=167989043200, percent=34.8, used=81668423680, free=8632020992, active=238029443072, inactive=6167699456, buffers=444092416, cached=166823546880, shared=5660557312, slab=3229835264) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=167989043200, percent=34.8, used=81668423680, free=8632020992, active=238029443072, inactive=6167699456, buffers=444092416, cached=166823546880, shared=5660557312, slab=3229835264) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:2 | Epoch: 7 | loss=13.032397270202637 | Batch Time=4.296875 +GPU:2 | Epoch: 7 | loss=13.036640167236328 | Batch Time=3.515625 +GPU:2 | Epoch: 7 | loss=12.972138404846191 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=13.082405090332031 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.833950996398926 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.844289779663086 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=13.155942916870117 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=13.073444366455078 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.881614685058594 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=13.007707595825195 | Batch Time=8.203125 +GPU:2 | Epoch: 7 | loss=12.841597557067871 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=13.061731338500977 | Batch Time=3.90625 +GPU:2 | Epoch: 7 | loss=12.88716983795166 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.905753135681152 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.93983268737793 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.988834381103516 | Batch Time=3.90625 +GPU:2 | Epoch: 7 | loss=12.820355415344238 | Batch Time=8.203125 +GPU:2 | Epoch: 7 | loss=12.705148696899414 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=13.036585807800293 | Batch Time=6.640625 +GPU:2 | Epoch: 7 | loss=12.838897705078125 | Batch Time=4.296875 +GPU:2 | Epoch: 7 | loss=12.887197494506836 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.98302936553955 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.81248664855957 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.901914596557617 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.90330696105957 | Batch Time=6.640625 +GPU:2 | Epoch: 7 | loss=12.927995681762695 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.94139575958252 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.992988586425781 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.981704711914062 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.719392776489258 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.903139114379883 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.794317245483398 | Batch Time=8.59375 +GPU:2 | Epoch: 7 | loss=12.921660423278809 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=13.084783554077148 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.760316848754883 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.899232864379883 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.822093963623047 | Batch Time=8.59375 +GPU:2 | Epoch: 7 | loss=12.880390167236328 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.900575637817383 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.959338188171387 | Batch Time=3.90625 +GPU:2 | Epoch: 7 | loss=12.757244110107422 | Batch Time=8.984375 +GPU:2 | Epoch: 7 | loss=12.837479591369629 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.9603271484375 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.86571216583252 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.908868789672852 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.8319673538208 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.788915634155273 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.844625473022461 | Batch Time=9.765625 +GPU:2 | Epoch: 7 | loss=12.882312774658203 | Batch Time=8.203125 +GPU:2 | Epoch: 7 | loss=12.851448059082031 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.834855079650879 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.813787460327148 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.908266067504883 | Batch Time=4.296875 +GPU:2 | Epoch: 7 | loss=12.676170349121094 | Batch Time=8.59375 +GPU:2 | Epoch: 7 | loss=12.761697769165039 | Batch Time=3.90625 +GPU:2 | Epoch: 7 | loss=12.841264724731445 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.83043098449707 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.806290626525879 | Batch Time=3.90625 +GPU:2 | Epoch: 7 | loss=12.7904052734375 | Batch Time=7.8125 +GPU:2 | Epoch: 7 | loss=12.766168594360352 | Batch Time=7.8125 +GPU:2 | Epoch: 7 | loss=12.819934844970703 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.77676773071289 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.901762008666992 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.884027481079102 | Batch Time=7.421875 +GPU:2 | Epoch: 7 | loss=12.924646377563477 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.850351333618164 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.891229629516602 | Batch Time=4.296875 +GPU:2 | Epoch: 7 | loss=12.867717742919922 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.76637077331543 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.694478034973145 | Batch Time=9.375 +GPU:2 | Epoch: 7 | loss=12.837029457092285 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.82573127746582 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.773820877075195 | Batch Time=8.59375 +GPU:2 | Epoch: 7 | loss=12.692745208740234 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.823257446289062 | Batch Time=4.296875 +GPU:2 | Epoch: 7 | loss=12.754889488220215 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.638154029846191 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.780964851379395 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.793455123901367 | Batch Time=3.90625 +GPU:2 | Epoch: 7 | loss=12.880334854125977 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.966632843017578 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.77747917175293 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.76630973815918 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.718603134155273 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.699055671691895 | Batch Time=7.421875 +GPU:2 | Epoch: 7 | loss=12.64670181274414 | Batch Time=7.8125 +GPU:2 | Epoch: 7 | loss=12.82742691040039 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.777600288391113 | Batch Time=6.640625 +GPU:2 | Epoch: 7 | loss=12.753780364990234 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.911699295043945 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.7754487991333 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.795858383178711 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.825113296508789 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.73162841796875 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.669301986694336 | Batch Time=6.640625 +GPU:2 | Epoch: 7 | loss=12.864288330078125 | Batch Time=3.90625 +GPU:2 | Epoch: 7 | loss=12.749592781066895 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.914566040039062 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.667211532592773 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.704179763793945 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.613838195800781 | Batch Time=7.421875 +GPU:2 | Epoch: 7 | loss=12.7596435546875 | Batch Time=4.296875 +GPU:2 | Epoch: 7 | loss=12.626928329467773 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.699943542480469 | Batch Time=7.421875 +GPU:2 | Epoch: 7 | loss=12.729604721069336 | Batch Time=5.078125 +GPU:2 | Epoch: 7 | loss=12.699098587036133 | Batch Time=4.296875 +GPU:2 | Epoch: 7 | loss=12.683218002319336 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.635828018188477 | Batch Time=6.640625 +GPU:2 | Epoch: 7 | loss=12.733549118041992 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.78719425201416 | Batch Time=7.8125 +GPU:2 | Epoch: 7 | loss=12.682387351989746 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.636353492736816 | Batch Time=6.25 +GPU:2 | Epoch: 7 | loss=12.772911071777344 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.854612350463867 | Batch Time=3.515625 +GPU:2 | Epoch: 7 | loss=12.760278701782227 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.684361457824707 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.752687454223633 | Batch Time=5.46875 +GPU:2 | Epoch: 7 | loss=12.596818923950195 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.747848510742188 | Batch Time=5.859375 +GPU:2 | Epoch: 7 | loss=12.762014389038086 | Batch Time=6.640625 +GPU:2 | Epoch: 7 | loss=12.740273475646973 | Batch Time=4.6875 +GPU:2 | Epoch: 7 | loss=12.754256248474121 | Batch Time=8.203125 +GPU:2 | Epoch: 7 | loss=12.79123306274414 | Batch Time=7.03125 +GPU:2 | Epoch: 7 | loss=12.704166412353516 | Batch Time=6.640625 +GPU:2 | Epoch: 7 | loss=12.594358444213867 | Batch Time=8.984375 +GPU:3 | Epoch: 7 | loss=12.88532829284668 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=13.094858169555664 | Batch Time=3.125 +GPU:3 | Epoch: 7 | loss=12.956297874450684 | Batch Time=2.734375 +GPU:3 | Epoch: 7 | loss=12.759483337402344 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=13.09447956085205 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.826377868652344 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=13.114309310913086 | Batch Time=3.125 +GPU:3 | Epoch: 7 | loss=12.975872039794922 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.76571273803711 | Batch Time=7.8125 +GPU:3 | Epoch: 7 | loss=12.943046569824219 | Batch Time=3.125 +GPU:3 | Epoch: 7 | loss=12.81635570526123 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=13.087272644042969 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=13.006814002990723 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=13.025622367858887 | Batch Time=3.90625 +GPU:3 | Epoch: 7 | loss=13.006795883178711 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.874006271362305 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.944368362426758 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.827889442443848 | Batch Time=8.59375 +GPU:3 | Epoch: 7 | loss=12.839473724365234 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.946451187133789 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.973962783813477 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.955642700195312 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.897501945495605 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.83637809753418 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.970052719116211 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.946592330932617 | Batch Time=4.296875 +GPU:3 | Epoch: 7 | loss=12.796731948852539 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.980708122253418 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.86834716796875 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.878129959106445 | Batch Time=8.203125 +GPU:3 | Epoch: 7 | loss=12.943187713623047 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=13.027009963989258 | Batch Time=3.90625 +GPU:3 | Epoch: 7 | loss=12.822054862976074 | Batch Time=9.375 +GPU:3 | Epoch: 7 | loss=13.130918502807617 | Batch Time=4.296875 +GPU:3 | Epoch: 7 | loss=12.885345458984375 | Batch Time=4.296875 +GPU:3 | Epoch: 7 | loss=12.863113403320312 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.836957931518555 | Batch Time=3.90625 +GPU:3 | Epoch: 7 | loss=12.949472427368164 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.76369857788086 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.933127403259277 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.779224395751953 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=13.008506774902344 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=13.066320419311523 | Batch Time=3.90625 +GPU:3 | Epoch: 7 | loss=12.943102836608887 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.87065315246582 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.745009422302246 | Batch Time=8.59375 +GPU:3 | Epoch: 7 | loss=12.927961349487305 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.81654167175293 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.95621109008789 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.704845428466797 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.689221382141113 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.854835510253906 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.787527084350586 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.780067443847656 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.827576637268066 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.80826187133789 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.966346740722656 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.781064987182617 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.970102310180664 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.902799606323242 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.84288215637207 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.685859680175781 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.956964492797852 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.702581405639648 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.755327224731445 | Batch Time=3.90625 +GPU:3 | Epoch: 7 | loss=12.874628067016602 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.899402618408203 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.966121673583984 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.650491714477539 | Batch Time=8.203125 +GPU:3 | Epoch: 7 | loss=12.78195858001709 | Batch Time=8.59375 +GPU:3 | Epoch: 7 | loss=12.765650749206543 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.746749877929688 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.94914436340332 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.910667419433594 | Batch Time=2.34375 +GPU:3 | Epoch: 7 | loss=12.7587890625 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.779232025146484 | Batch Time=8.203125 +GPU:3 | Epoch: 7 | loss=12.882843971252441 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.836801528930664 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.619916915893555 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.741277694702148 | Batch Time=7.8125 +GPU:3 | Epoch: 7 | loss=12.850652694702148 | Batch Time=8.59375 +GPU:3 | Epoch: 7 | loss=12.91798210144043 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.783895492553711 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.905801773071289 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.66330623626709 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.714065551757812 | Batch Time=8.984375 +GPU:3 | Epoch: 7 | loss=12.923837661743164 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.788949012756348 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.746788024902344 | Batch Time=8.59375 +GPU:3 | Epoch: 7 | loss=12.689800262451172 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.823840141296387 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.93040657043457 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.804998397827148 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.829407691955566 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.763528823852539 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.811309814453125 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.685895919799805 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.806022644042969 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.733144760131836 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.668493270874023 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.742544174194336 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.708404541015625 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.781023025512695 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.732873916625977 | Batch Time=7.8125 +GPU:3 | Epoch: 7 | loss=12.725767135620117 | Batch Time=4.296875 +GPU:3 | Epoch: 7 | loss=12.75967788696289 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.638105392456055 | Batch Time=6.25 +GPU:3 | Epoch: 7 | loss=12.582172393798828 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.821304321289062 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.732735633850098 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.704791069030762 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.673027038574219 | Batch Time=7.421875 +GPU:3 | Epoch: 7 | loss=12.755166053771973 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.76253890991211 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.912851333618164 | Batch Time=8.203125 +GPU:3 | Epoch: 7 | loss=12.748540878295898 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.802997589111328 | Batch Time=5.46875 +GPU:3 | Epoch: 7 | loss=12.610553741455078 | Batch Time=8.59375 +GPU:3 | Epoch: 7 | loss=12.649942398071289 | Batch Time=5.859375 +GPU:3 | Epoch: 7 | loss=12.85329818725586 | Batch Time=4.6875 +GPU:3 | Epoch: 7 | loss=12.623128890991211 | Batch Time=7.03125 +GPU:3 | Epoch: 7 | loss=12.731526374816895 | Batch Time=4.296875 +GPU:3 | Epoch: 7 | loss=12.831830978393555 | Batch Time=6.640625 +GPU:3 | Epoch: 7 | loss=12.657479286193848 | Batch Time=5.078125 +GPU:3 | Epoch: 7 | loss=12.662639617919922 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.955665588378906 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=12.86921501159668 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.97238540649414 | Batch Time=2.734375 +GPU:1 | Epoch: 7 | loss=13.102794647216797 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.968286514282227 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.897134780883789 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.958015441894531 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=12.824377059936523 | Batch Time=9.375 +GPU:1 | Epoch: 7 | loss=12.91903305053711 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=12.970842361450195 | Batch Time=3.90625 +GPU:1 | Epoch: 7 | loss=12.957870483398438 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.90084457397461 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=13.013970375061035 | Batch Time=7.421875 +GPU:1 | Epoch: 7 | loss=12.967070579528809 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.978265762329102 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=13.010250091552734 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.878271102905273 | Batch Time=3.90625 +GPU:1 | Epoch: 7 | loss=12.941317558288574 | Batch Time=7.421875 +GPU:1 | Epoch: 7 | loss=12.915667533874512 | Batch Time=8.203125 +GPU:1 | Epoch: 7 | loss=12.878284454345703 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.918210983276367 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=13.009098052978516 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=13.04338550567627 | Batch Time=7.8125 +GPU:1 | Epoch: 7 | loss=12.814619064331055 | Batch Time=6.640625 +GPU:1 | Epoch: 7 | loss=12.674539566040039 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.968392372131348 | Batch Time=7.8125 +GPU:1 | Epoch: 7 | loss=13.00536823272705 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=12.891728401184082 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.904708862304688 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.687484741210938 | Batch Time=7.8125 +GPU:1 | Epoch: 7 | loss=12.915828704833984 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.9315185546875 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.983031272888184 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=12.747772216796875 | Batch Time=6.640625 +GPU:1 | Epoch: 7 | loss=12.89341926574707 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.973798751831055 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=12.932369232177734 | Batch Time=7.421875 +GPU:1 | Epoch: 7 | loss=12.997171401977539 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=13.043020248413086 | Batch Time=3.90625 +GPU:1 | Epoch: 7 | loss=12.996256828308105 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.826242446899414 | Batch Time=3.90625 +GPU:1 | Epoch: 7 | loss=13.05583667755127 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.80488395690918 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.93896770477295 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.888643264770508 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=12.786763191223145 | Batch Time=8.203125 +GPU:1 | Epoch: 7 | loss=12.898481369018555 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.893186569213867 | Batch Time=3.90625 +GPU:1 | Epoch: 7 | loss=12.859746932983398 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=12.973529815673828 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.797101020812988 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=12.842243194580078 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.993341445922852 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.880636215209961 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.817594528198242 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=13.072572708129883 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.932472229003906 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.672425270080566 | Batch Time=7.8125 +GPU:1 | Epoch: 7 | loss=12.801704406738281 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.919973373413086 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=12.630849838256836 | Batch Time=8.59375 +GPU:1 | Epoch: 7 | loss=12.865935325622559 | Batch Time=2.734375 +GPU:1 | Epoch: 7 | loss=12.812213897705078 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.85508918762207 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.813066482543945 | Batch Time=7.8125 +GPU:1 | Epoch: 7 | loss=12.808967590332031 | Batch Time=7.421875 +GPU:1 | Epoch: 7 | loss=12.813579559326172 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.702703475952148 | Batch Time=8.203125 +GPU:1 | Epoch: 7 | loss=12.853931427001953 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.958831787109375 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=12.587786674499512 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.908055305480957 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=13.021936416625977 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.84908676147461 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.77520751953125 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.802200317382812 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.782267570495605 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.547768592834473 | Batch Time=6.640625 +GPU:1 | Epoch: 7 | loss=12.931509017944336 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=12.734111785888672 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.655269622802734 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.846891403198242 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.768598556518555 | Batch Time=8.203125 +GPU:1 | Epoch: 7 | loss=12.684879302978516 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.945036888122559 | Batch Time=3.515625 +GPU:1 | Epoch: 7 | loss=12.737052917480469 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.75851058959961 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=12.893288612365723 | Batch Time=3.515625 +GPU:1 | Epoch: 7 | loss=12.783208847045898 | Batch Time=6.640625 +GPU:1 | Epoch: 7 | loss=12.87175178527832 | Batch Time=8.203125 +GPU:1 | Epoch: 7 | loss=12.769734382629395 | Batch Time=8.59375 +GPU:1 | Epoch: 7 | loss=12.747810363769531 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.89322280883789 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.852405548095703 | Batch Time=4.6875 +GPU:1 | Epoch: 7 | loss=12.819601058959961 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.741626739501953 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.751173973083496 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.62152099609375 | Batch Time=8.203125 +GPU:1 | Epoch: 7 | loss=12.876184463500977 | Batch Time=3.125 +GPU:1 | Epoch: 7 | loss=12.765621185302734 | Batch Time=5.859375 +GPU:1 | Epoch: 7 | loss=12.705337524414062 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.765517234802246 | Batch Time=4.296875 +GPU:1 | Epoch: 7 | loss=12.787281036376953 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.913318634033203 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.821889877319336 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.795206069946289 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.740676879882812 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.669898986816406 | Batch Time=8.984375 +GPU:1 | Epoch: 7 | loss=12.777584075927734 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.686969757080078 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.855448722839355 | Batch Time=7.8125 +GPU:1 | Epoch: 7 | loss=12.717632293701172 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.619091987609863 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.809942245483398 | Batch Time=7.421875 +GPU:1 | Epoch: 7 | loss=12.733072280883789 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.718598365783691 | Batch Time=6.25 +GPU:1 | Epoch: 7 | loss=12.596918106079102 | Batch Time=7.03125 +GPU:1 | Epoch: 7 | loss=12.769123077392578 | Batch Time=7.8125 +GPU:1 | Epoch: 7 | loss=12.621219635009766 | Batch Time=5.46875 +GPU:1 | Epoch: 7 | loss=12.794109344482422 | Batch Time=2.734375 +GPU:1 | Epoch: 7 | loss=12.59955883026123 | Batch Time=7.421875 +GPU:1 | Epoch: 7 | loss=12.820467948913574 | Batch Time=7.421875 +GPU:1 | Epoch: 7 | loss=12.69472885131836 | Batch Time=5.078125 +GPU:1 | Epoch: 7 | loss=12.722145080566406 | Batch Time=6.640625 +GPU:1 | Epoch: 7 | loss=12.741384506225586 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.879703521728516 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.892925262451172 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.951269149780273 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=13.029529571533203 | Batch Time=5.46875 +GPU:0 | Epoch: 7 | loss=13.105435371398926 | Batch Time=5.46875 +GPU:0 | Epoch: 7 | loss=12.995536804199219 | Batch Time=8.203125 +GPU:0 | Epoch: 7 | loss=12.974813461303711 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=13.0096435546875 | Batch Time=5.46875 +GPU:0 | Epoch: 7 | loss=12.758668899536133 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=13.091863632202148 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.853028297424316 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.844751358032227 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.91381549835205 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.993746757507324 | Batch Time=5.46875 +GPU:0 | Epoch: 7 | loss=12.723609924316406 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.92652702331543 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=12.728463172912598 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.9281644821167 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.831963539123535 | Batch Time=7.03125 +GPU:0 | Epoch: 7 | loss=12.875085830688477 | Batch Time=10.15625 +GPU:0 | Epoch: 7 | loss=12.859222412109375 | Batch Time=5.46875 +GPU:0 | Epoch: 7 | loss=12.818744659423828 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=12.952672004699707 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.933244705200195 | Batch Time=2.734375 +GPU:0 | Epoch: 7 | loss=13.046188354492188 | Batch Time=3.515625 +GPU:0 | Epoch: 7 | loss=12.993494033813477 | Batch Time=3.515625 +GPU:0 | Epoch: 7 | loss=12.968583106994629 | Batch Time=3.125 +GPU:0 | Epoch: 7 | loss=12.89387035369873 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.910717010498047 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.816900253295898 | Batch Time=8.984375 +GPU:0 | Epoch: 7 | loss=12.8515625 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.96613883972168 | Batch Time=7.03125 +GPU:0 | Epoch: 7 | loss=12.835196495056152 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.782697677612305 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=12.783906936645508 | Batch Time=5.46875 +GPU:0 | Epoch: 7 | loss=12.849288940429688 | Batch Time=7.03125 +GPU:0 | Epoch: 7 | loss=12.92988395690918 | Batch Time=3.90625 +GPU:0 | Epoch: 7 | loss=12.904447555541992 | Batch Time=7.03125 +GPU:0 | Epoch: 7 | loss=12.806421279907227 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.837023735046387 | Batch Time=3.90625 +GPU:0 | Epoch: 7 | loss=12.805959701538086 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.938445091247559 | Batch Time=3.90625 +GPU:0 | Epoch: 7 | loss=12.822437286376953 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.944046974182129 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.912656784057617 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.7788667678833 | Batch Time=8.203125 +GPU:0 | Epoch: 7 | loss=13.016721725463867 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.943839073181152 | Batch Time=7.03125 +GPU:0 | Epoch: 7 | loss=12.738811492919922 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=12.855682373046875 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.772425651550293 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.968818664550781 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.845788955688477 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.721391677856445 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.839417457580566 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.912965774536133 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.829349517822266 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=12.848198890686035 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.785770416259766 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.834964752197266 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.980690002441406 | Batch Time=2.34375 +GPU:0 | Epoch: 7 | loss=12.938474655151367 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.985824584960938 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.761926651000977 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.77415657043457 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=12.80948257446289 | Batch Time=8.203125 +GPU:0 | Epoch: 7 | loss=12.743593215942383 | Batch Time=7.03125 +GPU:0 | Epoch: 7 | loss=12.882369995117188 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.900493621826172 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.699487686157227 | Batch Time=8.984375 +GPU:0 | Epoch: 7 | loss=12.837374687194824 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.744525909423828 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.779428482055664 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.878986358642578 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.690998077392578 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.930366516113281 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.740504264831543 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.833052635192871 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.755147933959961 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.971097946166992 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.906147003173828 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=12.808927536010742 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.723236083984375 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.793458938598633 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.829413414001465 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.89887809753418 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.816038131713867 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.783493995666504 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=12.703243255615234 | Batch Time=5.46875 +GPU:0 | Epoch: 7 | loss=12.768606185913086 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.76232624053955 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.675216674804688 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.874822616577148 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.890487670898438 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.692327499389648 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=12.80184555053711 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=12.567362785339355 | Batch Time=9.765625 +GPU:0 | Epoch: 7 | loss=12.702598571777344 | Batch Time=8.59375 +GPU:0 | Epoch: 7 | loss=12.995651245117188 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.80229377746582 | Batch Time=3.90625 +GPU:0 | Epoch: 7 | loss=12.446722030639648 | Batch Time=10.9375 +GPU:0 | Epoch: 7 | loss=12.810558319091797 | Batch Time=3.90625 +GPU:0 | Epoch: 7 | loss=12.821430206298828 | Batch Time=5.859375 +GPU:0 | Epoch: 7 | loss=12.683788299560547 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.761430740356445 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=12.532150268554688 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.613426208496094 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.705831527709961 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.774797439575195 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=12.795954704284668 | Batch Time=5.46875 +GPU:0 | Epoch: 7 | loss=12.736414909362793 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=12.774993896484375 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.775078773498535 | Batch Time=4.6875 +GPU:0 | Epoch: 7 | loss=12.507696151733398 | Batch Time=12.109375 +GPU:0 | Epoch: 7 | loss=12.676397323608398 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.775740623474121 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.659934997558594 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.705747604370117 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.628910064697266 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=12.711355209350586 | Batch Time=8.203125 +GPU:0 | Epoch: 7 | loss=12.802964210510254 | Batch Time=5.078125 +GPU:0 | Epoch: 7 | loss=12.8226900100708 | Batch Time=4.296875 +GPU:0 | Epoch: 7 | loss=12.821063995361328 | Batch Time=6.640625 +GPU:0 | Epoch: 7 | loss=12.781440734863281 | Batch Time=3.515625 +GPU:0 | Epoch: 7 | loss=12.893131256103516 | Batch Time=5.078125 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=178275008512, percent=30.8, used=71382499328, free=18913599488, active=227833413632, inactive=6121521152, buffers=444100608, cached=166827884544, shared=5660524544, slab=3229065216) +AFTER TRAIN LOOP: Epoch 7 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +STARTING TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +BEFORE TRAIN LOOP: Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=178275008512, percent=30.8, used=71382499328, free=18913599488, active=227833413632, inactive=6121521152, buffers=444100608, cached=166827884544, shared=5660524544, slab=3229065216) +AFTER TRAIN LOOP: Epoch 7 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +BEFORE VAL LOOP: GPU: 0 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=178275008512, percent=30.8, used=71382499328, free=18913599488, active=227833413632, inactive=6121521152, buffers=444100608, cached=166827884544, shared=5660524544, slab=3229065216) +AFTER TRAIN LOOP: Epoch 7 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +STARTING TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +BEFORE TRAIN LOOP: Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=178275008512, percent=30.8, used=71382499328, free=18913599488, active=227833413632, inactive=6121521152, buffers=444100608, cached=166827884544, shared=5660524544, slab=3229065216) +AFTER TRAIN LOOP: Epoch 7 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +STARTING TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +BEFORE TRAIN LOOP: Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=178276102144, percent=30.8, used=71381630976, free=18914689024, active=227833131008, inactive=6121422848, buffers=444100608, cached=166827663360, shared=5660278784, slab=3229036544) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 7 | loss=4.625619888305664 | Batch Time=10.546875 +GPU:0 | Epoch: 7 | loss=5.748968124389648 | Batch Time=3.515625 +GPU:0 | Epoch: 7 | loss=5.288938999176025 | Batch Time=9.765625 +GPU:0 | Epoch: 7 | loss=5.455835342407227 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=5.585315704345703 | Batch Time=0.0 +GPU:0 | Epoch: 7 | loss=5.036837577819824 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=5.904886245727539 | Batch Time=1.953125 +GPU:0 | Epoch: 7 | loss=5.8304057121276855 | Batch Time=0.78125 +GPU:0 | Epoch: 7 | loss=5.763697624206543 | Batch Time=0.0 +GPU:0 | Epoch: 7 | loss=6.283493995666504 | Batch Time=1.171875 +GPU:0 | Epoch: 7 | loss=6.026639938354492 | Batch Time=1.5625 +GPU:0 | Epoch: 7 | loss=5.49788236618042 | Batch Time=2.34375 +GPU:0 | Epoch: 7 | loss=5.777815818786621 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=5.378398418426514 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=5.510392189025879 | Batch Time=1.953125 +GPU:0 | Epoch: 7 | loss=5.622492790222168 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=5.234197616577148 | Batch Time=8.203125 +GPU:0 | Epoch: 7 | loss=4.849663257598877 | Batch Time=14.0625 +GPU:0 | Epoch: 7 | loss=5.58859920501709 | Batch Time=8.984375 +GPU:0 | Epoch: 7 | loss=4.393923282623291 | Batch Time=19.53125 +Model top1 Accuracy: 5.874 +Acc before rounding: 5.874 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 7 | loss=4.625619888305664 | Batch Time=10.546875 +GPU:0 | Epoch: 7 | loss=5.748968124389648 | Batch Time=3.515625 +GPU:0 | Epoch: 7 | loss=5.288938999176025 | Batch Time=9.765625 +GPU:0 | Epoch: 7 | loss=5.455835342407227 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=5.585315704345703 | Batch Time=0.0 +GPU:0 | Epoch: 7 | loss=5.036837577819824 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=5.904886245727539 | Batch Time=1.953125 +GPU:0 | Epoch: 7 | loss=5.8304057121276855 | Batch Time=0.78125 +GPU:0 | Epoch: 7 | loss=5.763697624206543 | Batch Time=0.0 +GPU:0 | Epoch: 7 | loss=6.283493995666504 | Batch Time=1.171875 +GPU:0 | Epoch: 7 | loss=6.026639938354492 | Batch Time=1.5625 +GPU:0 | Epoch: 7 | loss=5.49788236618042 | Batch Time=2.34375 +GPU:0 | Epoch: 7 | loss=5.777815818786621 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=5.378398418426514 | Batch Time=7.8125 +GPU:0 | Epoch: 7 | loss=5.510392189025879 | Batch Time=1.953125 +GPU:0 | Epoch: 7 | loss=5.622492790222168 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=5.234197616577148 | Batch Time=8.203125 +GPU:0 | Epoch: 7 | loss=4.849663257598877 | Batch Time=14.0625 +GPU:0 | Epoch: 7 | loss=5.58859920501709 | Batch Time=8.984375 +GPU:0 | Epoch: 7 | loss=4.393923282623291 | Batch Time=19.53125 +Model top1 Accuracy: 5.874 +Acc after rounding: 5.874 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 7 | loss=5.562169075012207 | Batch Time=7.421875 +GPU:0 | Epoch: 7 | loss=5.720274448394775 | Batch Time=7.03125 +GPU:0 | Epoch: 7 | loss=5.512559413909912 | Batch Time=6.25 +GPU:0 | Epoch: 7 | loss=5.672666549682617 | Batch Time=7.421875 +Model top1 Accuracy: 5.77 +Validation Acc after rounding: 5.77 +AFTER VAL LOOP: GPU: 0 | Epoch 7 | Memory Usage: svmem(total=257568083968, available=160513347584, percent=37.7, used=89143517184, free=5592588288, active=221338132480, inactive=25846419456, buffers=362004480, cached=162469974016, shared=5660663808, slab=3233935360) +Rounding model with scheme: naive +Model avg sparsity: 45.85025364634212 +GPU:0 | Epoch: 7 | Acc=5.874 | Epoch Time=15.827387603123983 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=160513368064, percent=37.7, used=89143582720, free=5592604672, active=221338116096, inactive=25846386688, buffers=362004480, cached=162469892096, shared=5660581888, slab=3233927168) +STARTING TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=160513368064, percent=37.7, used=89143582720, free=5592604672, active=221338116096, inactive=25846386688, buffers=362004480, cached=162469892096, shared=5660581888, slab=3233927168) +BEFORE TRAIN LOOP: Epoch 8 | Memory Usage: svmem(total=257568083968, available=160513368064, percent=37.7, used=89143582720, free=5592604672, active=221338116096, inactive=25846386688, buffers=362004480, cached=162469892096, shared=5660581888, slab=3233927168) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=160513368064, percent=37.7, used=89143582720, free=5592604672, active=221338116096, inactive=25846386688, buffers=362004480, cached=162469892096, shared=5660581888, slab=3233927168) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 8 | loss=12.787178039550781 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.797327041625977 | Batch Time=3.90625 +GPU:0 | Epoch: 8 | loss=12.78459358215332 | Batch Time=3.125 +GPU:0 | Epoch: 8 | loss=12.841545104980469 | Batch Time=3.125 +GPU:0 | Epoch: 8 | loss=12.882909774780273 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.676715850830078 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.783002853393555 | Batch Time=3.90625 +GPU:0 | Epoch: 8 | loss=12.69638442993164 | Batch Time=4.296875 +GPU:0 | Epoch: 8 | loss=12.756304740905762 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.73983383178711 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.719320297241211 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.7114839553833 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.837167739868164 | Batch Time=8.203125 +GPU:0 | Epoch: 8 | loss=12.681230545043945 | Batch Time=8.59375 +GPU:0 | Epoch: 8 | loss=12.55654525756836 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.74614143371582 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.788442611694336 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.657270431518555 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.652017593383789 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.597448348999023 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.672881126403809 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.747638702392578 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.728065490722656 | Batch Time=4.296875 +GPU:0 | Epoch: 8 | loss=12.568734169006348 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.66722583770752 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.778945922851562 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.666986465454102 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.677868843078613 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.600570678710938 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.631200790405273 | Batch Time=5.078125 +GPU:0 | Epoch: 8 | loss=12.809711456298828 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.692388534545898 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.47800350189209 | Batch Time=10.9375 +GPU:0 | Epoch: 8 | loss=12.679876327514648 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.733795166015625 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.7158203125 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.736152648925781 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.739381790161133 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.576257705688477 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.650867462158203 | Batch Time=8.59375 +GPU:0 | Epoch: 8 | loss=12.565093994140625 | Batch Time=8.203125 +GPU:0 | Epoch: 8 | loss=12.70339584350586 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.580604553222656 | Batch Time=8.203125 +GPU:0 | Epoch: 8 | loss=12.57077407836914 | Batch Time=5.078125 +GPU:0 | Epoch: 8 | loss=12.636452674865723 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.496560096740723 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.54142951965332 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.664360046386719 | Batch Time=8.203125 +GPU:0 | Epoch: 8 | loss=12.763753890991211 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.481169700622559 | Batch Time=10.15625 +GPU:0 | Epoch: 8 | loss=12.583988189697266 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.505694389343262 | Batch Time=5.078125 +GPU:0 | Epoch: 8 | loss=12.631206512451172 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.618081092834473 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.853816986083984 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.76190185546875 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.576553344726562 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.474002838134766 | Batch Time=8.203125 +GPU:0 | Epoch: 8 | loss=12.424877166748047 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.661142349243164 | Batch Time=4.296875 +GPU:0 | Epoch: 8 | loss=12.58324909210205 | Batch Time=8.984375 +GPU:0 | Epoch: 8 | loss=12.562460899353027 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.756340026855469 | Batch Time=5.078125 +GPU:0 | Epoch: 8 | loss=12.491111755371094 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.459264755249023 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.55178451538086 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.632224082946777 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.618066787719727 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.596048355102539 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.593584060668945 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.526836395263672 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.767120361328125 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.480377197265625 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.55936050415039 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.549150466918945 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.624316215515137 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.664127349853516 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.52953052520752 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.485507011413574 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.430877685546875 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.641735076904297 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.49112319946289 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.745148658752441 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.613372802734375 | Batch Time=8.203125 +GPU:0 | Epoch: 8 | loss=12.724185943603516 | Batch Time=5.078125 +GPU:0 | Epoch: 8 | loss=12.668046951293945 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.670448303222656 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.522331237792969 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.361529350280762 | Batch Time=10.546875 +GPU:0 | Epoch: 8 | loss=12.533628463745117 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.602896690368652 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.495996475219727 | Batch Time=8.984375 +GPU:0 | Epoch: 8 | loss=12.572272300720215 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.530865669250488 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.440918922424316 | Batch Time=9.765625 +GPU:0 | Epoch: 8 | loss=12.650283813476562 | Batch Time=4.296875 +GPU:0 | Epoch: 8 | loss=12.558856010437012 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.561270713806152 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.606141090393066 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.653101921081543 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.535226821899414 | Batch Time=8.59375 +GPU:0 | Epoch: 8 | loss=12.524084091186523 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.656801223754883 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.564848899841309 | Batch Time=4.6875 +GPU:0 | Epoch: 8 | loss=12.351722717285156 | Batch Time=8.203125 +GPU:0 | Epoch: 8 | loss=12.41988754272461 | Batch Time=6.25 +GPU:0 | Epoch: 8 | loss=12.406560897827148 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.53519344329834 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.427241325378418 | Batch Time=8.59375 +GPU:0 | Epoch: 8 | loss=12.545265197753906 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=12.586406707763672 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.446338653564453 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.592460632324219 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=12.413995742797852 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.40329360961914 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.36396598815918 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.466602325439453 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.454527854919434 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=12.411186218261719 | Batch Time=7.421875 +GPU:0 | Epoch: 8 | loss=12.313295364379883 | Batch Time=9.765625 +GPU:0 | Epoch: 8 | loss=12.39560317993164 | Batch Time=9.765625 +GPU:0 | Epoch: 8 | loss=12.601195335388184 | Batch Time=5.078125 +GPU:0 | Epoch: 8 | loss=12.42906379699707 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=12.373055458068848 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=12.363080978393555 | Batch Time=8.203125 +GPU:1 | Epoch: 8 | loss=12.94052505493164 | Batch Time=4.296875 +GPU:1 | Epoch: 8 | loss=12.775008201599121 | Batch Time=4.6875 +GPU:1 | Epoch: 8 | loss=12.761319160461426 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.776998519897461 | Batch Time=4.6875 +GPU:1 | Epoch: 8 | loss=12.556915283203125 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.51846694946289 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.718855857849121 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.714709281921387 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.742842674255371 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.608156204223633 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.801305770874023 | Batch Time=4.296875 +GPU:1 | Epoch: 8 | loss=12.673270225524902 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.539789199829102 | Batch Time=5.078125 +GPU:1 | Epoch: 8 | loss=12.630502700805664 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.633180618286133 | Batch Time=8.203125 +GPU:1 | Epoch: 8 | loss=12.721847534179688 | Batch Time=8.984375 +GPU:1 | Epoch: 8 | loss=12.725616455078125 | Batch Time=5.46875 +GPU:1 | Epoch: 8 | loss=12.634958267211914 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.613004684448242 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.493669509887695 | Batch Time=9.375 +GPU:1 | Epoch: 8 | loss=12.694449424743652 | Batch Time=5.078125 +GPU:1 | Epoch: 8 | loss=12.717708587646484 | Batch Time=10.15625 +GPU:1 | Epoch: 8 | loss=12.50714111328125 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.75463581085205 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.452573776245117 | Batch Time=8.203125 +GPU:1 | Epoch: 8 | loss=12.752220153808594 | Batch Time=4.6875 +GPU:1 | Epoch: 8 | loss=12.767210960388184 | Batch Time=3.125 +GPU:1 | Epoch: 8 | loss=12.636251449584961 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.714412689208984 | Batch Time=5.078125 +GPU:1 | Epoch: 8 | loss=12.682197570800781 | Batch Time=8.984375 +GPU:1 | Epoch: 8 | loss=12.653369903564453 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.61397933959961 | Batch Time=5.46875 +GPU:1 | Epoch: 8 | loss=12.708754539489746 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.591657638549805 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.686399459838867 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.68339729309082 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.59906005859375 | Batch Time=5.46875 +GPU:1 | Epoch: 8 | loss=12.652135848999023 | Batch Time=7.421875 +GPU:1 | Epoch: 8 | loss=12.573562622070312 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.641164779663086 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.614458084106445 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.726903915405273 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.631881713867188 | Batch Time=5.078125 +GPU:1 | Epoch: 8 | loss=12.551076889038086 | Batch Time=7.421875 +GPU:1 | Epoch: 8 | loss=12.368456840515137 | Batch Time=12.5 +GPU:1 | Epoch: 8 | loss=12.5908842086792 | Batch Time=5.078125 +GPU:1 | Epoch: 8 | loss=12.719659805297852 | Batch Time=4.6875 +GPU:1 | Epoch: 8 | loss=12.586132049560547 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.511878967285156 | Batch Time=5.46875 +GPU:1 | Epoch: 8 | loss=12.628875732421875 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.470348358154297 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.59533977508545 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.586095809936523 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.516668319702148 | Batch Time=4.6875 +GPU:1 | Epoch: 8 | loss=12.584602355957031 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.708480834960938 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.66323471069336 | Batch Time=3.90625 +GPU:1 | Epoch: 8 | loss=12.529850006103516 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.760538101196289 | Batch Time=8.984375 +GPU:1 | Epoch: 8 | loss=12.611940383911133 | Batch Time=5.46875 +GPU:1 | Epoch: 8 | loss=12.495555877685547 | Batch Time=8.203125 +GPU:1 | Epoch: 8 | loss=12.578590393066406 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.334125518798828 | Batch Time=10.546875 +GPU:1 | Epoch: 8 | loss=12.654287338256836 | Batch Time=8.984375 +GPU:1 | Epoch: 8 | loss=12.506071090698242 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.522283554077148 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.542945861816406 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.551279067993164 | Batch Time=7.421875 +GPU:1 | Epoch: 8 | loss=12.505239486694336 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.460819244384766 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.688163757324219 | Batch Time=5.46875 +GPU:1 | Epoch: 8 | loss=12.635889053344727 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.44100570678711 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.64943790435791 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.642541885375977 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.311623573303223 | Batch Time=9.765625 +GPU:1 | Epoch: 8 | loss=12.549996376037598 | Batch Time=9.765625 +GPU:1 | Epoch: 8 | loss=12.518013954162598 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.434325218200684 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.544570922851562 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.532451629638672 | Batch Time=8.203125 +GPU:1 | Epoch: 8 | loss=12.483230590820312 | Batch Time=8.984375 +GPU:1 | Epoch: 8 | loss=12.46253490447998 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.659486770629883 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.486322402954102 | Batch Time=4.296875 +GPU:1 | Epoch: 8 | loss=12.555158615112305 | Batch Time=7.421875 +GPU:1 | Epoch: 8 | loss=12.500690460205078 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.576688766479492 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.569345474243164 | Batch Time=4.296875 +GPU:1 | Epoch: 8 | loss=12.464683532714844 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.418009757995605 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.493316650390625 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.56956672668457 | Batch Time=5.078125 +GPU:1 | Epoch: 8 | loss=12.513916015625 | Batch Time=11.71875 +GPU:1 | Epoch: 8 | loss=12.534207344055176 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.522189140319824 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.551667213439941 | Batch Time=8.203125 +GPU:1 | Epoch: 8 | loss=12.393946647644043 | Batch Time=5.46875 +GPU:1 | Epoch: 8 | loss=12.544569969177246 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.45273494720459 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.430824279785156 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.39096450805664 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.378938674926758 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.602482795715332 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.478729248046875 | Batch Time=7.421875 +GPU:1 | Epoch: 8 | loss=12.53744125366211 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.480866432189941 | Batch Time=7.421875 +GPU:1 | Epoch: 8 | loss=12.531350135803223 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.472058296203613 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.500177383422852 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.613815307617188 | Batch Time=5.859375 +GPU:1 | Epoch: 8 | loss=12.413675308227539 | Batch Time=9.765625 +GPU:1 | Epoch: 8 | loss=12.469493865966797 | Batch Time=6.640625 +GPU:1 | Epoch: 8 | loss=12.533299446105957 | Batch Time=5.46875 +GPU:1 | Epoch: 8 | loss=12.495287895202637 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.480403900146484 | Batch Time=4.6875 +GPU:1 | Epoch: 8 | loss=12.414409637451172 | Batch Time=7.8125 +GPU:1 | Epoch: 8 | loss=12.472774505615234 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.493431091308594 | Batch Time=5.078125 +GPU:1 | Epoch: 8 | loss=12.508289337158203 | Batch Time=7.03125 +GPU:1 | Epoch: 8 | loss=12.672239303588867 | Batch Time=8.59375 +GPU:1 | Epoch: 8 | loss=12.394789695739746 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.57172966003418 | Batch Time=5.078125 +GPU:1 | Epoch: 8 | loss=12.4055814743042 | Batch Time=6.25 +GPU:1 | Epoch: 8 | loss=12.314859390258789 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.806184768676758 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.85146427154541 | Batch Time=3.125 +GPU:2 | Epoch: 8 | loss=12.94579792022705 | Batch Time=5.078125 +GPU:2 | Epoch: 8 | loss=12.76262092590332 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.82357406616211 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.717279434204102 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.611586570739746 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.714770317077637 | Batch Time=4.6875 +GPU:2 | Epoch: 8 | loss=12.594599723815918 | Batch Time=5.46875 +GPU:2 | Epoch: 8 | loss=12.628210067749023 | Batch Time=4.6875 +GPU:2 | Epoch: 8 | loss=12.630807876586914 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.788348197937012 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.577154159545898 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.714741706848145 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.81580924987793 | Batch Time=3.515625 +GPU:2 | Epoch: 8 | loss=12.811548233032227 | Batch Time=4.296875 +GPU:2 | Epoch: 8 | loss=12.801393508911133 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.708051681518555 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.526525497436523 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.747385025024414 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.732823371887207 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.702655792236328 | Batch Time=5.46875 +GPU:2 | Epoch: 8 | loss=12.831012725830078 | Batch Time=4.296875 +GPU:2 | Epoch: 8 | loss=12.598464965820312 | Batch Time=4.6875 +GPU:2 | Epoch: 8 | loss=12.416722297668457 | Batch Time=3.515625 +GPU:2 | Epoch: 8 | loss=12.641555786132812 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.572998046875 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.583281517028809 | Batch Time=5.46875 +GPU:2 | Epoch: 8 | loss=12.753337860107422 | Batch Time=8.984375 +GPU:2 | Epoch: 8 | loss=12.472848892211914 | Batch Time=9.375 +GPU:2 | Epoch: 8 | loss=12.438728332519531 | Batch Time=8.59375 +GPU:2 | Epoch: 8 | loss=12.649253845214844 | Batch Time=8.59375 +GPU:2 | Epoch: 8 | loss=12.610608100891113 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.633068084716797 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.730060577392578 | Batch Time=4.296875 +GPU:2 | Epoch: 8 | loss=12.682689666748047 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.59372329711914 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.603750228881836 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.742849349975586 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.677278518676758 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.473823547363281 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.724867820739746 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.44605541229248 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.50869083404541 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.628661155700684 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.373823165893555 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.730836868286133 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.544631958007812 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.716941833496094 | Batch Time=5.46875 +GPU:2 | Epoch: 8 | loss=12.463043212890625 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.728633880615234 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.521652221679688 | Batch Time=5.078125 +GPU:2 | Epoch: 8 | loss=12.661945343017578 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.478768348693848 | Batch Time=8.59375 +GPU:2 | Epoch: 8 | loss=12.495048522949219 | Batch Time=8.59375 +GPU:2 | Epoch: 8 | loss=12.446914672851562 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.704792022705078 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.522817611694336 | Batch Time=10.546875 +GPU:2 | Epoch: 8 | loss=12.366544723510742 | Batch Time=8.203125 +GPU:2 | Epoch: 8 | loss=12.550970077514648 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.43235969543457 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.511837005615234 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.573415756225586 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.497365951538086 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.355724334716797 | Batch Time=10.9375 +GPU:2 | Epoch: 8 | loss=12.53744888305664 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.607378005981445 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.596902847290039 | Batch Time=3.515625 +GPU:2 | Epoch: 8 | loss=12.46048355102539 | Batch Time=8.203125 +GPU:2 | Epoch: 8 | loss=12.585573196411133 | Batch Time=5.078125 +GPU:2 | Epoch: 8 | loss=12.641883850097656 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.435846328735352 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.508289337158203 | Batch Time=9.375 +GPU:2 | Epoch: 8 | loss=12.596692085266113 | Batch Time=8.59375 +GPU:2 | Epoch: 8 | loss=12.532295227050781 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.486001014709473 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.566014289855957 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.527327537536621 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.408452033996582 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.593429565429688 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.621355056762695 | Batch Time=5.46875 +GPU:2 | Epoch: 8 | loss=12.704513549804688 | Batch Time=4.296875 +GPU:2 | Epoch: 8 | loss=12.648948669433594 | Batch Time=4.296875 +GPU:2 | Epoch: 8 | loss=12.471097946166992 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.555456161499023 | Batch Time=5.078125 +GPU:2 | Epoch: 8 | loss=12.498464584350586 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.599058151245117 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.430408477783203 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.466547012329102 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.34829330444336 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.460355758666992 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.537940979003906 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.46500015258789 | Batch Time=9.765625 +GPU:2 | Epoch: 8 | loss=12.696990966796875 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.534989356994629 | Batch Time=3.90625 +GPU:2 | Epoch: 8 | loss=12.492548942565918 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.483874320983887 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.482136726379395 | Batch Time=3.90625 +GPU:2 | Epoch: 8 | loss=12.616291999816895 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.538069725036621 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.521812438964844 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.444828033447266 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.529731750488281 | Batch Time=2.734375 +GPU:2 | Epoch: 8 | loss=12.501591682434082 | Batch Time=5.46875 +GPU:2 | Epoch: 8 | loss=12.467809677124023 | Batch Time=5.46875 +GPU:2 | Epoch: 8 | loss=12.602508544921875 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.748848915100098 | Batch Time=6.25 +GPU:2 | Epoch: 8 | loss=12.468451499938965 | Batch Time=5.078125 +GPU:2 | Epoch: 8 | loss=12.352733612060547 | Batch Time=9.375 +GPU:2 | Epoch: 8 | loss=12.440593719482422 | Batch Time=9.765625 +GPU:2 | Epoch: 8 | loss=12.267122268676758 | Batch Time=12.109375 +GPU:2 | Epoch: 8 | loss=12.465225219726562 | Batch Time=7.421875 +GPU:2 | Epoch: 8 | loss=12.540470123291016 | Batch Time=7.8125 +GPU:2 | Epoch: 8 | loss=12.498115539550781 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.5535888671875 | Batch Time=6.640625 +GPU:2 | Epoch: 8 | loss=12.328369140625 | Batch Time=10.15625 +GPU:2 | Epoch: 8 | loss=12.427282333374023 | Batch Time=8.203125 +GPU:2 | Epoch: 8 | loss=12.631416320800781 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.432928085327148 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.51611614227295 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.381841659545898 | Batch Time=7.03125 +GPU:2 | Epoch: 8 | loss=12.50055980682373 | Batch Time=5.078125 +GPU:2 | Epoch: 8 | loss=12.590641021728516 | Batch Time=5.859375 +GPU:2 | Epoch: 8 | loss=12.409578323364258 | Batch Time=4.6875 +GPU:2 | Epoch: 8 | loss=12.521825790405273 | Batch Time=9.375 +GPU:3 | Epoch: 8 | loss=12.711261749267578 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.644433975219727 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.739167213439941 | Batch Time=4.6875 +GPU:3 | Epoch: 8 | loss=12.657917022705078 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.608345031738281 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.811544418334961 | Batch Time=1.953125 +GPU:3 | Epoch: 8 | loss=12.563097953796387 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.765074729919434 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.733336448669434 | Batch Time=5.078125 +GPU:3 | Epoch: 8 | loss=12.508321762084961 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.575621604919434 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.813525199890137 | Batch Time=5.078125 +GPU:3 | Epoch: 8 | loss=12.542438507080078 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.773738861083984 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.423818588256836 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.742593765258789 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.647491455078125 | Batch Time=3.515625 +GPU:3 | Epoch: 8 | loss=12.606691360473633 | Batch Time=10.15625 +GPU:3 | Epoch: 8 | loss=12.709489822387695 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.73982048034668 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.65640926361084 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.748870849609375 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.81174087524414 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.491438865661621 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.44582462310791 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.68052864074707 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.657247543334961 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.813176155090332 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.694225311279297 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.829252243041992 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.518173217773438 | Batch Time=8.59375 +GPU:3 | Epoch: 8 | loss=12.699312210083008 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.752314567565918 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.612279891967773 | Batch Time=7.8125 +GPU:3 | Epoch: 8 | loss=12.619749069213867 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.713188171386719 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.625232696533203 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.530712127685547 | Batch Time=8.984375 +GPU:3 | Epoch: 8 | loss=12.455068588256836 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.568023681640625 | Batch Time=4.296875 +GPU:3 | Epoch: 8 | loss=12.578531265258789 | Batch Time=4.6875 +GPU:3 | Epoch: 8 | loss=12.455795288085938 | Batch Time=8.59375 +GPU:3 | Epoch: 8 | loss=12.787339210510254 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.607786178588867 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.531746864318848 | Batch Time=9.375 +GPU:3 | Epoch: 8 | loss=12.61439323425293 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.658639907836914 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.752286911010742 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.663398742675781 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.695606231689453 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.718769073486328 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.558626174926758 | Batch Time=10.15625 +GPU:3 | Epoch: 8 | loss=12.520463943481445 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.409643173217773 | Batch Time=8.984375 +GPU:3 | Epoch: 8 | loss=12.598407745361328 | Batch Time=8.59375 +GPU:3 | Epoch: 8 | loss=12.422220230102539 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.558708190917969 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.469310760498047 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.737886428833008 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.505067825317383 | Batch Time=9.375 +GPU:3 | Epoch: 8 | loss=12.634434700012207 | Batch Time=5.078125 +GPU:3 | Epoch: 8 | loss=12.675575256347656 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.569263458251953 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.500276565551758 | Batch Time=7.8125 +GPU:3 | Epoch: 8 | loss=12.53236198425293 | Batch Time=8.984375 +GPU:3 | Epoch: 8 | loss=12.737232208251953 | Batch Time=7.8125 +GPU:3 | Epoch: 8 | loss=12.662355422973633 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.646156311035156 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.585453033447266 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.392257690429688 | Batch Time=4.296875 +GPU:3 | Epoch: 8 | loss=12.729496002197266 | Batch Time=4.6875 +GPU:3 | Epoch: 8 | loss=12.667217254638672 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.501829147338867 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.411271095275879 | Batch Time=10.15625 +GPU:3 | Epoch: 8 | loss=12.27900505065918 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.404236793518066 | Batch Time=9.765625 +GPU:3 | Epoch: 8 | loss=12.399273872375488 | Batch Time=10.15625 +GPU:3 | Epoch: 8 | loss=12.610278129577637 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.580111503601074 | Batch Time=3.90625 +GPU:3 | Epoch: 8 | loss=12.57197380065918 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.33599853515625 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.528059005737305 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.53222370147705 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.720136642456055 | Batch Time=3.90625 +GPU:3 | Epoch: 8 | loss=12.346372604370117 | Batch Time=9.375 +GPU:3 | Epoch: 8 | loss=12.528837203979492 | Batch Time=3.515625 +GPU:3 | Epoch: 8 | loss=12.502176284790039 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.479938507080078 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.553522109985352 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.320560455322266 | Batch Time=8.984375 +GPU:3 | Epoch: 8 | loss=12.548140525817871 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.438459396362305 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.440356254577637 | Batch Time=10.15625 +GPU:3 | Epoch: 8 | loss=12.487321853637695 | Batch Time=5.078125 +GPU:3 | Epoch: 8 | loss=12.529229164123535 | Batch Time=8.984375 +GPU:3 | Epoch: 8 | loss=12.804652214050293 | Batch Time=3.90625 +GPU:3 | Epoch: 8 | loss=12.324748039245605 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.477187156677246 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.565299987792969 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.491483688354492 | Batch Time=9.375 +GPU:3 | Epoch: 8 | loss=12.533391952514648 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.680797576904297 | Batch Time=3.90625 +GPU:3 | Epoch: 8 | loss=12.51247787475586 | Batch Time=7.8125 +GPU:3 | Epoch: 8 | loss=12.512386322021484 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.524467468261719 | Batch Time=8.984375 +GPU:3 | Epoch: 8 | loss=12.361492156982422 | Batch Time=8.59375 +GPU:3 | Epoch: 8 | loss=12.275083541870117 | Batch Time=7.8125 +GPU:3 | Epoch: 8 | loss=12.295660018920898 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.399864196777344 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.578718185424805 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.482505798339844 | Batch Time=7.8125 +GPU:3 | Epoch: 8 | loss=12.559825897216797 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.32084846496582 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.533374786376953 | Batch Time=5.859375 +GPU:3 | Epoch: 8 | loss=12.552473068237305 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.218345642089844 | Batch Time=8.59375 +GPU:3 | Epoch: 8 | loss=12.45025634765625 | Batch Time=7.03125 +GPU:3 | Epoch: 8 | loss=12.510014533996582 | Batch Time=4.6875 +GPU:3 | Epoch: 8 | loss=12.569135665893555 | Batch Time=6.25 +GPU:3 | Epoch: 8 | loss=12.545137405395508 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.42894172668457 | Batch Time=7.421875 +GPU:3 | Epoch: 8 | loss=12.453722953796387 | Batch Time=6.640625 +GPU:3 | Epoch: 8 | loss=12.468696594238281 | Batch Time=8.203125 +GPU:3 | Epoch: 8 | loss=12.554293632507324 | Batch Time=5.46875 +GPU:3 | Epoch: 8 | loss=12.444211959838867 | Batch Time=5.46875 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=170753720320, percent=33.7, used=78903181312, free=17512943616, active=201691774976, inactive=33620054016, buffers=223272960, cached=160928686080, shared=5660631040, slab=3222261760) +AFTER TRAIN LOOP: Epoch 8 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +BEFORE VAL LOOP: GPU: 0 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +AFTER TRAIN LOOP: Epoch 8 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +STARTING TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +BEFORE TRAIN LOOP: Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754600960, percent=33.7, used=78902595584, free=17513824256, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754600960, percent=33.7, used=78902595584, free=17513824256, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +AFTER TRAIN LOOP: Epoch 8 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +STARTING TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +BEFORE TRAIN LOOP: Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754600960, percent=33.7, used=78902595584, free=17513824256, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754600960, percent=33.7, used=78902595584, free=17513824256, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +AFTER TRAIN LOOP: Epoch 8 | Memory Usage: svmem(total=257568083968, available=170754592768, percent=33.7, used=78902603776, free=17513816064, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +STARTING TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754600960, percent=33.7, used=78902595584, free=17513824256, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +BEFORE TRAIN LOOP: Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754600960, percent=33.7, used=78902595584, free=17513824256, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=170754600960, percent=33.7, used=78902595584, free=17513824256, active=201691496448, inactive=33619931136, buffers=223272960, cached=160928391168, shared=5660327936, slab=3222171648) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 8 | loss=4.096848487854004 | Batch Time=16.40625 +GPU:0 | Epoch: 8 | loss=5.7600507736206055 | Batch Time=3.90625 +GPU:0 | Epoch: 8 | loss=5.155642986297607 | Batch Time=12.890625 +GPU:0 | Epoch: 8 | loss=5.075329780578613 | Batch Time=8.984375 +GPU:0 | Epoch: 8 | loss=5.422138214111328 | Batch Time=0.78125 +GPU:0 | Epoch: 8 | loss=4.906811714172363 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=5.783080101013184 | Batch Time=3.90625 +GPU:0 | Epoch: 8 | loss=5.778389930725098 | Batch Time=1.5625 +GPU:0 | Epoch: 8 | loss=5.833659648895264 | Batch Time=0.78125 +GPU:0 | Epoch: 8 | loss=6.397721290588379 | Batch Time=0.78125 +GPU:0 | Epoch: 8 | loss=6.119403839111328 | Batch Time=0.0 +GPU:0 | Epoch: 8 | loss=5.3806681632995605 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=5.925203800201416 | Batch Time=3.125 +GPU:0 | Epoch: 8 | loss=5.36162805557251 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=5.2281880378723145 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=5.6419219970703125 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=4.964168548583984 | Batch Time=9.375 +GPU:0 | Epoch: 8 | loss=4.336953639984131 | Batch Time=16.40625 +GPU:0 | Epoch: 8 | loss=4.896973133087158 | Batch Time=12.5 +GPU:0 | Epoch: 8 | loss=4.141764163970947 | Batch Time=18.359375 +Model top1 Accuracy: 7.08 +Acc before rounding: 7.08 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 8 | loss=4.096848487854004 | Batch Time=16.40625 +GPU:0 | Epoch: 8 | loss=5.7600507736206055 | Batch Time=3.90625 +GPU:0 | Epoch: 8 | loss=5.155642986297607 | Batch Time=12.890625 +GPU:0 | Epoch: 8 | loss=5.075329780578613 | Batch Time=8.984375 +GPU:0 | Epoch: 8 | loss=5.422138214111328 | Batch Time=0.78125 +GPU:0 | Epoch: 8 | loss=4.906811714172363 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=5.783080101013184 | Batch Time=3.90625 +GPU:0 | Epoch: 8 | loss=5.778389930725098 | Batch Time=1.5625 +GPU:0 | Epoch: 8 | loss=5.833659648895264 | Batch Time=0.78125 +GPU:0 | Epoch: 8 | loss=6.397721290588379 | Batch Time=0.78125 +GPU:0 | Epoch: 8 | loss=6.119403839111328 | Batch Time=0.0 +GPU:0 | Epoch: 8 | loss=5.3806681632995605 | Batch Time=5.46875 +GPU:0 | Epoch: 8 | loss=5.925203800201416 | Batch Time=3.125 +GPU:0 | Epoch: 8 | loss=5.36162805557251 | Batch Time=7.8125 +GPU:0 | Epoch: 8 | loss=5.2281880378723145 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=5.6419219970703125 | Batch Time=7.03125 +GPU:0 | Epoch: 8 | loss=4.964168548583984 | Batch Time=9.375 +GPU:0 | Epoch: 8 | loss=4.336953639984131 | Batch Time=16.40625 +GPU:0 | Epoch: 8 | loss=4.896973133087158 | Batch Time=12.5 +GPU:0 | Epoch: 8 | loss=4.141764163970947 | Batch Time=18.359375 +Model top1 Accuracy: 7.08 +Acc after rounding: 7.08 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 8 | loss=5.456350803375244 | Batch Time=5.859375 +GPU:0 | Epoch: 8 | loss=5.618973255157471 | Batch Time=6.640625 +GPU:0 | Epoch: 8 | loss=5.3979411125183105 | Batch Time=8.203125 +GPU:0 | Epoch: 8 | loss=5.514118194580078 | Batch Time=7.03125 +Model top1 Accuracy: 6.28 +Validation Acc after rounding: 6.28 +AFTER VAL LOOP: GPU: 0 | Epoch 8 | Memory Usage: svmem(total=257568083968, available=152935948288, percent=40.6, used=96720801792, free=6138560512, active=215102586880, inactive=31480438784, buffers=122269696, cached=154586451968, shared=5660774400, slab=3218817024) +Rounding model with scheme: naive +Model avg sparsity: 45.3144090335821 +GPU:0 | Epoch: 8 | Acc=7.08 | Epoch Time=15.906265115737915 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=152936026112, percent=40.6, used=96720769024, free=6138576896, active=215102599168, inactive=31480406016, buffers=122368000, cached=154586370048, shared=5660692480, slab=3218800640) +STARTING TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=152936026112, percent=40.6, used=96720769024, free=6138576896, active=215102599168, inactive=31480406016, buffers=122368000, cached=154586370048, shared=5660692480, slab=3218800640) +BEFORE TRAIN LOOP: Epoch 9 | Memory Usage: svmem(total=257568083968, available=152936026112, percent=40.6, used=96720769024, free=6138576896, active=215102599168, inactive=31480406016, buffers=122368000, cached=154586370048, shared=5660692480, slab=3218800640) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=152936026112, percent=40.6, used=96720769024, free=6138576896, active=215102599168, inactive=31480406016, buffers=122368000, cached=154586370048, shared=5660692480, slab=3218800640) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:3 | Epoch: 9 | loss=12.385870933532715 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.616342544555664 | Batch Time=5.46875 +GPU:3 | Epoch: 9 | loss=12.454246520996094 | Batch Time=5.859375 +GPU:3 | Epoch: 9 | loss=12.357881546020508 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.495899200439453 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.563512802124023 | Batch Time=5.078125 +GPU:3 | Epoch: 9 | loss=12.353832244873047 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.33201789855957 | Batch Time=10.546875 +GPU:3 | Epoch: 9 | loss=12.549333572387695 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.437240600585938 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.481891632080078 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.309562683105469 | Batch Time=8.984375 +GPU:3 | Epoch: 9 | loss=12.278488159179688 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.298864364624023 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.46082592010498 | Batch Time=5.078125 +GPU:3 | Epoch: 9 | loss=12.526691436767578 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.32011604309082 | Batch Time=6.25 +GPU:3 | Epoch: 9 | loss=12.384469985961914 | Batch Time=6.25 +GPU:3 | Epoch: 9 | loss=12.387304306030273 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.343574523925781 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.580360412597656 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.469868659973145 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.580024719238281 | Batch Time=4.6875 +GPU:3 | Epoch: 9 | loss=12.566251754760742 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.477025032043457 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.443214416503906 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.403253555297852 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.434253692626953 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.52638053894043 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.573902130126953 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.35228157043457 | Batch Time=5.859375 +GPU:3 | Epoch: 9 | loss=12.397968292236328 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.420267105102539 | Batch Time=6.25 +GPU:3 | Epoch: 9 | loss=12.468547821044922 | Batch Time=5.078125 +GPU:3 | Epoch: 9 | loss=12.405845642089844 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.477560043334961 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.393951416015625 | Batch Time=5.859375 +GPU:3 | Epoch: 9 | loss=12.120599746704102 | Batch Time=12.890625 +GPU:3 | Epoch: 9 | loss=12.3748140335083 | Batch Time=5.859375 +GPU:3 | Epoch: 9 | loss=12.221406936645508 | Batch Time=12.890625 +GPU:3 | Epoch: 9 | loss=12.482884407043457 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.25918197631836 | Batch Time=10.9375 +GPU:3 | Epoch: 9 | loss=12.326075553894043 | Batch Time=10.9375 +GPU:3 | Epoch: 9 | loss=12.349967002868652 | Batch Time=6.25 +GPU:3 | Epoch: 9 | loss=12.457764625549316 | Batch Time=5.46875 +GPU:3 | Epoch: 9 | loss=12.298576354980469 | Batch Time=8.984375 +GPU:3 | Epoch: 9 | loss=12.397842407226562 | Batch Time=5.859375 +GPU:3 | Epoch: 9 | loss=12.551055908203125 | Batch Time=5.078125 +GPU:3 | Epoch: 9 | loss=12.43266487121582 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.388386726379395 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.405344009399414 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.313297271728516 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.284818649291992 | Batch Time=9.765625 +GPU:3 | Epoch: 9 | loss=12.508134841918945 | Batch Time=8.984375 +GPU:3 | Epoch: 9 | loss=12.392902374267578 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.438380241394043 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.63986587524414 | Batch Time=5.078125 +GPU:3 | Epoch: 9 | loss=12.476847648620605 | Batch Time=6.25 +GPU:3 | Epoch: 9 | loss=12.472082138061523 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.175716400146484 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.34802532196045 | Batch Time=10.15625 +GPU:3 | Epoch: 9 | loss=12.316703796386719 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.384775161743164 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.454341888427734 | Batch Time=4.6875 +GPU:3 | Epoch: 9 | loss=12.294763565063477 | Batch Time=5.46875 +GPU:3 | Epoch: 9 | loss=12.029071807861328 | Batch Time=11.71875 +GPU:3 | Epoch: 9 | loss=12.20166015625 | Batch Time=9.375 +GPU:3 | Epoch: 9 | loss=12.324399948120117 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.240974426269531 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.282512664794922 | Batch Time=8.984375 +GPU:3 | Epoch: 9 | loss=12.198911666870117 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.370270729064941 | Batch Time=4.6875 +GPU:3 | Epoch: 9 | loss=12.264410018920898 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.286227226257324 | Batch Time=5.078125 +GPU:3 | Epoch: 9 | loss=12.385601043701172 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.548775672912598 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.342082023620605 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.1980562210083 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.394798278808594 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.194915771484375 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.153718948364258 | Batch Time=6.25 +GPU:3 | Epoch: 9 | loss=12.352561950683594 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.270618438720703 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.290740966796875 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.167610168457031 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.377641677856445 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.284259796142578 | Batch Time=8.984375 +GPU:3 | Epoch: 9 | loss=12.325370788574219 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.277969360351562 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.285196304321289 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.495794296264648 | Batch Time=6.25 +GPU:3 | Epoch: 9 | loss=12.058653831481934 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.357898712158203 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.408523559570312 | Batch Time=5.46875 +GPU:3 | Epoch: 9 | loss=12.405391693115234 | Batch Time=3.90625 +GPU:3 | Epoch: 9 | loss=12.271463394165039 | Batch Time=10.546875 +GPU:3 | Epoch: 9 | loss=12.31615161895752 | Batch Time=4.6875 +GPU:3 | Epoch: 9 | loss=12.267075538635254 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.462878227233887 | Batch Time=5.46875 +GPU:3 | Epoch: 9 | loss=12.403457641601562 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.251989364624023 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.30147933959961 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.3473482131958 | Batch Time=8.984375 +GPU:3 | Epoch: 9 | loss=12.359088897705078 | Batch Time=8.984375 +GPU:3 | Epoch: 9 | loss=12.238090515136719 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.181035995483398 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.427818298339844 | Batch Time=6.640625 +GPU:3 | Epoch: 9 | loss=12.180967330932617 | Batch Time=5.859375 +GPU:3 | Epoch: 9 | loss=12.322452545166016 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.231622695922852 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.169170379638672 | Batch Time=10.15625 +GPU:3 | Epoch: 9 | loss=12.258707046508789 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.154932975769043 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.280710220336914 | Batch Time=8.203125 +GPU:3 | Epoch: 9 | loss=12.249242782592773 | Batch Time=8.984375 +GPU:3 | Epoch: 9 | loss=12.27588176727295 | Batch Time=6.25 +GPU:3 | Epoch: 9 | loss=12.210296630859375 | Batch Time=10.15625 +GPU:3 | Epoch: 9 | loss=12.132664680480957 | Batch Time=9.375 +GPU:3 | Epoch: 9 | loss=12.34041690826416 | Batch Time=4.6875 +GPU:3 | Epoch: 9 | loss=12.276144027709961 | Batch Time=5.46875 +GPU:3 | Epoch: 9 | loss=12.495378494262695 | Batch Time=7.03125 +GPU:3 | Epoch: 9 | loss=12.272037506103516 | Batch Time=7.421875 +GPU:3 | Epoch: 9 | loss=12.245394706726074 | Batch Time=8.59375 +GPU:3 | Epoch: 9 | loss=12.161628723144531 | Batch Time=7.8125 +GPU:3 | Epoch: 9 | loss=12.352945327758789 | Batch Time=5.46875 +GPU:2 | Epoch: 9 | loss=12.312047004699707 | Batch Time=8.203125 +GPU:2 | Epoch: 9 | loss=12.63272476196289 | Batch Time=6.25 +GPU:2 | Epoch: 9 | loss=12.39720344543457 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.51113510131836 | Batch Time=6.640625 +GPU:2 | Epoch: 9 | loss=12.484155654907227 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.405384063720703 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.415045738220215 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.487428665161133 | Batch Time=8.203125 +GPU:2 | Epoch: 9 | loss=12.302492141723633 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.427608489990234 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.30606746673584 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.533758163452148 | Batch Time=5.46875 +GPU:2 | Epoch: 9 | loss=12.473363876342773 | Batch Time=4.296875 +GPU:2 | Epoch: 9 | loss=12.393991470336914 | Batch Time=4.6875 +GPU:2 | Epoch: 9 | loss=12.355792045593262 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.521793365478516 | Batch Time=5.859375 +GPU:2 | Epoch: 9 | loss=12.39297866821289 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.263965606689453 | Batch Time=9.765625 +GPU:2 | Epoch: 9 | loss=12.43295669555664 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.528863906860352 | Batch Time=6.25 +GPU:2 | Epoch: 9 | loss=12.373270034790039 | Batch Time=6.25 +GPU:2 | Epoch: 9 | loss=12.359424591064453 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.506706237792969 | Batch Time=6.25 +GPU:2 | Epoch: 9 | loss=12.349748611450195 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.551092147827148 | Batch Time=9.765625 +GPU:2 | Epoch: 9 | loss=12.397485733032227 | Batch Time=3.125 +GPU:2 | Epoch: 9 | loss=12.355422973632812 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.26321029663086 | Batch Time=10.9375 +GPU:2 | Epoch: 9 | loss=12.495403289794922 | Batch Time=5.859375 +GPU:2 | Epoch: 9 | loss=12.466121673583984 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.371728897094727 | Batch Time=9.765625 +GPU:2 | Epoch: 9 | loss=12.345674514770508 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.280939102172852 | Batch Time=9.765625 +GPU:2 | Epoch: 9 | loss=12.30963134765625 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.15097427368164 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.43632698059082 | Batch Time=4.6875 +GPU:2 | Epoch: 9 | loss=12.329017639160156 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.365591049194336 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.622220039367676 | Batch Time=4.6875 +GPU:2 | Epoch: 9 | loss=12.398290634155273 | Batch Time=9.375 +GPU:2 | Epoch: 9 | loss=12.35775089263916 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.423717498779297 | Batch Time=8.203125 +GPU:2 | Epoch: 9 | loss=12.37988567352295 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.463896751403809 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.457337379455566 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.4569091796875 | Batch Time=9.375 +GPU:2 | Epoch: 9 | loss=12.505857467651367 | Batch Time=6.640625 +GPU:2 | Epoch: 9 | loss=12.258462905883789 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.526519775390625 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.667801856994629 | Batch Time=5.859375 +GPU:2 | Epoch: 9 | loss=12.269594192504883 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.498234748840332 | Batch Time=6.25 +GPU:2 | Epoch: 9 | loss=12.284595489501953 | Batch Time=9.765625 +GPU:2 | Epoch: 9 | loss=12.293951034545898 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.493836402893066 | Batch Time=4.6875 +GPU:2 | Epoch: 9 | loss=12.467185020446777 | Batch Time=5.46875 +GPU:2 | Epoch: 9 | loss=12.572826385498047 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.437705039978027 | Batch Time=9.765625 +GPU:2 | Epoch: 9 | loss=12.356386184692383 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.330280303955078 | Batch Time=11.328125 +GPU:2 | Epoch: 9 | loss=12.529130935668945 | Batch Time=6.640625 +GPU:2 | Epoch: 9 | loss=12.278240203857422 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.392594337463379 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.474607467651367 | Batch Time=4.296875 +GPU:2 | Epoch: 9 | loss=12.410711288452148 | Batch Time=5.078125 +GPU:2 | Epoch: 9 | loss=12.160322189331055 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.490341186523438 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.321788787841797 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.29731559753418 | Batch Time=6.640625 +GPU:2 | Epoch: 9 | loss=12.117431640625 | Batch Time=10.15625 +GPU:2 | Epoch: 9 | loss=12.225065231323242 | Batch Time=5.46875 +GPU:2 | Epoch: 9 | loss=12.39887809753418 | Batch Time=5.078125 +GPU:2 | Epoch: 9 | loss=12.236833572387695 | Batch Time=9.375 +GPU:2 | Epoch: 9 | loss=12.152132034301758 | Batch Time=8.203125 +GPU:2 | Epoch: 9 | loss=12.34620475769043 | Batch Time=10.546875 +GPU:2 | Epoch: 9 | loss=12.2822847366333 | Batch Time=6.25 +GPU:2 | Epoch: 9 | loss=12.403189659118652 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.416776657104492 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.32893180847168 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.258769989013672 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.124406814575195 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.300154685974121 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.270822525024414 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.106534957885742 | Batch Time=10.15625 +GPU:2 | Epoch: 9 | loss=12.283248901367188 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.212968826293945 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.143348693847656 | Batch Time=9.765625 +GPU:2 | Epoch: 9 | loss=12.291421890258789 | Batch Time=8.203125 +GPU:2 | Epoch: 9 | loss=12.201757431030273 | Batch Time=11.71875 +GPU:2 | Epoch: 9 | loss=12.268766403198242 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.205554962158203 | Batch Time=8.59375 +GPU:2 | Epoch: 9 | loss=12.226137161254883 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.317676544189453 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.353843688964844 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.372882843017578 | Batch Time=8.203125 +GPU:2 | Epoch: 9 | loss=12.377580642700195 | Batch Time=4.6875 +GPU:2 | Epoch: 9 | loss=12.178060531616211 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.371174812316895 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.321540832519531 | Batch Time=5.859375 +GPU:2 | Epoch: 9 | loss=12.327629089355469 | Batch Time=6.25 +GPU:2 | Epoch: 9 | loss=12.408937454223633 | Batch Time=6.25 +GPU:2 | Epoch: 9 | loss=12.20018196105957 | Batch Time=6.640625 +GPU:2 | Epoch: 9 | loss=12.202656745910645 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.309280395507812 | Batch Time=4.296875 +GPU:2 | Epoch: 9 | loss=12.243887901306152 | Batch Time=6.640625 +GPU:2 | Epoch: 9 | loss=12.034017562866211 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.322269439697266 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.303707122802734 | Batch Time=11.328125 +GPU:2 | Epoch: 9 | loss=12.318257331848145 | Batch Time=9.375 +GPU:2 | Epoch: 9 | loss=12.153593063354492 | Batch Time=9.375 +GPU:2 | Epoch: 9 | loss=12.259538650512695 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.197460174560547 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.17348575592041 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.250558853149414 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.324310302734375 | Batch Time=7.03125 +GPU:2 | Epoch: 9 | loss=12.15084457397461 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.366888046264648 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.186800003051758 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.30872917175293 | Batch Time=7.8125 +GPU:2 | Epoch: 9 | loss=12.16635513305664 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.10739517211914 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.283327102661133 | Batch Time=8.984375 +GPU:2 | Epoch: 9 | loss=12.083956718444824 | Batch Time=10.546875 +GPU:2 | Epoch: 9 | loss=12.215221405029297 | Batch Time=7.421875 +GPU:2 | Epoch: 9 | loss=12.299936294555664 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.397368431091309 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.602445602416992 | Batch Time=5.859375 +GPU:1 | Epoch: 9 | loss=12.550027847290039 | Batch Time=5.078125 +GPU:1 | Epoch: 9 | loss=12.286710739135742 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.471879959106445 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.389274597167969 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.423369407653809 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.45720386505127 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.405252456665039 | Batch Time=9.375 +GPU:1 | Epoch: 9 | loss=12.45562744140625 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.490154266357422 | Batch Time=5.46875 +GPU:1 | Epoch: 9 | loss=12.457319259643555 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.441877365112305 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.381986618041992 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.335583686828613 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.472015380859375 | Batch Time=6.640625 +GPU:1 | Epoch: 9 | loss=12.561138153076172 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.409791946411133 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.41042709350586 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.402355194091797 | Batch Time=4.6875 +GPU:1 | Epoch: 9 | loss=12.41110610961914 | Batch Time=9.375 +GPU:1 | Epoch: 9 | loss=12.572096824645996 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.451255798339844 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.371706008911133 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.332108497619629 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.384790420532227 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.343368530273438 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.409679412841797 | Batch Time=6.640625 +GPU:1 | Epoch: 9 | loss=12.211845397949219 | Batch Time=10.546875 +GPU:1 | Epoch: 9 | loss=12.603704452514648 | Batch Time=5.859375 +GPU:1 | Epoch: 9 | loss=12.271032333374023 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.344491958618164 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.563800811767578 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.457450866699219 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.228874206542969 | Batch Time=10.15625 +GPU:1 | Epoch: 9 | loss=12.40671157836914 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.520870208740234 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.365246772766113 | Batch Time=9.375 +GPU:1 | Epoch: 9 | loss=12.464436531066895 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.312982559204102 | Batch Time=9.375 +GPU:1 | Epoch: 9 | loss=12.513311386108398 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.498111724853516 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.410013198852539 | Batch Time=6.640625 +GPU:1 | Epoch: 9 | loss=12.366517066955566 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.221650123596191 | Batch Time=8.984375 +GPU:1 | Epoch: 9 | loss=12.486066818237305 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.285987854003906 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.309048652648926 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.298206329345703 | Batch Time=11.71875 +GPU:1 | Epoch: 9 | loss=12.141497611999512 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.297735214233398 | Batch Time=11.328125 +GPU:1 | Epoch: 9 | loss=12.4116792678833 | Batch Time=5.46875 +GPU:1 | Epoch: 9 | loss=12.324501037597656 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.40040397644043 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.340034484863281 | Batch Time=5.859375 +GPU:1 | Epoch: 9 | loss=12.454882621765137 | Batch Time=5.078125 +GPU:1 | Epoch: 9 | loss=12.089069366455078 | Batch Time=10.15625 +GPU:1 | Epoch: 9 | loss=12.421761512756348 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.39384651184082 | Batch Time=4.6875 +GPU:1 | Epoch: 9 | loss=12.03643798828125 | Batch Time=12.109375 +GPU:1 | Epoch: 9 | loss=12.184144020080566 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.162969589233398 | Batch Time=11.71875 +GPU:1 | Epoch: 9 | loss=12.402841567993164 | Batch Time=3.90625 +GPU:1 | Epoch: 9 | loss=12.261957168579102 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.180793762207031 | Batch Time=10.15625 +GPU:1 | Epoch: 9 | loss=12.23577880859375 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.313333511352539 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.24172592163086 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.490198135375977 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.492361068725586 | Batch Time=9.375 +GPU:1 | Epoch: 9 | loss=12.317052841186523 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.292608261108398 | Batch Time=9.375 +GPU:1 | Epoch: 9 | loss=12.324068069458008 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.205635070800781 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.352757453918457 | Batch Time=10.546875 +GPU:1 | Epoch: 9 | loss=12.354680061340332 | Batch Time=5.859375 +GPU:1 | Epoch: 9 | loss=12.317615509033203 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.314175605773926 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.33643913269043 | Batch Time=4.6875 +GPU:1 | Epoch: 9 | loss=12.409658432006836 | Batch Time=4.6875 +GPU:1 | Epoch: 9 | loss=12.28078556060791 | Batch Time=10.546875 +GPU:1 | Epoch: 9 | loss=12.220711708068848 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.46762466430664 | Batch Time=8.984375 +GPU:1 | Epoch: 9 | loss=12.495098114013672 | Batch Time=5.46875 +GPU:1 | Epoch: 9 | loss=12.239561080932617 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.255645751953125 | Batch Time=9.765625 +GPU:1 | Epoch: 9 | loss=12.187919616699219 | Batch Time=9.765625 +GPU:1 | Epoch: 9 | loss=12.350202560424805 | Batch Time=8.984375 +GPU:1 | Epoch: 9 | loss=12.384471893310547 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.347627639770508 | Batch Time=6.640625 +GPU:1 | Epoch: 9 | loss=12.276948928833008 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.411638259887695 | Batch Time=5.078125 +GPU:1 | Epoch: 9 | loss=12.337078094482422 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.431941986083984 | Batch Time=5.46875 +GPU:1 | Epoch: 9 | loss=12.132598876953125 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.325939178466797 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.281328201293945 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.166401863098145 | Batch Time=10.15625 +GPU:1 | Epoch: 9 | loss=12.291159629821777 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.284086227416992 | Batch Time=9.765625 +GPU:1 | Epoch: 9 | loss=12.330144882202148 | Batch Time=6.640625 +GPU:1 | Epoch: 9 | loss=12.134185791015625 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.352079391479492 | Batch Time=6.640625 +GPU:1 | Epoch: 9 | loss=12.294452667236328 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.188394546508789 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.494512557983398 | Batch Time=5.46875 +GPU:1 | Epoch: 9 | loss=12.270137786865234 | Batch Time=9.375 +GPU:1 | Epoch: 9 | loss=12.293773651123047 | Batch Time=8.203125 +GPU:1 | Epoch: 9 | loss=12.21550464630127 | Batch Time=8.984375 +GPU:1 | Epoch: 9 | loss=12.380200386047363 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.290975570678711 | Batch Time=4.6875 +GPU:1 | Epoch: 9 | loss=12.300914764404297 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.156084060668945 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.402328491210938 | Batch Time=5.859375 +GPU:1 | Epoch: 9 | loss=12.303369522094727 | Batch Time=5.46875 +GPU:1 | Epoch: 9 | loss=12.237846374511719 | Batch Time=6.25 +GPU:1 | Epoch: 9 | loss=12.11552619934082 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.04019832611084 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.278210639953613 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.192314147949219 | Batch Time=7.421875 +GPU:1 | Epoch: 9 | loss=12.404960632324219 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.173576354980469 | Batch Time=8.59375 +GPU:1 | Epoch: 9 | loss=12.052518844604492 | Batch Time=7.8125 +GPU:1 | Epoch: 9 | loss=12.29897689819336 | Batch Time=7.03125 +GPU:1 | Epoch: 9 | loss=12.379486083984375 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.570687294006348 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.469234466552734 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.504402160644531 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.459232330322266 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.601533889770508 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.485916137695312 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.448758125305176 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.374195098876953 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.428363800048828 | Batch Time=10.15625 +GPU:0 | Epoch: 9 | loss=12.54676628112793 | Batch Time=9.375 +GPU:0 | Epoch: 9 | loss=12.44967269897461 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.489736557006836 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.437141418457031 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.200462341308594 | Batch Time=10.15625 +GPU:0 | Epoch: 9 | loss=12.300040245056152 | Batch Time=10.546875 +GPU:0 | Epoch: 9 | loss=12.431371688842773 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.441957473754883 | Batch Time=8.203125 +GPU:0 | Epoch: 9 | loss=12.39546012878418 | Batch Time=3.125 +GPU:0 | Epoch: 9 | loss=12.409622192382812 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.686067581176758 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=12.594108581542969 | Batch Time=3.515625 +GPU:0 | Epoch: 9 | loss=12.313496589660645 | Batch Time=8.984375 +GPU:0 | Epoch: 9 | loss=12.371377944946289 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.56743049621582 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.345704078674316 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.459884643554688 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=12.312374114990234 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.399616241455078 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.531532287597656 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=12.603560447692871 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=12.487558364868164 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.501935958862305 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.31927490234375 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.396621704101562 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.420896530151367 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=12.42532730102539 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.330429077148438 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=12.38074016571045 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.44829273223877 | Batch Time=9.375 +GPU:0 | Epoch: 9 | loss=12.232351303100586 | Batch Time=8.984375 +GPU:0 | Epoch: 9 | loss=12.296147346496582 | Batch Time=11.328125 +GPU:0 | Epoch: 9 | loss=12.323680877685547 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.248251914978027 | Batch Time=8.203125 +GPU:0 | Epoch: 9 | loss=12.330957412719727 | Batch Time=8.984375 +GPU:0 | Epoch: 9 | loss=12.486037254333496 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=12.349624633789062 | Batch Time=5.46875 +GPU:0 | Epoch: 9 | loss=12.341529846191406 | Batch Time=7.421875 +GPU:0 | Epoch: 9 | loss=12.14389419555664 | Batch Time=9.375 +GPU:0 | Epoch: 9 | loss=12.417081832885742 | Batch Time=7.421875 +GPU:0 | Epoch: 9 | loss=12.452855110168457 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.362516403198242 | Batch Time=9.765625 +GPU:0 | Epoch: 9 | loss=12.252988815307617 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.329200744628906 | Batch Time=8.984375 +GPU:0 | Epoch: 9 | loss=12.253957748413086 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=12.502311706542969 | Batch Time=5.46875 +GPU:0 | Epoch: 9 | loss=12.5282621383667 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.499818801879883 | Batch Time=3.90625 +GPU:0 | Epoch: 9 | loss=12.34580135345459 | Batch Time=9.375 +GPU:0 | Epoch: 9 | loss=12.269708633422852 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=12.410673141479492 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.273262023925781 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.33967399597168 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.428743362426758 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.437193870544434 | Batch Time=8.984375 +GPU:0 | Epoch: 9 | loss=12.14201545715332 | Batch Time=10.546875 +GPU:0 | Epoch: 9 | loss=12.486080169677734 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.245051383972168 | Batch Time=7.421875 +GPU:0 | Epoch: 9 | loss=12.18326187133789 | Batch Time=10.546875 +GPU:0 | Epoch: 9 | loss=12.358837127685547 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.362997055053711 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.20489501953125 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.432178497314453 | Batch Time=4.296875 +GPU:0 | Epoch: 9 | loss=12.31178092956543 | Batch Time=8.203125 +GPU:0 | Epoch: 9 | loss=12.341985702514648 | Batch Time=9.375 +GPU:0 | Epoch: 9 | loss=12.289466857910156 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.257515907287598 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.400197982788086 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.402738571166992 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.247488975524902 | Batch Time=9.375 +GPU:0 | Epoch: 9 | loss=12.324522018432617 | Batch Time=7.421875 +GPU:0 | Epoch: 9 | loss=12.38873291015625 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.0931978225708 | Batch Time=11.71875 +GPU:0 | Epoch: 9 | loss=12.290443420410156 | Batch Time=8.984375 +GPU:0 | Epoch: 9 | loss=12.257137298583984 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=12.320512771606445 | Batch Time=7.421875 +GPU:0 | Epoch: 9 | loss=12.440664291381836 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.325786590576172 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.162649154663086 | Batch Time=8.203125 +GPU:0 | Epoch: 9 | loss=12.22828483581543 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.251428604125977 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.232009887695312 | Batch Time=8.984375 +GPU:0 | Epoch: 9 | loss=12.18825912475586 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.354406356811523 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=12.362266540527344 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=12.351127624511719 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.419694900512695 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.36959171295166 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.386136054992676 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=12.35409164428711 | Batch Time=5.859375 +GPU:0 | Epoch: 9 | loss=12.072725296020508 | Batch Time=10.546875 +GPU:0 | Epoch: 9 | loss=11.986040115356445 | Batch Time=12.5 +GPU:0 | Epoch: 9 | loss=12.190807342529297 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.51954460144043 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.240312576293945 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.168436050415039 | Batch Time=9.375 +GPU:0 | Epoch: 9 | loss=12.289812088012695 | Batch Time=10.15625 +GPU:0 | Epoch: 9 | loss=12.206735610961914 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.319881439208984 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.096822738647461 | Batch Time=9.765625 +GPU:0 | Epoch: 9 | loss=12.299577713012695 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.303121566772461 | Batch Time=7.421875 +GPU:0 | Epoch: 9 | loss=12.205602645874023 | Batch Time=8.984375 +GPU:0 | Epoch: 9 | loss=12.123536109924316 | Batch Time=7.421875 +GPU:0 | Epoch: 9 | loss=12.225946426391602 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.439416885375977 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=12.361113548278809 | Batch Time=11.328125 +GPU:0 | Epoch: 9 | loss=12.244281768798828 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=12.16222095489502 | Batch Time=8.203125 +GPU:0 | Epoch: 9 | loss=12.009511947631836 | Batch Time=11.71875 +GPU:0 | Epoch: 9 | loss=12.280603408813477 | Batch Time=4.6875 +GPU:0 | Epoch: 9 | loss=12.17027473449707 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=12.211048126220703 | Batch Time=8.59375 +GPU:0 | Epoch: 9 | loss=12.305747032165527 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=12.21932601928711 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=12.284158706665039 | Batch Time=7.8125 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=163212566528, percent=36.6, used=86444306432, free=17403482112, active=198077378560, inactive=37293268992, buffers=406351872, cached=153313943552, shared=5660659712, slab=3203084288) +AFTER TRAIN LOOP: Epoch 9 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +STARTING TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +BEFORE TRAIN LOOP: Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=163212582912, percent=36.6, used=86444290048, free=17403498496, active=198077378560, inactive=37293268992, buffers=406351872, cached=153313943552, shared=5660659712, slab=3203084288) +AFTER TRAIN LOOP: Epoch 9 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +STARTING TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +BEFORE TRAIN LOOP: Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=163212582912, percent=36.6, used=86444290048, free=17403498496, active=198077378560, inactive=37293268992, buffers=406351872, cached=153313943552, shared=5660659712, slab=3203084288) +AFTER TRAIN LOOP: Epoch 9 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +BEFORE VAL LOOP: GPU: 0 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=163212582912, percent=36.6, used=86444290048, free=17403498496, active=198077378560, inactive=37293268992, buffers=406351872, cached=153313943552, shared=5660659712, slab=3203084288) +AFTER TRAIN LOOP: Epoch 9 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +STARTING TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +BEFORE TRAIN LOOP: Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=163213115392, percent=36.6, used=86443978752, free=17404030976, active=198077120512, inactive=37293170688, buffers=406351872, cached=153313722368, shared=5660413952, slab=3203067904) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 9 | loss=4.2433366775512695 | Batch Time=21.484375 +GPU:0 | Epoch: 9 | loss=5.987119197845459 | Batch Time=1.5625 +GPU:0 | Epoch: 9 | loss=5.338128566741943 | Batch Time=12.5 +GPU:0 | Epoch: 9 | loss=5.423455715179443 | Batch Time=3.515625 +GPU:0 | Epoch: 9 | loss=5.871593475341797 | Batch Time=1.5625 +GPU:0 | Epoch: 9 | loss=5.401772975921631 | Batch Time=5.46875 +GPU:0 | Epoch: 9 | loss=5.996971130371094 | Batch Time=1.5625 +GPU:0 | Epoch: 9 | loss=6.052364349365234 | Batch Time=1.953125 +GPU:0 | Epoch: 9 | loss=5.63732385635376 | Batch Time=2.734375 +GPU:0 | Epoch: 9 | loss=6.368051052093506 | Batch Time=0.0 +GPU:0 | Epoch: 9 | loss=5.886529922485352 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=5.216063499450684 | Batch Time=4.6875 +GPU:0 | Epoch: 9 | loss=6.008157253265381 | Batch Time=1.953125 +GPU:0 | Epoch: 9 | loss=5.435529708862305 | Batch Time=9.765625 +GPU:0 | Epoch: 9 | loss=5.166008949279785 | Batch Time=4.296875 +GPU:0 | Epoch: 9 | loss=5.443054676055908 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=5.22536563873291 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=4.474084854125977 | Batch Time=14.0625 +GPU:0 | Epoch: 9 | loss=4.8729047775268555 | Batch Time=13.671875 +GPU:0 | Epoch: 9 | loss=3.9705650806427 | Batch Time=18.75 +Model top1 Accuracy: 7.022 +Acc before rounding: 7.022 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 9 | loss=4.2433366775512695 | Batch Time=21.484375 +GPU:0 | Epoch: 9 | loss=5.987119197845459 | Batch Time=1.5625 +GPU:0 | Epoch: 9 | loss=5.338128566741943 | Batch Time=12.5 +GPU:0 | Epoch: 9 | loss=5.423455715179443 | Batch Time=3.515625 +GPU:0 | Epoch: 9 | loss=5.871593475341797 | Batch Time=1.5625 +GPU:0 | Epoch: 9 | loss=5.401772975921631 | Batch Time=5.46875 +GPU:0 | Epoch: 9 | loss=5.996971130371094 | Batch Time=1.5625 +GPU:0 | Epoch: 9 | loss=6.052364349365234 | Batch Time=1.953125 +GPU:0 | Epoch: 9 | loss=5.63732385635376 | Batch Time=2.734375 +GPU:0 | Epoch: 9 | loss=6.368051052093506 | Batch Time=0.0 +GPU:0 | Epoch: 9 | loss=5.886529922485352 | Batch Time=6.640625 +GPU:0 | Epoch: 9 | loss=5.216063499450684 | Batch Time=4.6875 +GPU:0 | Epoch: 9 | loss=6.008157253265381 | Batch Time=1.953125 +GPU:0 | Epoch: 9 | loss=5.435529708862305 | Batch Time=9.765625 +GPU:0 | Epoch: 9 | loss=5.166008949279785 | Batch Time=4.296875 +GPU:0 | Epoch: 9 | loss=5.443054676055908 | Batch Time=6.25 +GPU:0 | Epoch: 9 | loss=5.22536563873291 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=4.474084854125977 | Batch Time=14.0625 +GPU:0 | Epoch: 9 | loss=4.8729047775268555 | Batch Time=13.671875 +GPU:0 | Epoch: 9 | loss=3.9705650806427 | Batch Time=18.75 +Model top1 Accuracy: 7.022 +Acc after rounding: 7.022 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 9 | loss=5.606781005859375 | Batch Time=5.078125 +GPU:0 | Epoch: 9 | loss=5.5264129638671875 | Batch Time=7.8125 +GPU:0 | Epoch: 9 | loss=5.442775726318359 | Batch Time=7.03125 +GPU:0 | Epoch: 9 | loss=5.571305274963379 | Batch Time=6.640625 +Model top1 Accuracy: 5.92 +Validation Acc after rounding: 5.92 +AFTER VAL LOOP: GPU: 0 | Epoch 9 | Memory Usage: svmem(total=257568083968, available=145393479680, percent=43.6, used=104263139328, free=6324887552, active=206688681984, inactive=39644921856, buffers=408797184, cached=146571259904, shared=5660889088, slab=3207606272) +Rounding model with scheme: naive +Model avg sparsity: 44.776743361137775 +GPU:0 | Epoch: 9 | Acc=7.022 | Epoch Time=16.059971292813618 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=145393479680, percent=43.6, used=104263139328, free=6324887552, active=206688665600, inactive=39644921856, buffers=408805376, cached=146571251712, shared=5660889088, slab=3207602176) +STARTING TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=145393479680, percent=43.6, used=104263139328, free=6324887552, active=206688665600, inactive=39644921856, buffers=408805376, cached=146571251712, shared=5660889088, slab=3207602176) +BEFORE TRAIN LOOP: Epoch 10 | Memory Usage: svmem(total=257568083968, available=145393479680, percent=43.6, used=104263139328, free=6324887552, active=206688665600, inactive=39644921856, buffers=408805376, cached=146571251712, shared=5660889088, slab=3207602176) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=145393479680, percent=43.6, used=104263139328, free=6324887552, active=206688665600, inactive=39644921856, buffers=408805376, cached=146571251712, shared=5660889088, slab=3207602176) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 10 | loss=12.222861289978027 | Batch Time=5.46875 +GPU:1 | Epoch: 10 | loss=12.164102554321289 | Batch Time=8.984375 +GPU:1 | Epoch: 10 | loss=12.335487365722656 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=12.166179656982422 | Batch Time=6.25 +GPU:1 | Epoch: 10 | loss=12.131935119628906 | Batch Time=10.15625 +GPU:1 | Epoch: 10 | loss=12.277926445007324 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.088736534118652 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=12.328235626220703 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=12.23371696472168 | Batch Time=3.515625 +GPU:1 | Epoch: 10 | loss=12.303352355957031 | Batch Time=7.8125 +GPU:1 | Epoch: 10 | loss=11.963785171508789 | Batch Time=11.328125 +GPU:1 | Epoch: 10 | loss=12.145079612731934 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.057672500610352 | Batch Time=11.71875 +GPU:1 | Epoch: 10 | loss=12.308793067932129 | Batch Time=6.640625 +GPU:1 | Epoch: 10 | loss=11.992729187011719 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=12.199979782104492 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=12.199642181396484 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=11.958080291748047 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=12.137426376342773 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=12.072700500488281 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.23497486114502 | Batch Time=3.90625 +GPU:1 | Epoch: 10 | loss=12.1279878616333 | Batch Time=6.25 +GPU:1 | Epoch: 10 | loss=12.154169082641602 | Batch Time=5.078125 +GPU:1 | Epoch: 10 | loss=12.154794692993164 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.128819465637207 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.018407821655273 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.076921463012695 | Batch Time=10.15625 +GPU:1 | Epoch: 10 | loss=12.27298355102539 | Batch Time=7.8125 +GPU:1 | Epoch: 10 | loss=12.267694473266602 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=12.196188926696777 | Batch Time=7.8125 +GPU:1 | Epoch: 10 | loss=12.216405868530273 | Batch Time=7.8125 +GPU:1 | Epoch: 10 | loss=12.335894584655762 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=12.219164848327637 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.059813499450684 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.040307998657227 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=12.046799659729004 | Batch Time=11.71875 +GPU:1 | Epoch: 10 | loss=12.179256439208984 | Batch Time=6.640625 +GPU:1 | Epoch: 10 | loss=12.065084457397461 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.21135139465332 | Batch Time=7.8125 +GPU:1 | Epoch: 10 | loss=11.893973350524902 | Batch Time=12.109375 +GPU:1 | Epoch: 10 | loss=12.207956314086914 | Batch Time=5.46875 +GPU:1 | Epoch: 10 | loss=12.030908584594727 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.189571380615234 | Batch Time=6.25 +GPU:1 | Epoch: 10 | loss=12.21246337890625 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=12.183260917663574 | Batch Time=8.984375 +GPU:1 | Epoch: 10 | loss=12.135080337524414 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=12.129792213439941 | Batch Time=10.546875 +GPU:1 | Epoch: 10 | loss=12.122566223144531 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.218647003173828 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=12.012887954711914 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.136434555053711 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.165621757507324 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.105761528015137 | Batch Time=10.9375 +GPU:1 | Epoch: 10 | loss=12.125555038452148 | Batch Time=10.9375 +GPU:1 | Epoch: 10 | loss=12.096357345581055 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.053997039794922 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.035429954528809 | Batch Time=10.15625 +GPU:1 | Epoch: 10 | loss=12.197809219360352 | Batch Time=4.6875 +GPU:1 | Epoch: 10 | loss=12.132040023803711 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=12.125404357910156 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=12.094985961914062 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=11.879642486572266 | Batch Time=10.15625 +GPU:1 | Epoch: 10 | loss=12.201597213745117 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.073654174804688 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=11.970346450805664 | Batch Time=6.25 +GPU:1 | Epoch: 10 | loss=12.021038055419922 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.192316055297852 | Batch Time=8.984375 +GPU:1 | Epoch: 10 | loss=12.153987884521484 | Batch Time=6.640625 +GPU:1 | Epoch: 10 | loss=12.149307250976562 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=11.979755401611328 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=11.996960639953613 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.143331527709961 | Batch Time=11.328125 +GPU:1 | Epoch: 10 | loss=12.093263626098633 | Batch Time=10.15625 +GPU:1 | Epoch: 10 | loss=12.134353637695312 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.132797241210938 | Batch Time=10.546875 +GPU:1 | Epoch: 10 | loss=12.146888732910156 | Batch Time=5.859375 +GPU:1 | Epoch: 10 | loss=11.999964714050293 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=12.129642486572266 | Batch Time=5.859375 +GPU:1 | Epoch: 10 | loss=12.139518737792969 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=11.919164657592773 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.137252807617188 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=12.12441635131836 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.042449951171875 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=12.007160186767578 | Batch Time=7.8125 +GPU:1 | Epoch: 10 | loss=12.073022842407227 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=12.204898834228516 | Batch Time=5.859375 +GPU:1 | Epoch: 10 | loss=11.929977416992188 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=12.012714385986328 | Batch Time=11.71875 +GPU:1 | Epoch: 10 | loss=12.14002799987793 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.167736053466797 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=11.919146537780762 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=12.185783386230469 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=11.995416641235352 | Batch Time=6.25 +GPU:1 | Epoch: 10 | loss=12.173213958740234 | Batch Time=8.59375 +GPU:1 | Epoch: 10 | loss=12.130008697509766 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=12.157844543457031 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=12.16014289855957 | Batch Time=6.640625 +GPU:1 | Epoch: 10 | loss=12.100790977478027 | Batch Time=10.9375 +GPU:1 | Epoch: 10 | loss=11.90018081665039 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.032083511352539 | Batch Time=8.984375 +GPU:1 | Epoch: 10 | loss=12.123163223266602 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=11.922321319580078 | Batch Time=10.9375 +GPU:1 | Epoch: 10 | loss=12.097375869750977 | Batch Time=7.03125 +GPU:1 | Epoch: 10 | loss=11.936378479003906 | Batch Time=8.984375 +GPU:1 | Epoch: 10 | loss=12.196634292602539 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.029193878173828 | Batch Time=9.765625 +GPU:1 | Epoch: 10 | loss=12.067911148071289 | Batch Time=6.640625 +GPU:1 | Epoch: 10 | loss=12.018106460571289 | Batch Time=6.640625 +GPU:1 | Epoch: 10 | loss=12.089536666870117 | Batch Time=6.640625 +GPU:1 | Epoch: 10 | loss=11.966096878051758 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=12.078784942626953 | Batch Time=6.25 +GPU:1 | Epoch: 10 | loss=11.93497371673584 | Batch Time=9.375 +GPU:1 | Epoch: 10 | loss=11.900781631469727 | Batch Time=10.15625 +GPU:1 | Epoch: 10 | loss=12.133859634399414 | Batch Time=8.984375 +GPU:1 | Epoch: 10 | loss=11.908184051513672 | Batch Time=8.203125 +GPU:1 | Epoch: 10 | loss=11.954273223876953 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=12.093816757202148 | Batch Time=7.8125 +GPU:1 | Epoch: 10 | loss=12.067992210388184 | Batch Time=10.9375 +GPU:1 | Epoch: 10 | loss=11.93096923828125 | Batch Time=11.328125 +GPU:1 | Epoch: 10 | loss=12.033744812011719 | Batch Time=7.8125 +GPU:1 | Epoch: 10 | loss=12.025249481201172 | Batch Time=8.984375 +GPU:1 | Epoch: 10 | loss=11.999083518981934 | Batch Time=11.71875 +GPU:1 | Epoch: 10 | loss=12.094386100769043 | Batch Time=8.984375 +GPU:1 | Epoch: 10 | loss=12.059637069702148 | Batch Time=7.421875 +GPU:1 | Epoch: 10 | loss=11.908292770385742 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.068130493164062 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=12.084806442260742 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=12.095365524291992 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.184944152832031 | Batch Time=6.25 +GPU:3 | Epoch: 10 | loss=12.208459854125977 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.216811180114746 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.371759414672852 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=11.976669311523438 | Batch Time=8.59375 +GPU:3 | Epoch: 10 | loss=12.349374771118164 | Batch Time=6.640625 +GPU:3 | Epoch: 10 | loss=12.139474868774414 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.065105438232422 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.089540481567383 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=12.301752090454102 | Batch Time=5.859375 +GPU:3 | Epoch: 10 | loss=12.081145286560059 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.267839431762695 | Batch Time=4.296875 +GPU:3 | Epoch: 10 | loss=12.293756484985352 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.120532989501953 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.154293060302734 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.196435928344727 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.174610137939453 | Batch Time=6.25 +GPU:3 | Epoch: 10 | loss=12.380906105041504 | Batch Time=4.296875 +GPU:3 | Epoch: 10 | loss=12.090670585632324 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.21954345703125 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.141201972961426 | Batch Time=10.15625 +GPU:3 | Epoch: 10 | loss=12.03397274017334 | Batch Time=11.71875 +GPU:3 | Epoch: 10 | loss=12.138895034790039 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=12.085367202758789 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.215076446533203 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.095415115356445 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.137961387634277 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.122003555297852 | Batch Time=6.640625 +GPU:3 | Epoch: 10 | loss=12.12619686126709 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.21962833404541 | Batch Time=6.25 +GPU:3 | Epoch: 10 | loss=12.132159233093262 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.094888687133789 | Batch Time=8.59375 +GPU:3 | Epoch: 10 | loss=12.307258605957031 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=12.235002517700195 | Batch Time=5.46875 +GPU:3 | Epoch: 10 | loss=12.07191276550293 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.162208557128906 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.204842567443848 | Batch Time=6.640625 +GPU:3 | Epoch: 10 | loss=12.244406700134277 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.194615364074707 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.038074493408203 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.038846969604492 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.200813293457031 | Batch Time=4.6875 +GPU:3 | Epoch: 10 | loss=12.229765892028809 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.130982398986816 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=11.992437362670898 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.056419372558594 | Batch Time=10.9375 +GPU:3 | Epoch: 10 | loss=12.21640396118164 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.151588439941406 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=12.240053176879883 | Batch Time=10.546875 +GPU:3 | Epoch: 10 | loss=11.88476276397705 | Batch Time=10.9375 +GPU:3 | Epoch: 10 | loss=12.117269515991211 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=11.978084564208984 | Batch Time=11.328125 +GPU:3 | Epoch: 10 | loss=12.251495361328125 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.150824546813965 | Batch Time=8.59375 +GPU:3 | Epoch: 10 | loss=12.166702270507812 | Batch Time=10.15625 +GPU:3 | Epoch: 10 | loss=12.055477142333984 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.00246810913086 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.074947357177734 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.100990295410156 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=12.025875091552734 | Batch Time=10.546875 +GPU:3 | Epoch: 10 | loss=12.119608879089355 | Batch Time=8.59375 +GPU:3 | Epoch: 10 | loss=12.250089645385742 | Batch Time=6.25 +GPU:3 | Epoch: 10 | loss=12.125085830688477 | Batch Time=6.640625 +GPU:3 | Epoch: 10 | loss=12.201011657714844 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.133551597595215 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=11.921844482421875 | Batch Time=11.71875 +GPU:3 | Epoch: 10 | loss=12.273048400878906 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.044744491577148 | Batch Time=11.71875 +GPU:3 | Epoch: 10 | loss=12.11056137084961 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.23634147644043 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.164287567138672 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.109989166259766 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=12.251861572265625 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=11.965377807617188 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.146862030029297 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=11.860137939453125 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.203466415405273 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=11.997779846191406 | Batch Time=10.9375 +GPU:3 | Epoch: 10 | loss=12.077461242675781 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.152973175048828 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.252307891845703 | Batch Time=6.25 +GPU:3 | Epoch: 10 | loss=12.130685806274414 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=12.195594787597656 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.179548263549805 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.14542007446289 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.115591049194336 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=11.99929428100586 | Batch Time=10.9375 +GPU:3 | Epoch: 10 | loss=12.058646202087402 | Batch Time=10.9375 +GPU:3 | Epoch: 10 | loss=11.968952178955078 | Batch Time=8.59375 +GPU:3 | Epoch: 10 | loss=12.165604591369629 | Batch Time=5.46875 +GPU:3 | Epoch: 10 | loss=11.903172492980957 | Batch Time=10.15625 +GPU:3 | Epoch: 10 | loss=11.795951843261719 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=12.019491195678711 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=12.081369400024414 | Batch Time=10.15625 +GPU:3 | Epoch: 10 | loss=11.993762969970703 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.025585174560547 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=12.119169235229492 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.04351806640625 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.030332565307617 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=12.082759857177734 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=12.03641128540039 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.076314926147461 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=11.790386199951172 | Batch Time=13.28125 +GPU:3 | Epoch: 10 | loss=11.874167442321777 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=11.919013977050781 | Batch Time=7.8125 +GPU:3 | Epoch: 10 | loss=12.026924133300781 | Batch Time=10.546875 +GPU:3 | Epoch: 10 | loss=11.88453483581543 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.00876235961914 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.061120986938477 | Batch Time=8.203125 +GPU:3 | Epoch: 10 | loss=12.067438125610352 | Batch Time=5.859375 +GPU:3 | Epoch: 10 | loss=12.097383499145508 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=12.052459716796875 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=11.915060043334961 | Batch Time=9.765625 +GPU:3 | Epoch: 10 | loss=11.789773941040039 | Batch Time=12.5 +GPU:3 | Epoch: 10 | loss=12.165130615234375 | Batch Time=7.421875 +GPU:3 | Epoch: 10 | loss=12.105125427246094 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.000191688537598 | Batch Time=6.640625 +GPU:3 | Epoch: 10 | loss=12.052679061889648 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=12.054718971252441 | Batch Time=7.03125 +GPU:3 | Epoch: 10 | loss=11.99846363067627 | Batch Time=9.375 +GPU:3 | Epoch: 10 | loss=12.05482292175293 | Batch Time=8.984375 +GPU:3 | Epoch: 10 | loss=11.864677429199219 | Batch Time=12.109375 +GPU:0 | Epoch: 10 | loss=12.235042572021484 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.213460922241211 | Batch Time=6.25 +GPU:0 | Epoch: 10 | loss=12.307662963867188 | Batch Time=6.25 +GPU:0 | Epoch: 10 | loss=12.303390502929688 | Batch Time=6.640625 +GPU:0 | Epoch: 10 | loss=12.127819061279297 | Batch Time=10.15625 +GPU:0 | Epoch: 10 | loss=12.349344253540039 | Batch Time=6.640625 +GPU:0 | Epoch: 10 | loss=12.244297981262207 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.188047409057617 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=12.33144760131836 | Batch Time=5.46875 +GPU:0 | Epoch: 10 | loss=12.071414947509766 | Batch Time=10.15625 +GPU:0 | Epoch: 10 | loss=12.121902465820312 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=12.076225280761719 | Batch Time=9.765625 +GPU:0 | Epoch: 10 | loss=12.287239074707031 | Batch Time=5.46875 +GPU:0 | Epoch: 10 | loss=12.185677528381348 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=11.962656021118164 | Batch Time=12.109375 +GPU:0 | Epoch: 10 | loss=12.215034484863281 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.09892463684082 | Batch Time=10.546875 +GPU:0 | Epoch: 10 | loss=12.175172805786133 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.187827110290527 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.194965362548828 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.350052833557129 | Batch Time=5.859375 +GPU:0 | Epoch: 10 | loss=12.197405815124512 | Batch Time=7.8125 +GPU:0 | Epoch: 10 | loss=12.068885803222656 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.11532974243164 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=12.168683052062988 | Batch Time=5.078125 +GPU:0 | Epoch: 10 | loss=12.203643798828125 | Batch Time=7.03125 +GPU:0 | Epoch: 10 | loss=12.18779182434082 | Batch Time=5.859375 +GPU:0 | Epoch: 10 | loss=12.100665092468262 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=11.92572021484375 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.257169723510742 | Batch Time=10.546875 +GPU:0 | Epoch: 10 | loss=12.202455520629883 | Batch Time=6.25 +GPU:0 | Epoch: 10 | loss=12.066146850585938 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.173089981079102 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.388155937194824 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=12.11838150024414 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.157220840454102 | Batch Time=7.03125 +GPU:0 | Epoch: 10 | loss=12.031047821044922 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=12.180051803588867 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=12.0044527053833 | Batch Time=10.546875 +GPU:0 | Epoch: 10 | loss=12.2225980758667 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=12.020710945129395 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.276620864868164 | Batch Time=6.25 +GPU:0 | Epoch: 10 | loss=12.348564147949219 | Batch Time=7.03125 +GPU:0 | Epoch: 10 | loss=12.186495780944824 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.061421394348145 | Batch Time=11.328125 +GPU:0 | Epoch: 10 | loss=12.094853401184082 | Batch Time=10.546875 +GPU:0 | Epoch: 10 | loss=12.01114273071289 | Batch Time=5.859375 +GPU:0 | Epoch: 10 | loss=12.303871154785156 | Batch Time=5.46875 +GPU:0 | Epoch: 10 | loss=12.18880844116211 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.25429916381836 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.12385368347168 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.209145545959473 | Batch Time=5.46875 +GPU:0 | Epoch: 10 | loss=12.090608596801758 | Batch Time=10.9375 +GPU:0 | Epoch: 10 | loss=12.171449661254883 | Batch Time=5.46875 +GPU:0 | Epoch: 10 | loss=12.04836368560791 | Batch Time=7.8125 +GPU:0 | Epoch: 10 | loss=12.233062744140625 | Batch Time=10.15625 +GPU:0 | Epoch: 10 | loss=12.078654289245605 | Batch Time=7.8125 +GPU:0 | Epoch: 10 | loss=12.164590835571289 | Batch Time=6.640625 +GPU:0 | Epoch: 10 | loss=11.88565444946289 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.23324966430664 | Batch Time=4.296875 +GPU:0 | Epoch: 10 | loss=12.096881866455078 | Batch Time=10.9375 +GPU:0 | Epoch: 10 | loss=12.071582794189453 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.108037948608398 | Batch Time=11.71875 +GPU:0 | Epoch: 10 | loss=12.040218353271484 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.298654556274414 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=12.219137191772461 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.031965255737305 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.12332534790039 | Batch Time=11.328125 +GPU:0 | Epoch: 10 | loss=12.12301254272461 | Batch Time=6.640625 +GPU:0 | Epoch: 10 | loss=12.07492446899414 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=11.887928009033203 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.154875755310059 | Batch Time=6.640625 +GPU:0 | Epoch: 10 | loss=11.961694717407227 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=12.162042617797852 | Batch Time=9.765625 +GPU:0 | Epoch: 10 | loss=12.150583267211914 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.06123161315918 | Batch Time=7.8125 +GPU:0 | Epoch: 10 | loss=11.99404525756836 | Batch Time=11.328125 +GPU:0 | Epoch: 10 | loss=11.958541870117188 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=11.960025787353516 | Batch Time=12.109375 +GPU:0 | Epoch: 10 | loss=11.957719802856445 | Batch Time=10.9375 +GPU:0 | Epoch: 10 | loss=12.019403457641602 | Batch Time=9.765625 +GPU:0 | Epoch: 10 | loss=12.167875289916992 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=12.084253311157227 | Batch Time=7.03125 +GPU:0 | Epoch: 10 | loss=12.033735275268555 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.082683563232422 | Batch Time=7.8125 +GPU:0 | Epoch: 10 | loss=12.141172409057617 | Batch Time=5.078125 +GPU:0 | Epoch: 10 | loss=12.023504257202148 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.186241149902344 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.026206970214844 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.058622360229492 | Batch Time=10.9375 +GPU:0 | Epoch: 10 | loss=12.071263313293457 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.045722961425781 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=12.088330268859863 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.211016654968262 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=11.95112419128418 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=12.08657455444336 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=11.91996955871582 | Batch Time=10.546875 +GPU:0 | Epoch: 10 | loss=11.847001075744629 | Batch Time=14.84375 +GPU:0 | Epoch: 10 | loss=12.064213752746582 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.014738082885742 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.131500244140625 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.037811279296875 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.128814697265625 | Batch Time=5.46875 +GPU:0 | Epoch: 10 | loss=12.116681098937988 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.087761878967285 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.055181503295898 | Batch Time=6.25 +GPU:0 | Epoch: 10 | loss=11.861574172973633 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.063728332519531 | Batch Time=10.546875 +GPU:0 | Epoch: 10 | loss=11.985885620117188 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=12.074310302734375 | Batch Time=4.6875 +GPU:0 | Epoch: 10 | loss=12.090431213378906 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.06236457824707 | Batch Time=6.640625 +GPU:0 | Epoch: 10 | loss=11.95776081085205 | Batch Time=9.765625 +GPU:0 | Epoch: 10 | loss=11.91754150390625 | Batch Time=9.765625 +GPU:0 | Epoch: 10 | loss=11.907003402709961 | Batch Time=8.984375 +GPU:0 | Epoch: 10 | loss=12.016998291015625 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=12.018281936645508 | Batch Time=8.59375 +GPU:0 | Epoch: 10 | loss=11.904573440551758 | Batch Time=10.9375 +GPU:0 | Epoch: 10 | loss=12.10392951965332 | Batch Time=7.03125 +GPU:0 | Epoch: 10 | loss=12.035966873168945 | Batch Time=5.46875 +GPU:0 | Epoch: 10 | loss=12.068668365478516 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=11.981595993041992 | Batch Time=5.46875 +GPU:0 | Epoch: 10 | loss=12.099932670593262 | Batch Time=8.203125 +GPU:0 | Epoch: 10 | loss=12.068954467773438 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=12.003012657165527 | Batch Time=7.03125 +GPU:2 | Epoch: 10 | loss=12.141033172607422 | Batch Time=13.28125 +GPU:2 | Epoch: 10 | loss=12.123458862304688 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=12.319406509399414 | Batch Time=5.859375 +GPU:2 | Epoch: 10 | loss=12.358510971069336 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.146358489990234 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.118718147277832 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.114065170288086 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=12.239450454711914 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.412277221679688 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.342893600463867 | Batch Time=5.078125 +GPU:2 | Epoch: 10 | loss=12.098197937011719 | Batch Time=11.71875 +GPU:2 | Epoch: 10 | loss=12.307626724243164 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.321004867553711 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.244514465332031 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=12.39547348022461 | Batch Time=5.859375 +GPU:2 | Epoch: 10 | loss=12.190882682800293 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.097424507141113 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.283178329467773 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.145559310913086 | Batch Time=6.25 +GPU:2 | Epoch: 10 | loss=12.313190460205078 | Batch Time=4.6875 +GPU:2 | Epoch: 10 | loss=12.097073554992676 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.230168342590332 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=12.153127670288086 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.017410278320312 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=12.216267585754395 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=12.100666046142578 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.129958152770996 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=11.893848419189453 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=12.337827682495117 | Batch Time=4.6875 +GPU:2 | Epoch: 10 | loss=12.18786334991455 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=12.178651809692383 | Batch Time=5.46875 +GPU:2 | Epoch: 10 | loss=12.18493366241455 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=11.899419784545898 | Batch Time=12.890625 +GPU:2 | Epoch: 10 | loss=12.0869779586792 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.177427291870117 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.405845642089844 | Batch Time=5.46875 +GPU:2 | Epoch: 10 | loss=12.11712646484375 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=12.19260025024414 | Batch Time=5.078125 +GPU:2 | Epoch: 10 | loss=11.941139221191406 | Batch Time=10.546875 +GPU:2 | Epoch: 10 | loss=12.156862258911133 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=12.154455184936523 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.13602066040039 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=11.986095428466797 | Batch Time=10.9375 +GPU:2 | Epoch: 10 | loss=12.075994491577148 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.33565616607666 | Batch Time=6.640625 +GPU:2 | Epoch: 10 | loss=12.162034034729004 | Batch Time=6.640625 +GPU:2 | Epoch: 10 | loss=12.135476112365723 | Batch Time=6.25 +GPU:2 | Epoch: 10 | loss=12.147052764892578 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.017684936523438 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=12.089553833007812 | Batch Time=10.546875 +GPU:2 | Epoch: 10 | loss=11.975393295288086 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=12.337763786315918 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.200989723205566 | Batch Time=6.25 +GPU:2 | Epoch: 10 | loss=12.115114212036133 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=12.227123260498047 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=11.98487663269043 | Batch Time=11.328125 +GPU:2 | Epoch: 10 | loss=12.195651054382324 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=11.902456283569336 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=11.868890762329102 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.18929672241211 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.012048721313477 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=11.997892379760742 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.065057754516602 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.018260955810547 | Batch Time=12.109375 +GPU:2 | Epoch: 10 | loss=12.074901580810547 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=11.982990264892578 | Batch Time=7.03125 +GPU:2 | Epoch: 10 | loss=12.015077590942383 | Batch Time=12.109375 +GPU:2 | Epoch: 10 | loss=12.243959426879883 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.17454719543457 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=12.08687973022461 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=11.814449310302734 | Batch Time=10.546875 +GPU:2 | Epoch: 10 | loss=12.196258544921875 | Batch Time=6.640625 +GPU:2 | Epoch: 10 | loss=12.102225303649902 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.053695678710938 | Batch Time=11.71875 +GPU:2 | Epoch: 10 | loss=11.80053424835205 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=12.159599304199219 | Batch Time=7.03125 +GPU:2 | Epoch: 10 | loss=12.209508895874023 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=12.079586029052734 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.21413803100586 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=11.997049331665039 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=12.173652648925781 | Batch Time=7.03125 +GPU:2 | Epoch: 10 | loss=12.10942268371582 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=12.028236389160156 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=12.15804672241211 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.09727668762207 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.127670288085938 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=12.016279220581055 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=11.986108779907227 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=12.125438690185547 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.042661666870117 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.096263885498047 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=11.95742416381836 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.036986351013184 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.039225578308105 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.039691925048828 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.072105407714844 | Batch Time=9.765625 +GPU:2 | Epoch: 10 | loss=12.120315551757812 | Batch Time=7.03125 +GPU:2 | Epoch: 10 | loss=12.05209732055664 | Batch Time=12.109375 +GPU:2 | Epoch: 10 | loss=12.204465866088867 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=11.966974258422852 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.10521125793457 | Batch Time=9.375 +GPU:2 | Epoch: 10 | loss=12.139192581176758 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.085243225097656 | Batch Time=6.640625 +GPU:2 | Epoch: 10 | loss=12.102418899536133 | Batch Time=7.03125 +GPU:2 | Epoch: 10 | loss=11.879301071166992 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=11.991092681884766 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.020977020263672 | Batch Time=5.078125 +GPU:2 | Epoch: 10 | loss=12.105766296386719 | Batch Time=7.8125 +GPU:2 | Epoch: 10 | loss=11.989965438842773 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.115333557128906 | Batch Time=5.859375 +GPU:2 | Epoch: 10 | loss=11.989482879638672 | Batch Time=10.9375 +GPU:2 | Epoch: 10 | loss=12.095745086669922 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=11.980720520019531 | Batch Time=10.9375 +GPU:2 | Epoch: 10 | loss=12.020753860473633 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=11.900047302246094 | Batch Time=8.59375 +GPU:2 | Epoch: 10 | loss=11.944023132324219 | Batch Time=8.203125 +GPU:2 | Epoch: 10 | loss=12.037080764770508 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=12.061994552612305 | Batch Time=9.765625 +GPU:2 | Epoch: 10 | loss=11.801340103149414 | Batch Time=11.328125 +GPU:2 | Epoch: 10 | loss=11.980297088623047 | Batch Time=7.421875 +GPU:2 | Epoch: 10 | loss=11.984594345092773 | Batch Time=10.15625 +GPU:2 | Epoch: 10 | loss=12.06212329864502 | Batch Time=6.640625 +GPU:2 | Epoch: 10 | loss=11.983565330505371 | Batch Time=9.765625 +GPU:2 | Epoch: 10 | loss=12.089141845703125 | Batch Time=8.984375 +GPU:2 | Epoch: 10 | loss=12.05805778503418 | Batch Time=9.375 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666071552, percent=39.6, used=93989793792, free=17360642048, active=192267911168, inactive=43066056704, buffers=419983360, cached=145797664768, shared=5660856320, slab=3210096640) +AFTER TRAIN LOOP: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666071552, percent=39.6, used=93989793792, free=17360642048, active=192267898880, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210096640) +BEFORE VAL LOOP: GPU: 0 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666587648, percent=39.6, used=93989277696, free=17361158144, active=192267382784, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210096640) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666038784, percent=39.6, used=93989826560, free=17360609280, active=192267923456, inactive=43066056704, buffers=419983360, cached=145797664768, shared=5660856320, slab=3210104832) +AFTER TRAIN LOOP: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666071552, percent=39.6, used=93989793792, free=17360642048, active=192267911168, inactive=43066056704, buffers=419983360, cached=145797664768, shared=5660856320, slab=3210096640) +BEFORE PRUNE: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666071552, percent=39.6, used=93989793792, free=17360642048, active=192267911168, inactive=43066056704, buffers=419983360, cached=145797664768, shared=5660856320, slab=3210096640) +Pruning Model: +AFTER PRUNE: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +STARTING TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +BEFORE TRAIN LOOP: Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666071552, percent=39.6, used=93989793792, free=17360642048, active=192267911168, inactive=43066056704, buffers=419983360, cached=145797664768, shared=5660856320, slab=3210096640) +AFTER TRAIN LOOP: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666071552, percent=39.6, used=93989793792, free=17360642048, active=192267898880, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210096640) +BEFORE PRUNE: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666587648, percent=39.6, used=93989277696, free=17361158144, active=192267382784, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210096640) +Pruning Model: +AFTER PRUNE: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +STARTING TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +BEFORE TRAIN LOOP: Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666071552, percent=39.6, used=93989793792, free=17360642048, active=192267911168, inactive=43066056704, buffers=419983360, cached=145797664768, shared=5660856320, slab=3210096640) +AFTER TRAIN LOOP: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666587648, percent=39.6, used=93989277696, free=17361158144, active=192267382784, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210096640) +BEFORE PRUNE: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666587648, percent=39.6, used=93989277696, free=17361158144, active=192267382784, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210096640) +Pruning Model: +AFTER PRUNE: Epoch 10 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +STARTING TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +BEFORE TRAIN LOOP: Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=155666939904, percent=39.6, used=93988954112, free=17361481728, active=192267321344, inactive=43066056704, buffers=419987456, cached=145797660672, shared=5660856320, slab=3210031104) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 10 | loss=3.9154460430145264 | Batch Time=17.1875 +GPU:0 | Epoch: 10 | loss=5.334771633148193 | Batch Time=5.859375 +GPU:0 | Epoch: 10 | loss=4.437833309173584 | Batch Time=20.3125 +GPU:0 | Epoch: 10 | loss=4.787899494171143 | Batch Time=3.515625 +GPU:0 | Epoch: 10 | loss=5.207398891448975 | Batch Time=1.5625 +GPU:0 | Epoch: 10 | loss=4.584014892578125 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=5.346462249755859 | Batch Time=6.25 +GPU:0 | Epoch: 10 | loss=5.173274040222168 | Batch Time=3.515625 +GPU:0 | Epoch: 10 | loss=5.702432632446289 | Batch Time=2.734375 +GPU:0 | Epoch: 10 | loss=6.184159278869629 | Batch Time=1.171875 +GPU:0 | Epoch: 10 | loss=5.915040493011475 | Batch Time=1.953125 +GPU:0 | Epoch: 10 | loss=5.211398601531982 | Batch Time=10.546875 +GPU:0 | Epoch: 10 | loss=5.685645580291748 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=5.0179057121276855 | Batch Time=16.015625 +GPU:0 | Epoch: 10 | loss=5.220674991607666 | Batch Time=5.859375 +GPU:0 | Epoch: 10 | loss=5.254832744598389 | Batch Time=9.765625 +GPU:0 | Epoch: 10 | loss=5.193588733673096 | Batch Time=7.03125 +GPU:0 | Epoch: 10 | loss=4.287652969360352 | Batch Time=20.3125 +GPU:0 | Epoch: 10 | loss=4.889397621154785 | Batch Time=12.890625 +GPU:0 | Epoch: 10 | loss=3.849865674972534 | Batch Time=25.78125 +Model top1 Accuracy: 9.028 +Acc before rounding: 9.028 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 10 | loss=3.9154460430145264 | Batch Time=17.1875 +GPU:0 | Epoch: 10 | loss=5.334771633148193 | Batch Time=5.859375 +GPU:0 | Epoch: 10 | loss=4.437833309173584 | Batch Time=20.3125 +GPU:0 | Epoch: 10 | loss=4.787899494171143 | Batch Time=3.515625 +GPU:0 | Epoch: 10 | loss=5.207398891448975 | Batch Time=1.5625 +GPU:0 | Epoch: 10 | loss=4.584014892578125 | Batch Time=9.375 +GPU:0 | Epoch: 10 | loss=5.346462249755859 | Batch Time=6.25 +GPU:0 | Epoch: 10 | loss=5.173274040222168 | Batch Time=3.515625 +GPU:0 | Epoch: 10 | loss=5.702432632446289 | Batch Time=2.734375 +GPU:0 | Epoch: 10 | loss=6.184159278869629 | Batch Time=1.171875 +GPU:0 | Epoch: 10 | loss=5.915040493011475 | Batch Time=1.953125 +GPU:0 | Epoch: 10 | loss=5.211398601531982 | Batch Time=10.546875 +GPU:0 | Epoch: 10 | loss=5.685645580291748 | Batch Time=7.421875 +GPU:0 | Epoch: 10 | loss=5.0179057121276855 | Batch Time=16.015625 +GPU:0 | Epoch: 10 | loss=5.220674991607666 | Batch Time=5.859375 +GPU:0 | Epoch: 10 | loss=5.254832744598389 | Batch Time=9.765625 +GPU:0 | Epoch: 10 | loss=5.193588733673096 | Batch Time=7.03125 +GPU:0 | Epoch: 10 | loss=4.287652969360352 | Batch Time=20.3125 +GPU:0 | Epoch: 10 | loss=4.889397621154785 | Batch Time=12.890625 +GPU:0 | Epoch: 10 | loss=3.849865674972534 | Batch Time=25.78125 +Model top1 Accuracy: 9.028 +Acc after rounding: 9.028 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 10 | loss=5.272210597991943 | Batch Time=6.640625 +GPU:0 | Epoch: 10 | loss=5.296005725860596 | Batch Time=9.765625 +GPU:0 | Epoch: 10 | loss=5.09865665435791 | Batch Time=11.328125 +GPU:0 | Epoch: 10 | loss=5.374573707580566 | Batch Time=6.640625 +Model top1 Accuracy: 7.58 +Validation Acc after rounding: 7.58 +AFTER VAL LOOP: GPU: 0 | Epoch 10 | Memory Usage: svmem(total=257568083968, available=137846464512, percent=46.5, used=111809286144, free=5377703936, active=203880345600, inactive=43320623104, buffers=425971712, cached=139955122176, shared=5660999680, slab=3214508032) +BEFORE PRUNE: Epoch 10 | Memory Usage: svmem(total=257568083968, available=137846464512, percent=46.5, used=111809286144, free=5377703936, active=203880345600, inactive=43320623104, buffers=425971712, cached=139955122176, shared=5660999680, slab=3214508032) +Pruning Model: +AFTER PRUNE: Epoch 10 | Memory Usage: svmem(total=257568083968, available=137846788096, percent=46.5, used=111808962560, free=5378027520, active=203880329216, inactive=43320623104, buffers=425971712, cached=139955122176, shared=5660999680, slab=3214442496) +Rounding model with scheme: naive +Model avg sparsity: 44.235470758883956 +GPU:0 | Epoch: 10 | Acc=9.028 | Epoch Time=16.276396083831788 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=137846788096, percent=46.5, used=111808962560, free=5378027520, active=203880329216, inactive=43320623104, buffers=425971712, cached=139955122176, shared=5660999680, slab=3214442496) +STARTING TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=137846788096, percent=46.5, used=111808962560, free=5378027520, active=203880329216, inactive=43320623104, buffers=425971712, cached=139955122176, shared=5660999680, slab=3214442496) +BEFORE TRAIN LOOP: Epoch 11 | Memory Usage: svmem(total=257568083968, available=137846788096, percent=46.5, used=111808962560, free=5378027520, active=203880329216, inactive=43320623104, buffers=425971712, cached=139955122176, shared=5660999680, slab=3214442496) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=137846788096, percent=46.5, used=111808962560, free=5378027520, active=203880329216, inactive=43320623104, buffers=425971712, cached=139955122176, shared=5660999680, slab=3214442496) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 11 | loss=11.88163948059082 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.863443374633789 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=12.002388000488281 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=12.055479049682617 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=12.02720832824707 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=12.215974807739258 | Batch Time=6.640625 +GPU:0 | Epoch: 11 | loss=12.085494995117188 | Batch Time=5.46875 +GPU:0 | Epoch: 11 | loss=12.066754341125488 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=11.90604305267334 | Batch Time=11.71875 +GPU:0 | Epoch: 11 | loss=11.867876052856445 | Batch Time=7.03125 +GPU:0 | Epoch: 11 | loss=11.882331848144531 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=11.947930335998535 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=12.012261390686035 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=12.008126258850098 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=11.901942253112793 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=11.96767807006836 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=12.061445236206055 | Batch Time=7.03125 +GPU:0 | Epoch: 11 | loss=11.906327247619629 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=11.97941780090332 | Batch Time=5.46875 +GPU:0 | Epoch: 11 | loss=11.982645034790039 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=12.053009033203125 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=11.82119369506836 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=12.056926727294922 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=11.998390197753906 | Batch Time=7.03125 +GPU:0 | Epoch: 11 | loss=11.834394454956055 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=12.002151489257812 | Batch Time=5.46875 +GPU:0 | Epoch: 11 | loss=11.940686225891113 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=12.03322982788086 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.972493171691895 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.985403060913086 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=12.028928756713867 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=12.05009651184082 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=11.882364273071289 | Batch Time=6.640625 +GPU:0 | Epoch: 11 | loss=12.007035255432129 | Batch Time=10.15625 +GPU:0 | Epoch: 11 | loss=11.953503608703613 | Batch Time=11.71875 +GPU:0 | Epoch: 11 | loss=11.893535614013672 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=12.011957168579102 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=12.042664527893066 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.867473602294922 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=11.797806739807129 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.915812492370605 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.908088684082031 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=11.875022888183594 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=12.179903030395508 | Batch Time=6.640625 +GPU:0 | Epoch: 11 | loss=11.845834732055664 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=11.869283676147461 | Batch Time=10.15625 +GPU:0 | Epoch: 11 | loss=11.856744766235352 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=11.803720474243164 | Batch Time=10.15625 +GPU:0 | Epoch: 11 | loss=11.954181671142578 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.676136016845703 | Batch Time=12.109375 +GPU:0 | Epoch: 11 | loss=11.904821395874023 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=11.941962242126465 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.86019515991211 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=11.819587707519531 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=12.006902694702148 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=12.074972152709961 | Batch Time=7.03125 +GPU:0 | Epoch: 11 | loss=11.904912948608398 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=11.850564956665039 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=11.880163192749023 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.83858871459961 | Batch Time=11.328125 +GPU:0 | Epoch: 11 | loss=11.943842887878418 | Batch Time=11.71875 +GPU:0 | Epoch: 11 | loss=12.055068016052246 | Batch Time=7.03125 +GPU:0 | Epoch: 11 | loss=11.901809692382812 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.865072250366211 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=11.901517868041992 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.891792297363281 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=12.003799438476562 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=11.763197898864746 | Batch Time=11.71875 +GPU:0 | Epoch: 11 | loss=11.853143692016602 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=11.962240219116211 | Batch Time=12.5 +GPU:0 | Epoch: 11 | loss=11.95415210723877 | Batch Time=11.328125 +GPU:0 | Epoch: 11 | loss=11.812917709350586 | Batch Time=11.71875 +GPU:0 | Epoch: 11 | loss=11.99574089050293 | Batch Time=7.03125 +GPU:0 | Epoch: 11 | loss=11.898191452026367 | Batch Time=5.859375 +GPU:0 | Epoch: 11 | loss=11.903966903686523 | Batch Time=8.203125 +GPU:0 | Epoch: 11 | loss=11.88966178894043 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=12.027946472167969 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.917388916015625 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=11.75644302368164 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.752569198608398 | Batch Time=10.9375 +GPU:0 | Epoch: 11 | loss=11.76291561126709 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.772180557250977 | Batch Time=10.9375 +GPU:0 | Epoch: 11 | loss=11.842466354370117 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=11.900314331054688 | Batch Time=6.640625 +GPU:0 | Epoch: 11 | loss=11.875370025634766 | Batch Time=6.640625 +GPU:0 | Epoch: 11 | loss=11.729085922241211 | Batch Time=14.0625 +GPU:0 | Epoch: 11 | loss=11.758002281188965 | Batch Time=10.9375 +GPU:0 | Epoch: 11 | loss=12.012165069580078 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=11.726045608520508 | Batch Time=11.71875 +GPU:0 | Epoch: 11 | loss=11.8226957321167 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.688618659973145 | Batch Time=12.890625 +GPU:0 | Epoch: 11 | loss=11.805191040039062 | Batch Time=10.15625 +GPU:0 | Epoch: 11 | loss=11.924129486083984 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=11.64937686920166 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.823160171508789 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.843931198120117 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=11.92220687866211 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.66131591796875 | Batch Time=6.640625 +GPU:0 | Epoch: 11 | loss=11.723060607910156 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.954205513000488 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=11.796880722045898 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=11.595739364624023 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.753524780273438 | Batch Time=9.765625 +GPU:0 | Epoch: 11 | loss=11.779035568237305 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.764625549316406 | Batch Time=10.9375 +GPU:0 | Epoch: 11 | loss=11.793248176574707 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=11.723505020141602 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=11.795700073242188 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.949882507324219 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=11.791536331176758 | Batch Time=10.15625 +GPU:0 | Epoch: 11 | loss=11.649211883544922 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.772928237915039 | Batch Time=5.859375 +GPU:0 | Epoch: 11 | loss=11.66455078125 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=11.887393951416016 | Batch Time=12.109375 +GPU:0 | Epoch: 11 | loss=11.957138061523438 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.904499053955078 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.820670127868652 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.821003913879395 | Batch Time=12.109375 +GPU:0 | Epoch: 11 | loss=11.768675804138184 | Batch Time=8.59375 +GPU:0 | Epoch: 11 | loss=11.768377304077148 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=11.753787994384766 | Batch Time=11.328125 +GPU:0 | Epoch: 11 | loss=11.749073028564453 | Batch Time=10.15625 +GPU:0 | Epoch: 11 | loss=11.783995628356934 | Batch Time=12.109375 +GPU:0 | Epoch: 11 | loss=11.80095100402832 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=11.617544174194336 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=12.193309783935547 | Batch Time=6.25 +GPU:1 | Epoch: 11 | loss=11.809617042541504 | Batch Time=10.9375 +GPU:1 | Epoch: 11 | loss=11.900806427001953 | Batch Time=10.546875 +GPU:1 | Epoch: 11 | loss=11.99018669128418 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=12.214193344116211 | Batch Time=6.640625 +GPU:1 | Epoch: 11 | loss=11.978042602539062 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.812095642089844 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=11.925180435180664 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.937278747558594 | Batch Time=8.59375 +GPU:1 | Epoch: 11 | loss=11.96644401550293 | Batch Time=11.71875 +GPU:1 | Epoch: 11 | loss=11.962273597717285 | Batch Time=7.8125 +GPU:1 | Epoch: 11 | loss=11.976264953613281 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.974493980407715 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.94170093536377 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=11.913581848144531 | Batch Time=8.59375 +GPU:1 | Epoch: 11 | loss=11.925909042358398 | Batch Time=7.421875 +GPU:1 | Epoch: 11 | loss=11.99604606628418 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.852087020874023 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=12.05782699584961 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=11.951677322387695 | Batch Time=10.546875 +GPU:1 | Epoch: 11 | loss=11.851190567016602 | Batch Time=10.546875 +GPU:1 | Epoch: 11 | loss=11.973001480102539 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=12.02978515625 | Batch Time=7.421875 +GPU:1 | Epoch: 11 | loss=11.830242156982422 | Batch Time=11.71875 +GPU:1 | Epoch: 11 | loss=11.92326545715332 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=11.992631912231445 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=11.824074745178223 | Batch Time=12.5 +GPU:1 | Epoch: 11 | loss=11.97054672241211 | Batch Time=11.71875 +GPU:1 | Epoch: 11 | loss=11.803885459899902 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.951543807983398 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=12.089667320251465 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.864297866821289 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=12.022132873535156 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=12.084153175354004 | Batch Time=7.03125 +GPU:1 | Epoch: 11 | loss=12.038962364196777 | Batch Time=6.25 +GPU:1 | Epoch: 11 | loss=11.965753555297852 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.915685653686523 | Batch Time=11.328125 +GPU:1 | Epoch: 11 | loss=11.84289836883545 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=12.010091781616211 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.775897026062012 | Batch Time=11.71875 +GPU:1 | Epoch: 11 | loss=11.932095527648926 | Batch Time=8.59375 +GPU:1 | Epoch: 11 | loss=11.62503433227539 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=11.999774932861328 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=12.099388122558594 | Batch Time=7.8125 +GPU:1 | Epoch: 11 | loss=11.760990142822266 | Batch Time=10.9375 +GPU:1 | Epoch: 11 | loss=11.966197967529297 | Batch Time=10.546875 +GPU:1 | Epoch: 11 | loss=11.977931022644043 | Batch Time=5.859375 +GPU:1 | Epoch: 11 | loss=12.02151107788086 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=11.989786148071289 | Batch Time=7.03125 +GPU:1 | Epoch: 11 | loss=11.937596321105957 | Batch Time=6.25 +GPU:1 | Epoch: 11 | loss=11.816556930541992 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=11.786978721618652 | Batch Time=12.890625 +GPU:1 | Epoch: 11 | loss=11.97021484375 | Batch Time=7.03125 +GPU:1 | Epoch: 11 | loss=12.079385757446289 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.789308547973633 | Batch Time=10.546875 +GPU:1 | Epoch: 11 | loss=12.044599533081055 | Batch Time=7.421875 +GPU:1 | Epoch: 11 | loss=11.997474670410156 | Batch Time=7.421875 +GPU:1 | Epoch: 11 | loss=12.002716064453125 | Batch Time=7.421875 +GPU:1 | Epoch: 11 | loss=12.069272994995117 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=12.03085708618164 | Batch Time=7.421875 +GPU:1 | Epoch: 11 | loss=11.86431884765625 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.905163764953613 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=11.913633346557617 | Batch Time=7.03125 +GPU:1 | Epoch: 11 | loss=11.663763046264648 | Batch Time=12.5 +GPU:1 | Epoch: 11 | loss=11.970220565795898 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=11.967206001281738 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.860647201538086 | Batch Time=8.59375 +GPU:1 | Epoch: 11 | loss=11.855864524841309 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=11.894296646118164 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=12.029136657714844 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.55799674987793 | Batch Time=12.890625 +GPU:1 | Epoch: 11 | loss=11.967589378356934 | Batch Time=10.9375 +GPU:1 | Epoch: 11 | loss=11.661449432373047 | Batch Time=14.453125 +GPU:1 | Epoch: 11 | loss=11.738712310791016 | Batch Time=11.71875 +GPU:1 | Epoch: 11 | loss=11.98420524597168 | Batch Time=7.8125 +GPU:1 | Epoch: 11 | loss=11.913759231567383 | Batch Time=10.9375 +GPU:1 | Epoch: 11 | loss=12.03718376159668 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.914741516113281 | Batch Time=10.9375 +GPU:1 | Epoch: 11 | loss=11.705185890197754 | Batch Time=11.71875 +GPU:1 | Epoch: 11 | loss=11.754829406738281 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.789164543151855 | Batch Time=11.328125 +GPU:1 | Epoch: 11 | loss=11.831632614135742 | Batch Time=10.546875 +GPU:1 | Epoch: 11 | loss=11.721752166748047 | Batch Time=10.9375 +GPU:1 | Epoch: 11 | loss=11.81588363647461 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.762662887573242 | Batch Time=11.328125 +GPU:1 | Epoch: 11 | loss=11.636354446411133 | Batch Time=7.8125 +GPU:1 | Epoch: 11 | loss=11.544379234313965 | Batch Time=12.890625 +GPU:1 | Epoch: 11 | loss=11.815023422241211 | Batch Time=7.03125 +GPU:1 | Epoch: 11 | loss=11.915447235107422 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.85619068145752 | Batch Time=7.421875 +GPU:1 | Epoch: 11 | loss=11.807652473449707 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.928337097167969 | Batch Time=8.59375 +GPU:1 | Epoch: 11 | loss=12.005435943603516 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.865955352783203 | Batch Time=7.8125 +GPU:1 | Epoch: 11 | loss=11.944272994995117 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.80587387084961 | Batch Time=11.71875 +GPU:1 | Epoch: 11 | loss=11.736547470092773 | Batch Time=11.328125 +GPU:1 | Epoch: 11 | loss=11.836526870727539 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.766611099243164 | Batch Time=8.59375 +GPU:1 | Epoch: 11 | loss=11.970858573913574 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.79768180847168 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=11.945891380310059 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.746261596679688 | Batch Time=12.109375 +GPU:1 | Epoch: 11 | loss=11.927146911621094 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.65396499633789 | Batch Time=10.546875 +GPU:1 | Epoch: 11 | loss=11.969000816345215 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.887889862060547 | Batch Time=10.546875 +GPU:1 | Epoch: 11 | loss=11.927810668945312 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.665843963623047 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=11.879676818847656 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=11.814485549926758 | Batch Time=9.765625 +GPU:1 | Epoch: 11 | loss=11.73742389678955 | Batch Time=10.9375 +GPU:1 | Epoch: 11 | loss=11.761601448059082 | Batch Time=7.8125 +GPU:1 | Epoch: 11 | loss=11.759014129638672 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.87183952331543 | Batch Time=8.59375 +GPU:1 | Epoch: 11 | loss=11.904279708862305 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.886932373046875 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.889812469482422 | Batch Time=9.375 +GPU:1 | Epoch: 11 | loss=11.739749908447266 | Batch Time=10.15625 +GPU:1 | Epoch: 11 | loss=11.853822708129883 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.699906349182129 | Batch Time=8.59375 +GPU:1 | Epoch: 11 | loss=11.900777816772461 | Batch Time=8.203125 +GPU:1 | Epoch: 11 | loss=11.650300979614258 | Batch Time=10.9375 +GPU:1 | Epoch: 11 | loss=11.850764274597168 | Batch Time=8.984375 +GPU:1 | Epoch: 11 | loss=11.885614395141602 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.811506271362305 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=11.954684257507324 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=12.017401695251465 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=12.074918746948242 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=12.0740966796875 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=11.926933288574219 | Batch Time=5.46875 +GPU:3 | Epoch: 11 | loss=11.9586181640625 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=11.938209533691406 | Batch Time=6.640625 +GPU:3 | Epoch: 11 | loss=11.999512672424316 | Batch Time=10.546875 +GPU:3 | Epoch: 11 | loss=11.84945297241211 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=12.001813888549805 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=11.874837875366211 | Batch Time=12.5 +GPU:3 | Epoch: 11 | loss=11.994222640991211 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.90953540802002 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=12.037408828735352 | Batch Time=6.640625 +GPU:3 | Epoch: 11 | loss=11.748991966247559 | Batch Time=10.9375 +GPU:3 | Epoch: 11 | loss=11.856245040893555 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.79397964477539 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.952238082885742 | Batch Time=6.25 +GPU:3 | Epoch: 11 | loss=12.126602172851562 | Batch Time=6.640625 +GPU:3 | Epoch: 11 | loss=12.04758071899414 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.791681289672852 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.991218566894531 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=12.099128723144531 | Batch Time=4.6875 +GPU:3 | Epoch: 11 | loss=11.842559814453125 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=12.063676834106445 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=12.028420448303223 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=11.91292667388916 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.945988655090332 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=11.921415328979492 | Batch Time=13.671875 +GPU:3 | Epoch: 11 | loss=11.950677871704102 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.988945007324219 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.913824081420898 | Batch Time=12.109375 +GPU:3 | Epoch: 11 | loss=11.73901081085205 | Batch Time=10.546875 +GPU:3 | Epoch: 11 | loss=11.864396095275879 | Batch Time=12.890625 +GPU:3 | Epoch: 11 | loss=11.700103759765625 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.833257675170898 | Batch Time=13.28125 +GPU:3 | Epoch: 11 | loss=12.153336524963379 | Batch Time=5.859375 +GPU:3 | Epoch: 11 | loss=12.098352432250977 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=11.93239688873291 | Batch Time=10.15625 +GPU:3 | Epoch: 11 | loss=11.860480308532715 | Batch Time=11.71875 +GPU:3 | Epoch: 11 | loss=12.00495719909668 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=11.895065307617188 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.947383880615234 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=11.738962173461914 | Batch Time=10.15625 +GPU:3 | Epoch: 11 | loss=11.7255859375 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=12.104540824890137 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=11.902132034301758 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.985819816589355 | Batch Time=10.15625 +GPU:3 | Epoch: 11 | loss=11.78579330444336 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=12.03422737121582 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=11.98000717163086 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=11.899333953857422 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=12.047330856323242 | Batch Time=6.640625 +GPU:3 | Epoch: 11 | loss=12.077630996704102 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=11.898164749145508 | Batch Time=6.640625 +GPU:3 | Epoch: 11 | loss=11.974653244018555 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.865734100341797 | Batch Time=10.546875 +GPU:3 | Epoch: 11 | loss=11.835973739624023 | Batch Time=10.9375 +GPU:3 | Epoch: 11 | loss=11.741179466247559 | Batch Time=13.28125 +GPU:3 | Epoch: 11 | loss=11.731675148010254 | Batch Time=10.9375 +GPU:3 | Epoch: 11 | loss=11.880476951599121 | Batch Time=11.328125 +GPU:3 | Epoch: 11 | loss=11.74647331237793 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=12.016633987426758 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=11.93020248413086 | Batch Time=10.15625 +GPU:3 | Epoch: 11 | loss=11.797757148742676 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.831758499145508 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=11.802221298217773 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.63917350769043 | Batch Time=10.546875 +GPU:3 | Epoch: 11 | loss=11.966562271118164 | Batch Time=7.03125 +GPU:3 | Epoch: 11 | loss=12.224637985229492 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=11.893564224243164 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.760904312133789 | Batch Time=10.15625 +GPU:3 | Epoch: 11 | loss=11.900466918945312 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=12.007949829101562 | Batch Time=6.640625 +GPU:3 | Epoch: 11 | loss=11.996219635009766 | Batch Time=7.03125 +GPU:3 | Epoch: 11 | loss=11.945964813232422 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=11.869096755981445 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=11.742441177368164 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=11.85223388671875 | Batch Time=12.5 +GPU:3 | Epoch: 11 | loss=11.938005447387695 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.833474159240723 | Batch Time=10.546875 +GPU:3 | Epoch: 11 | loss=11.930660247802734 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.820959091186523 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=11.689634323120117 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.888961791992188 | Batch Time=8.203125 +GPU:3 | Epoch: 11 | loss=11.870660781860352 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.796390533447266 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=11.879138946533203 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.904480934143066 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.63646125793457 | Batch Time=14.453125 +GPU:3 | Epoch: 11 | loss=12.085411071777344 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.859716415405273 | Batch Time=10.15625 +GPU:3 | Epoch: 11 | loss=11.818303108215332 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=11.944089889526367 | Batch Time=6.25 +GPU:3 | Epoch: 11 | loss=11.787466049194336 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.72389030456543 | Batch Time=6.25 +GPU:3 | Epoch: 11 | loss=11.672077178955078 | Batch Time=13.28125 +GPU:3 | Epoch: 11 | loss=11.669050216674805 | Batch Time=12.5 +GPU:3 | Epoch: 11 | loss=11.905476570129395 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.633158683776855 | Batch Time=10.546875 +GPU:3 | Epoch: 11 | loss=11.7931547164917 | Batch Time=12.109375 +GPU:3 | Epoch: 11 | loss=11.944465637207031 | Batch Time=6.640625 +GPU:3 | Epoch: 11 | loss=11.822376251220703 | Batch Time=6.25 +GPU:3 | Epoch: 11 | loss=11.832880020141602 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.87840747833252 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=12.010442733764648 | Batch Time=8.59375 +GPU:3 | Epoch: 11 | loss=11.89498519897461 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=11.766159057617188 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=11.758691787719727 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=11.774017333984375 | Batch Time=9.765625 +GPU:3 | Epoch: 11 | loss=11.915996551513672 | Batch Time=6.25 +GPU:3 | Epoch: 11 | loss=11.888566017150879 | Batch Time=9.375 +GPU:3 | Epoch: 11 | loss=11.846498489379883 | Batch Time=7.421875 +GPU:3 | Epoch: 11 | loss=11.823101043701172 | Batch Time=11.71875 +GPU:3 | Epoch: 11 | loss=11.794053077697754 | Batch Time=10.546875 +GPU:3 | Epoch: 11 | loss=11.924734115600586 | Batch Time=7.03125 +GPU:3 | Epoch: 11 | loss=11.751825332641602 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.742974281311035 | Batch Time=11.71875 +GPU:3 | Epoch: 11 | loss=11.662893295288086 | Batch Time=14.0625 +GPU:3 | Epoch: 11 | loss=11.809745788574219 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.645635604858398 | Batch Time=11.328125 +GPU:3 | Epoch: 11 | loss=11.817508697509766 | Batch Time=7.8125 +GPU:3 | Epoch: 11 | loss=11.799897193908691 | Batch Time=8.984375 +GPU:3 | Epoch: 11 | loss=11.547735214233398 | Batch Time=14.0625 +GPU:2 | Epoch: 11 | loss=11.956178665161133 | Batch Time=9.375 +GPU:2 | Epoch: 11 | loss=11.989601135253906 | Batch Time=11.328125 +GPU:2 | Epoch: 11 | loss=12.03713607788086 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=12.096368789672852 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=12.034154891967773 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.869878768920898 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.843694686889648 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=11.976125717163086 | Batch Time=7.03125 +GPU:2 | Epoch: 11 | loss=12.218857765197754 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.958473205566406 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=12.11925983428955 | Batch Time=9.375 +GPU:2 | Epoch: 11 | loss=11.901615142822266 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.897128105163574 | Batch Time=13.28125 +GPU:2 | Epoch: 11 | loss=11.7782564163208 | Batch Time=10.15625 +GPU:2 | Epoch: 11 | loss=11.929914474487305 | Batch Time=10.9375 +GPU:2 | Epoch: 11 | loss=11.856369018554688 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.930601119995117 | Batch Time=10.15625 +GPU:2 | Epoch: 11 | loss=12.053306579589844 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.97623062133789 | Batch Time=10.546875 +GPU:2 | Epoch: 11 | loss=12.004478454589844 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=12.053409576416016 | Batch Time=7.03125 +GPU:2 | Epoch: 11 | loss=11.954817771911621 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.916322708129883 | Batch Time=10.15625 +GPU:2 | Epoch: 11 | loss=11.948835372924805 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=11.908493041992188 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.890249252319336 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=12.00525951385498 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=12.008118629455566 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=11.96357536315918 | Batch Time=7.8125 +GPU:2 | Epoch: 11 | loss=11.853399276733398 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.80900764465332 | Batch Time=11.71875 +GPU:2 | Epoch: 11 | loss=12.009071350097656 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=12.150619506835938 | Batch Time=6.640625 +GPU:2 | Epoch: 11 | loss=11.882542610168457 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=11.856005668640137 | Batch Time=11.328125 +GPU:2 | Epoch: 11 | loss=11.79710578918457 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.908916473388672 | Batch Time=10.15625 +GPU:2 | Epoch: 11 | loss=11.957207679748535 | Batch Time=7.8125 +GPU:2 | Epoch: 11 | loss=12.027785301208496 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.993622779846191 | Batch Time=6.25 +GPU:2 | Epoch: 11 | loss=11.994132041931152 | Batch Time=9.375 +GPU:2 | Epoch: 11 | loss=12.024754524230957 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=12.012897491455078 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.968082427978516 | Batch Time=7.03125 +GPU:2 | Epoch: 11 | loss=11.956802368164062 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=12.04046630859375 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=12.02588939666748 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.841419219970703 | Batch Time=7.8125 +GPU:2 | Epoch: 11 | loss=11.922562599182129 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.94639778137207 | Batch Time=6.640625 +GPU:2 | Epoch: 11 | loss=11.773286819458008 | Batch Time=11.328125 +GPU:2 | Epoch: 11 | loss=11.887035369873047 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=12.051340103149414 | Batch Time=6.25 +GPU:2 | Epoch: 11 | loss=12.055892944335938 | Batch Time=7.03125 +GPU:2 | Epoch: 11 | loss=11.821714401245117 | Batch Time=10.546875 +GPU:2 | Epoch: 11 | loss=11.865035057067871 | Batch Time=5.078125 +GPU:2 | Epoch: 11 | loss=11.816213607788086 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=11.82650375366211 | Batch Time=12.890625 +GPU:2 | Epoch: 11 | loss=11.881715774536133 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.883526802062988 | Batch Time=7.8125 +GPU:2 | Epoch: 11 | loss=11.735575675964355 | Batch Time=11.328125 +GPU:2 | Epoch: 11 | loss=11.931132316589355 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.875131607055664 | Batch Time=7.03125 +GPU:2 | Epoch: 11 | loss=12.146549224853516 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.768760681152344 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=11.969520568847656 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.821891784667969 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=11.846847534179688 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=11.84199333190918 | Batch Time=10.15625 +GPU:2 | Epoch: 11 | loss=11.834365844726562 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.925434112548828 | Batch Time=7.03125 +GPU:2 | Epoch: 11 | loss=11.849180221557617 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=12.110057830810547 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.995901107788086 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.91244125366211 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=12.053010940551758 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=11.97446060180664 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.762687683105469 | Batch Time=11.71875 +GPU:2 | Epoch: 11 | loss=11.779964447021484 | Batch Time=11.328125 +GPU:2 | Epoch: 11 | loss=11.912946701049805 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.907164573669434 | Batch Time=8.203125 +GPU:2 | Epoch: 11 | loss=11.795246124267578 | Batch Time=12.109375 +GPU:2 | Epoch: 11 | loss=11.94537353515625 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.775711059570312 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=12.004528999328613 | Batch Time=7.8125 +GPU:2 | Epoch: 11 | loss=11.799474716186523 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=11.71793270111084 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.945554733276367 | Batch Time=6.640625 +GPU:2 | Epoch: 11 | loss=11.830833435058594 | Batch Time=7.421875 +GPU:2 | Epoch: 11 | loss=11.80965518951416 | Batch Time=10.546875 +GPU:2 | Epoch: 11 | loss=11.934507369995117 | Batch Time=9.375 +GPU:2 | Epoch: 11 | loss=11.774924278259277 | Batch Time=12.109375 +GPU:2 | Epoch: 11 | loss=11.772001266479492 | Batch Time=10.9375 +GPU:2 | Epoch: 11 | loss=11.728503227233887 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=11.990890502929688 | Batch Time=7.8125 +GPU:2 | Epoch: 11 | loss=11.87746524810791 | Batch Time=8.984375 +GPU:2 | Epoch: 11 | loss=11.829095840454102 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.714146614074707 | Batch Time=9.765625 +GPU:2 | Epoch: 11 | loss=11.797309875488281 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.92458438873291 | Batch Time=6.25 +GPU:2 | Epoch: 11 | loss=11.893033981323242 | Batch Time=9.375 +GPU:2 | Epoch: 11 | loss=11.891020774841309 | Batch Time=10.9375 +GPU:2 | Epoch: 11 | loss=11.841096878051758 | Batch Time=12.5 +GPU:2 | Epoch: 11 | loss=11.740812301635742 | Batch Time=12.5 +GPU:2 | Epoch: 11 | loss=11.731667518615723 | Batch Time=11.71875 +GPU:2 | Epoch: 11 | loss=11.777769088745117 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.753280639648438 | Batch Time=10.9375 +GPU:2 | Epoch: 11 | loss=11.967653274536133 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.661338806152344 | Batch Time=7.03125 +GPU:2 | Epoch: 11 | loss=11.879117012023926 | Batch Time=9.375 +GPU:2 | Epoch: 11 | loss=11.84471321105957 | Batch Time=10.15625 +GPU:2 | Epoch: 11 | loss=11.743733406066895 | Batch Time=10.546875 +GPU:2 | Epoch: 11 | loss=11.752638816833496 | Batch Time=9.375 +GPU:2 | Epoch: 11 | loss=12.055225372314453 | Batch Time=3.90625 +GPU:2 | Epoch: 11 | loss=11.764373779296875 | Batch Time=10.546875 +GPU:2 | Epoch: 11 | loss=11.786443710327148 | Batch Time=10.15625 +GPU:2 | Epoch: 11 | loss=11.737566947937012 | Batch Time=15.234375 +GPU:2 | Epoch: 11 | loss=11.547981262207031 | Batch Time=12.890625 +GPU:2 | Epoch: 11 | loss=11.668686866760254 | Batch Time=10.546875 +GPU:2 | Epoch: 11 | loss=11.812952041625977 | Batch Time=9.375 +GPU:2 | Epoch: 11 | loss=11.87228775024414 | Batch Time=11.328125 +GPU:2 | Epoch: 11 | loss=11.607375144958496 | Batch Time=13.28125 +GPU:2 | Epoch: 11 | loss=11.730649948120117 | Batch Time=11.328125 +GPU:2 | Epoch: 11 | loss=11.791402816772461 | Batch Time=8.59375 +GPU:2 | Epoch: 11 | loss=11.712331771850586 | Batch Time=10.546875 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118372352, percent=42.5, used=101537415168, free=17405251584, active=186837999616, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218116608) +AFTER TRAIN LOOP: Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +STARTING TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +BEFORE TRAIN LOOP: Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118372352, percent=42.5, used=101537415168, free=17405251584, active=186837999616, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218116608) +AFTER TRAIN LOOP: Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +BEFORE VAL LOOP: GPU: 0 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118372352, percent=42.5, used=101537415168, free=17405251584, active=186837999616, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218116608) +AFTER TRAIN LOOP: Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +STARTING TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +BEFORE TRAIN LOOP: Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118372352, percent=42.5, used=101537415168, free=17405251584, active=186837999616, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218116608) +AFTER TRAIN LOOP: Epoch 11 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +STARTING TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +BEFORE TRAIN LOOP: Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=148118896640, percent=42.5, used=101536890880, free=17405775872, active=186837946368, inactive=48369029120, buffers=430100480, cached=138195316736, shared=5660966912, slab=3218108416) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 11 | loss=4.497452259063721 | Batch Time=13.671875 +GPU:0 | Epoch: 11 | loss=5.444174289703369 | Batch Time=1.5625 +GPU:0 | Epoch: 11 | loss=4.89951753616333 | Batch Time=17.96875 +GPU:0 | Epoch: 11 | loss=4.2246012687683105 | Batch Time=15.234375 +GPU:0 | Epoch: 11 | loss=4.924347877502441 | Batch Time=3.125 +GPU:0 | Epoch: 11 | loss=4.596309185028076 | Batch Time=6.640625 +GPU:0 | Epoch: 11 | loss=5.6670122146606445 | Batch Time=1.953125 +GPU:0 | Epoch: 11 | loss=5.279356956481934 | Batch Time=2.734375 +GPU:0 | Epoch: 11 | loss=5.80866003036499 | Batch Time=1.5625 +GPU:0 | Epoch: 11 | loss=6.157575607299805 | Batch Time=1.171875 +GPU:0 | Epoch: 11 | loss=5.908509731292725 | Batch Time=0.0 +GPU:0 | Epoch: 11 | loss=5.257696151733398 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=5.569103240966797 | Batch Time=10.15625 +GPU:0 | Epoch: 11 | loss=5.1570258140563965 | Batch Time=13.28125 +GPU:0 | Epoch: 11 | loss=5.294081687927246 | Batch Time=3.90625 +GPU:0 | Epoch: 11 | loss=5.318950176239014 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=5.182179927825928 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=4.672565460205078 | Batch Time=14.453125 +GPU:0 | Epoch: 11 | loss=4.795968532562256 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=4.8914713859558105 | Batch Time=16.015625 +Model top1 Accuracy: 8.026 +Acc before rounding: 8.026 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 11 | loss=4.497452259063721 | Batch Time=13.671875 +GPU:0 | Epoch: 11 | loss=5.444174289703369 | Batch Time=1.5625 +GPU:0 | Epoch: 11 | loss=4.89951753616333 | Batch Time=17.96875 +GPU:0 | Epoch: 11 | loss=4.2246012687683105 | Batch Time=15.234375 +GPU:0 | Epoch: 11 | loss=4.924347877502441 | Batch Time=3.125 +GPU:0 | Epoch: 11 | loss=4.596309185028076 | Batch Time=6.640625 +GPU:0 | Epoch: 11 | loss=5.6670122146606445 | Batch Time=1.953125 +GPU:0 | Epoch: 11 | loss=5.279356956481934 | Batch Time=2.734375 +GPU:0 | Epoch: 11 | loss=5.80866003036499 | Batch Time=1.5625 +GPU:0 | Epoch: 11 | loss=6.157575607299805 | Batch Time=1.171875 +GPU:0 | Epoch: 11 | loss=5.908509731292725 | Batch Time=0.0 +GPU:0 | Epoch: 11 | loss=5.257696151733398 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=5.569103240966797 | Batch Time=10.15625 +GPU:0 | Epoch: 11 | loss=5.1570258140563965 | Batch Time=13.28125 +GPU:0 | Epoch: 11 | loss=5.294081687927246 | Batch Time=3.90625 +GPU:0 | Epoch: 11 | loss=5.318950176239014 | Batch Time=9.375 +GPU:0 | Epoch: 11 | loss=5.182179927825928 | Batch Time=7.421875 +GPU:0 | Epoch: 11 | loss=4.672565460205078 | Batch Time=14.453125 +GPU:0 | Epoch: 11 | loss=4.795968532562256 | Batch Time=10.546875 +GPU:0 | Epoch: 11 | loss=4.8914713859558105 | Batch Time=16.015625 +Model top1 Accuracy: 8.026 +Acc after rounding: 8.026 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 11 | loss=5.530457019805908 | Batch Time=7.8125 +GPU:0 | Epoch: 11 | loss=5.491560459136963 | Batch Time=8.984375 +GPU:0 | Epoch: 11 | loss=5.4356560707092285 | Batch Time=6.25 +GPU:0 | Epoch: 11 | loss=5.547266006469727 | Batch Time=5.46875 +Model top1 Accuracy: 6.76 +Validation Acc after rounding: 6.76 +AFTER VAL LOOP: GPU: 0 | Epoch 11 | Memory Usage: svmem(total=257568083968, available=130266038272, percent=49.4, used=119389511680, free=5457575936, active=196272271360, inactive=50769043456, buffers=448114688, cached=132272881664, shared=5661114368, slab=3223994368) +Rounding model with scheme: naive +Model avg sparsity: 43.690217337755016 +GPU:0 | Epoch: 11 | Acc=8.026 | Epoch Time=17.619495964050294 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=130266038272, percent=49.4, used=119389511680, free=5457575936, active=196272271360, inactive=50769043456, buffers=448217088, cached=132272779264, shared=5661114368, slab=3223994368) +STARTING TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=130266038272, percent=49.4, used=119389511680, free=5457575936, active=196272271360, inactive=50769043456, buffers=448217088, cached=132272779264, shared=5661114368, slab=3223994368) +BEFORE TRAIN LOOP: Epoch 12 | Memory Usage: svmem(total=257568083968, available=130266038272, percent=49.4, used=119389511680, free=5457575936, active=196272271360, inactive=50769043456, buffers=448217088, cached=132272779264, shared=5661114368, slab=3223994368) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=130266038272, percent=49.4, used=119389511680, free=5457575936, active=196272271360, inactive=50769043456, buffers=448217088, cached=132272779264, shared=5661114368, slab=3223994368) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:3 | Epoch: 12 | loss=11.815813064575195 | Batch Time=8.59375 +GPU:3 | Epoch: 12 | loss=11.710421562194824 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.748992919921875 | Batch Time=9.375 +GPU:3 | Epoch: 12 | loss=11.696858406066895 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.531253814697266 | Batch Time=14.0625 +GPU:3 | Epoch: 12 | loss=11.820505142211914 | Batch Time=6.25 +GPU:3 | Epoch: 12 | loss=11.812871932983398 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.659608840942383 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.707576751708984 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.934776306152344 | Batch Time=6.640625 +GPU:3 | Epoch: 12 | loss=11.655542373657227 | Batch Time=10.546875 +GPU:3 | Epoch: 12 | loss=11.750661849975586 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.795551300048828 | Batch Time=14.0625 +GPU:3 | Epoch: 12 | loss=11.654121398925781 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.683551788330078 | Batch Time=10.15625 +GPU:3 | Epoch: 12 | loss=11.871413230895996 | Batch Time=12.109375 +GPU:3 | Epoch: 12 | loss=11.784414291381836 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.753559112548828 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.624252319335938 | Batch Time=8.59375 +GPU:3 | Epoch: 12 | loss=11.793025970458984 | Batch Time=6.25 +GPU:3 | Epoch: 12 | loss=11.58102798461914 | Batch Time=15.625 +GPU:3 | Epoch: 12 | loss=11.687748908996582 | Batch Time=10.546875 +GPU:3 | Epoch: 12 | loss=11.72584342956543 | Batch Time=10.9375 +GPU:3 | Epoch: 12 | loss=11.86148452758789 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.82573127746582 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.70674991607666 | Batch Time=13.28125 +GPU:3 | Epoch: 12 | loss=11.726408004760742 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.683914184570312 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.721658706665039 | Batch Time=9.375 +GPU:3 | Epoch: 12 | loss=11.583677291870117 | Batch Time=14.84375 +GPU:3 | Epoch: 12 | loss=11.662339210510254 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.814552307128906 | Batch Time=6.640625 +GPU:3 | Epoch: 12 | loss=11.548421859741211 | Batch Time=12.109375 +GPU:3 | Epoch: 12 | loss=11.678586959838867 | Batch Time=10.9375 +GPU:3 | Epoch: 12 | loss=11.604653358459473 | Batch Time=11.71875 +GPU:3 | Epoch: 12 | loss=11.867130279541016 | Batch Time=8.203125 +GPU:3 | Epoch: 12 | loss=11.689069747924805 | Batch Time=13.671875 +GPU:3 | Epoch: 12 | loss=11.534990310668945 | Batch Time=12.5 +GPU:3 | Epoch: 12 | loss=11.549696922302246 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.751886367797852 | Batch Time=7.03125 +GPU:3 | Epoch: 12 | loss=11.672660827636719 | Batch Time=9.375 +GPU:3 | Epoch: 12 | loss=11.786490440368652 | Batch Time=8.59375 +GPU:3 | Epoch: 12 | loss=11.746177673339844 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.772722244262695 | Batch Time=10.15625 +GPU:3 | Epoch: 12 | loss=11.62044906616211 | Batch Time=10.15625 +GPU:3 | Epoch: 12 | loss=11.770512580871582 | Batch Time=8.203125 +GPU:3 | Epoch: 12 | loss=11.77798080444336 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.655143737792969 | Batch Time=8.59375 +GPU:3 | Epoch: 12 | loss=11.773902893066406 | Batch Time=6.640625 +GPU:3 | Epoch: 12 | loss=11.622629165649414 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.706809997558594 | Batch Time=11.71875 +GPU:3 | Epoch: 12 | loss=11.859180450439453 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.763708114624023 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.677896499633789 | Batch Time=6.25 +GPU:3 | Epoch: 12 | loss=11.550331115722656 | Batch Time=9.375 +GPU:3 | Epoch: 12 | loss=11.832143783569336 | Batch Time=8.203125 +GPU:3 | Epoch: 12 | loss=11.556843757629395 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.550348281860352 | Batch Time=10.9375 +GPU:3 | Epoch: 12 | loss=11.546090126037598 | Batch Time=13.28125 +GPU:3 | Epoch: 12 | loss=11.807373046875 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.510636329650879 | Batch Time=14.84375 +GPU:3 | Epoch: 12 | loss=11.664573669433594 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.583939552307129 | Batch Time=11.71875 +GPU:3 | Epoch: 12 | loss=11.88237190246582 | Batch Time=7.421875 +GPU:3 | Epoch: 12 | loss=11.755570411682129 | Batch Time=7.8125 +GPU:3 | Epoch: 12 | loss=11.527460098266602 | Batch Time=14.453125 +GPU:3 | Epoch: 12 | loss=11.569870948791504 | Batch Time=12.5 +GPU:3 | Epoch: 12 | loss=11.733224868774414 | Batch Time=7.03125 +GPU:3 | Epoch: 12 | loss=11.724197387695312 | Batch Time=10.9375 +GPU:3 | Epoch: 12 | loss=11.687549591064453 | Batch Time=12.5 +GPU:3 | Epoch: 12 | loss=11.84780502319336 | Batch Time=6.25 +GPU:3 | Epoch: 12 | loss=11.551443099975586 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.721229553222656 | Batch Time=12.109375 +GPU:3 | Epoch: 12 | loss=11.667987823486328 | Batch Time=6.640625 +GPU:3 | Epoch: 12 | loss=11.578878402709961 | Batch Time=9.375 +GPU:3 | Epoch: 12 | loss=11.468015670776367 | Batch Time=10.9375 +GPU:3 | Epoch: 12 | loss=11.549217224121094 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.613380432128906 | Batch Time=10.546875 +GPU:3 | Epoch: 12 | loss=11.719135284423828 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.790075302124023 | Batch Time=9.375 +GPU:3 | Epoch: 12 | loss=11.691242218017578 | Batch Time=10.15625 +GPU:3 | Epoch: 12 | loss=11.669771194458008 | Batch Time=12.109375 +GPU:3 | Epoch: 12 | loss=11.639966011047363 | Batch Time=11.71875 +GPU:3 | Epoch: 12 | loss=11.78326416015625 | Batch Time=8.203125 +GPU:3 | Epoch: 12 | loss=11.497674942016602 | Batch Time=10.15625 +GPU:3 | Epoch: 12 | loss=11.740531921386719 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.652833938598633 | Batch Time=5.859375 +GPU:3 | Epoch: 12 | loss=11.563828468322754 | Batch Time=12.109375 +GPU:3 | Epoch: 12 | loss=11.605110168457031 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.733917236328125 | Batch Time=11.71875 +GPU:3 | Epoch: 12 | loss=11.759538650512695 | Batch Time=5.46875 +GPU:3 | Epoch: 12 | loss=11.463743209838867 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.416492462158203 | Batch Time=11.71875 +GPU:3 | Epoch: 12 | loss=11.560001373291016 | Batch Time=11.71875 +GPU:3 | Epoch: 12 | loss=11.47313117980957 | Batch Time=8.59375 +GPU:3 | Epoch: 12 | loss=11.718141555786133 | Batch Time=10.15625 +GPU:3 | Epoch: 12 | loss=11.528614044189453 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.644309997558594 | Batch Time=8.59375 +GPU:3 | Epoch: 12 | loss=11.60736083984375 | Batch Time=10.546875 +GPU:3 | Epoch: 12 | loss=11.74286937713623 | Batch Time=7.421875 +GPU:3 | Epoch: 12 | loss=11.750282287597656 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.895427703857422 | Batch Time=10.15625 +GPU:3 | Epoch: 12 | loss=11.71025276184082 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.745473861694336 | Batch Time=10.546875 +GPU:3 | Epoch: 12 | loss=11.55457878112793 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.588690757751465 | Batch Time=12.109375 +GPU:3 | Epoch: 12 | loss=11.380752563476562 | Batch Time=10.9375 +GPU:3 | Epoch: 12 | loss=11.820032119750977 | Batch Time=5.859375 +GPU:3 | Epoch: 12 | loss=11.5960054397583 | Batch Time=8.203125 +GPU:3 | Epoch: 12 | loss=11.543624877929688 | Batch Time=8.59375 +GPU:3 | Epoch: 12 | loss=11.452104568481445 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.474588394165039 | Batch Time=12.109375 +GPU:3 | Epoch: 12 | loss=11.744449615478516 | Batch Time=7.03125 +GPU:3 | Epoch: 12 | loss=11.596698760986328 | Batch Time=10.546875 +GPU:3 | Epoch: 12 | loss=11.525978088378906 | Batch Time=8.203125 +GPU:3 | Epoch: 12 | loss=11.542262077331543 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.590548515319824 | Batch Time=11.328125 +GPU:3 | Epoch: 12 | loss=11.510919570922852 | Batch Time=9.765625 +GPU:3 | Epoch: 12 | loss=11.639366149902344 | Batch Time=10.15625 +GPU:3 | Epoch: 12 | loss=11.752279281616211 | Batch Time=7.421875 +GPU:3 | Epoch: 12 | loss=11.696775436401367 | Batch Time=7.03125 +GPU:3 | Epoch: 12 | loss=11.668210983276367 | Batch Time=8.984375 +GPU:3 | Epoch: 12 | loss=11.444681167602539 | Batch Time=13.28125 +GPU:3 | Epoch: 12 | loss=11.462328910827637 | Batch Time=11.71875 +GPU:3 | Epoch: 12 | loss=11.648554801940918 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.84507942199707 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.685023307800293 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.88785171508789 | Batch Time=7.8125 +GPU:2 | Epoch: 12 | loss=11.739542961120605 | Batch Time=8.59375 +GPU:2 | Epoch: 12 | loss=11.811223983764648 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.809398651123047 | Batch Time=6.25 +GPU:2 | Epoch: 12 | loss=11.845117568969727 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.694664001464844 | Batch Time=8.984375 +GPU:2 | Epoch: 12 | loss=11.747669219970703 | Batch Time=15.625 +GPU:2 | Epoch: 12 | loss=11.696533203125 | Batch Time=12.5 +GPU:2 | Epoch: 12 | loss=11.630403518676758 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.760449409484863 | Batch Time=11.328125 +GPU:2 | Epoch: 12 | loss=11.75433349609375 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.874332427978516 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.771172523498535 | Batch Time=10.9375 +GPU:2 | Epoch: 12 | loss=11.635083198547363 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.753681182861328 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.842147827148438 | Batch Time=7.8125 +GPU:2 | Epoch: 12 | loss=11.541034698486328 | Batch Time=14.453125 +GPU:2 | Epoch: 12 | loss=11.579965591430664 | Batch Time=7.421875 +GPU:2 | Epoch: 12 | loss=11.745176315307617 | Batch Time=11.328125 +GPU:2 | Epoch: 12 | loss=11.791499137878418 | Batch Time=8.984375 +GPU:2 | Epoch: 12 | loss=11.825929641723633 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.657581329345703 | Batch Time=7.8125 +GPU:2 | Epoch: 12 | loss=11.446565628051758 | Batch Time=14.0625 +GPU:2 | Epoch: 12 | loss=11.634540557861328 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.756324768066406 | Batch Time=8.984375 +GPU:2 | Epoch: 12 | loss=11.673686981201172 | Batch Time=8.984375 +GPU:2 | Epoch: 12 | loss=11.704858779907227 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.655343055725098 | Batch Time=5.859375 +GPU:2 | Epoch: 12 | loss=11.862717628479004 | Batch Time=6.25 +GPU:2 | Epoch: 12 | loss=11.723625183105469 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.83928394317627 | Batch Time=8.984375 +GPU:2 | Epoch: 12 | loss=11.800561904907227 | Batch Time=8.59375 +GPU:2 | Epoch: 12 | loss=11.581238746643066 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.645790100097656 | Batch Time=15.234375 +GPU:2 | Epoch: 12 | loss=11.785154342651367 | Batch Time=9.765625 +GPU:2 | Epoch: 12 | loss=11.612607955932617 | Batch Time=12.109375 +GPU:2 | Epoch: 12 | loss=11.828129768371582 | Batch Time=12.890625 +GPU:2 | Epoch: 12 | loss=11.69439697265625 | Batch Time=6.640625 +GPU:2 | Epoch: 12 | loss=11.807476043701172 | Batch Time=8.984375 +GPU:2 | Epoch: 12 | loss=11.660957336425781 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.81355094909668 | Batch Time=11.328125 +GPU:2 | Epoch: 12 | loss=11.658149719238281 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.532791137695312 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.697096824645996 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.769363403320312 | Batch Time=7.8125 +GPU:2 | Epoch: 12 | loss=11.707979202270508 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.878668785095215 | Batch Time=7.03125 +GPU:2 | Epoch: 12 | loss=11.690945625305176 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.608097076416016 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.839263916015625 | Batch Time=7.8125 +GPU:2 | Epoch: 12 | loss=11.648233413696289 | Batch Time=9.765625 +GPU:2 | Epoch: 12 | loss=11.702259063720703 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.75992202758789 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.739364624023438 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.652949333190918 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.734260559082031 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.752955436706543 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.538726806640625 | Batch Time=8.984375 +GPU:2 | Epoch: 12 | loss=11.595597267150879 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.700826644897461 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.658537864685059 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.573036193847656 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.800983428955078 | Batch Time=7.421875 +GPU:2 | Epoch: 12 | loss=11.681314468383789 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.543353080749512 | Batch Time=9.765625 +GPU:2 | Epoch: 12 | loss=11.548349380493164 | Batch Time=11.328125 +GPU:2 | Epoch: 12 | loss=11.705053329467773 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.770723342895508 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.69649887084961 | Batch Time=7.03125 +GPU:2 | Epoch: 12 | loss=11.730255126953125 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.704971313476562 | Batch Time=8.59375 +GPU:2 | Epoch: 12 | loss=11.808387756347656 | Batch Time=7.8125 +GPU:2 | Epoch: 12 | loss=11.718326568603516 | Batch Time=11.328125 +GPU:2 | Epoch: 12 | loss=11.545343399047852 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.656583786010742 | Batch Time=8.59375 +GPU:2 | Epoch: 12 | loss=11.408167839050293 | Batch Time=14.0625 +GPU:2 | Epoch: 12 | loss=11.52658462524414 | Batch Time=11.328125 +GPU:2 | Epoch: 12 | loss=11.52054214477539 | Batch Time=13.671875 +GPU:2 | Epoch: 12 | loss=11.577268600463867 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.719560623168945 | Batch Time=10.9375 +GPU:2 | Epoch: 12 | loss=11.717605590820312 | Batch Time=11.328125 +GPU:2 | Epoch: 12 | loss=11.564220428466797 | Batch Time=12.109375 +GPU:2 | Epoch: 12 | loss=11.522077560424805 | Batch Time=12.890625 +GPU:2 | Epoch: 12 | loss=11.695972442626953 | Batch Time=8.59375 +GPU:2 | Epoch: 12 | loss=11.754650115966797 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.643009185791016 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.643224716186523 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.437167167663574 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.580619812011719 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.585098266601562 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.744682312011719 | Batch Time=12.5 +GPU:2 | Epoch: 12 | loss=11.584297180175781 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.590248107910156 | Batch Time=7.421875 +GPU:2 | Epoch: 12 | loss=11.444011688232422 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.771665573120117 | Batch Time=7.8125 +GPU:2 | Epoch: 12 | loss=11.586703300476074 | Batch Time=6.640625 +GPU:2 | Epoch: 12 | loss=11.639749526977539 | Batch Time=9.375 +GPU:2 | Epoch: 12 | loss=11.540363311767578 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.670181274414062 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.589410781860352 | Batch Time=10.15625 +GPU:2 | Epoch: 12 | loss=11.824777603149414 | Batch Time=7.421875 +GPU:2 | Epoch: 12 | loss=11.59946060180664 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.525280952453613 | Batch Time=13.671875 +GPU:2 | Epoch: 12 | loss=11.61454963684082 | Batch Time=9.765625 +GPU:2 | Epoch: 12 | loss=11.681297302246094 | Batch Time=7.03125 +GPU:2 | Epoch: 12 | loss=11.559486389160156 | Batch Time=9.765625 +GPU:2 | Epoch: 12 | loss=11.803363800048828 | Batch Time=8.59375 +GPU:2 | Epoch: 12 | loss=11.913564682006836 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.455527305603027 | Batch Time=12.5 +GPU:2 | Epoch: 12 | loss=11.54646110534668 | Batch Time=11.71875 +GPU:2 | Epoch: 12 | loss=11.566448211669922 | Batch Time=8.59375 +GPU:2 | Epoch: 12 | loss=11.520055770874023 | Batch Time=7.8125 +GPU:2 | Epoch: 12 | loss=11.387062072753906 | Batch Time=12.109375 +GPU:2 | Epoch: 12 | loss=11.731956481933594 | Batch Time=9.765625 +GPU:2 | Epoch: 12 | loss=11.55739974975586 | Batch Time=8.984375 +GPU:2 | Epoch: 12 | loss=11.555401802062988 | Batch Time=10.9375 +GPU:2 | Epoch: 12 | loss=11.810646057128906 | Batch Time=8.203125 +GPU:2 | Epoch: 12 | loss=11.37852668762207 | Batch Time=13.671875 +GPU:2 | Epoch: 12 | loss=11.60909652709961 | Batch Time=7.421875 +GPU:2 | Epoch: 12 | loss=11.820718765258789 | Batch Time=7.421875 +GPU:2 | Epoch: 12 | loss=11.616270065307617 | Batch Time=10.546875 +GPU:2 | Epoch: 12 | loss=11.467548370361328 | Batch Time=12.109375 +GPU:2 | Epoch: 12 | loss=11.803620338439941 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.743045806884766 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.717911720275879 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.846651077270508 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.886919975280762 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.719642639160156 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.737201690673828 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.693876266479492 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.837987899780273 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.69107437133789 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.611224174499512 | Batch Time=12.890625 +GPU:1 | Epoch: 12 | loss=11.817268371582031 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.798078536987305 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.706083297729492 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.620560646057129 | Batch Time=12.5 +GPU:1 | Epoch: 12 | loss=11.722174644470215 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.594244956970215 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.5858154296875 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.708118438720703 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.91840934753418 | Batch Time=9.375 +GPU:1 | Epoch: 12 | loss=11.816734313964844 | Batch Time=6.640625 +GPU:1 | Epoch: 12 | loss=11.767496109008789 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.6363525390625 | Batch Time=11.71875 +GPU:1 | Epoch: 12 | loss=11.653785705566406 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.711797714233398 | Batch Time=13.28125 +GPU:1 | Epoch: 12 | loss=11.854911804199219 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=12.035589218139648 | Batch Time=6.640625 +GPU:1 | Epoch: 12 | loss=11.719646453857422 | Batch Time=12.109375 +GPU:1 | Epoch: 12 | loss=11.815793991088867 | Batch Time=9.375 +GPU:1 | Epoch: 12 | loss=11.808053970336914 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.74973201751709 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.749810218811035 | Batch Time=7.8125 +GPU:1 | Epoch: 12 | loss=11.726762771606445 | Batch Time=7.8125 +GPU:1 | Epoch: 12 | loss=11.632100105285645 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.792509078979492 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.634475708007812 | Batch Time=12.109375 +GPU:1 | Epoch: 12 | loss=11.634380340576172 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.689382553100586 | Batch Time=7.8125 +GPU:1 | Epoch: 12 | loss=11.792425155639648 | Batch Time=7.421875 +GPU:1 | Epoch: 12 | loss=11.81640911102295 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.651914596557617 | Batch Time=12.5 +GPU:1 | Epoch: 12 | loss=11.730414390563965 | Batch Time=11.71875 +GPU:1 | Epoch: 12 | loss=11.64668083190918 | Batch Time=13.28125 +GPU:1 | Epoch: 12 | loss=11.721307754516602 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.54963493347168 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.51991081237793 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.583185195922852 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.750337600708008 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.58993911743164 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.653644561767578 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.760586738586426 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.758630752563477 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.81265640258789 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.708712577819824 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.628555297851562 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.76240348815918 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.664037704467773 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.658245086669922 | Batch Time=12.5 +GPU:1 | Epoch: 12 | loss=11.585901260375977 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.769732475280762 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.686628341674805 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.805273056030273 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.672239303588867 | Batch Time=12.5 +GPU:1 | Epoch: 12 | loss=11.602850914001465 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.72922134399414 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.781793594360352 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.675512313842773 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.69809627532959 | Batch Time=11.71875 +GPU:1 | Epoch: 12 | loss=11.72433090209961 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.775529861450195 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.646483421325684 | Batch Time=13.28125 +GPU:1 | Epoch: 12 | loss=11.667730331420898 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.51231575012207 | Batch Time=12.109375 +GPU:1 | Epoch: 12 | loss=11.656270980834961 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.444652557373047 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.447327613830566 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.721940994262695 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.698152542114258 | Batch Time=4.6875 +GPU:1 | Epoch: 12 | loss=11.667108535766602 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.634342193603516 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.478225708007812 | Batch Time=14.84375 +GPU:1 | Epoch: 12 | loss=11.733114242553711 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.606325149536133 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.607575416564941 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.587125778198242 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.692028045654297 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.559528350830078 | Batch Time=12.5 +GPU:1 | Epoch: 12 | loss=11.917755126953125 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.649008750915527 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.529823303222656 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.60533618927002 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.587644577026367 | Batch Time=9.375 +GPU:1 | Epoch: 12 | loss=11.689556121826172 | Batch Time=5.46875 +GPU:1 | Epoch: 12 | loss=11.780683517456055 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.747718811035156 | Batch Time=8.203125 +GPU:1 | Epoch: 12 | loss=11.504331588745117 | Batch Time=9.375 +GPU:1 | Epoch: 12 | loss=11.626792907714844 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.661264419555664 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.639500617980957 | Batch Time=7.421875 +GPU:1 | Epoch: 12 | loss=11.643186569213867 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.539151191711426 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.785113334655762 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.55685043334961 | Batch Time=12.109375 +GPU:1 | Epoch: 12 | loss=11.705738067626953 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.465152740478516 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.689786911010742 | Batch Time=7.8125 +GPU:1 | Epoch: 12 | loss=11.552457809448242 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.603803634643555 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.695564270019531 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.67776107788086 | Batch Time=7.8125 +GPU:1 | Epoch: 12 | loss=11.504781723022461 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.676825523376465 | Batch Time=8.59375 +GPU:1 | Epoch: 12 | loss=11.637237548828125 | Batch Time=7.8125 +GPU:1 | Epoch: 12 | loss=11.485458374023438 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.606765747070312 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.76457691192627 | Batch Time=10.546875 +GPU:1 | Epoch: 12 | loss=11.53384017944336 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.714876174926758 | Batch Time=10.15625 +GPU:1 | Epoch: 12 | loss=11.415128707885742 | Batch Time=12.890625 +GPU:1 | Epoch: 12 | loss=11.578706741333008 | Batch Time=11.328125 +GPU:1 | Epoch: 12 | loss=11.624504089355469 | Batch Time=8.984375 +GPU:1 | Epoch: 12 | loss=11.498832702636719 | Batch Time=10.9375 +GPU:1 | Epoch: 12 | loss=11.611356735229492 | Batch Time=12.109375 +GPU:1 | Epoch: 12 | loss=11.582569122314453 | Batch Time=9.765625 +GPU:1 | Epoch: 12 | loss=11.575653076171875 | Batch Time=12.5 +GPU:1 | Epoch: 12 | loss=11.412398338317871 | Batch Time=13.28125 +GPU:0 | Epoch: 12 | loss=11.541400909423828 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.819510459899902 | Batch Time=7.421875 +GPU:0 | Epoch: 12 | loss=11.732858657836914 | Batch Time=12.109375 +GPU:0 | Epoch: 12 | loss=12.030962944030762 | Batch Time=6.640625 +GPU:0 | Epoch: 12 | loss=11.772481918334961 | Batch Time=8.984375 +GPU:0 | Epoch: 12 | loss=11.836786270141602 | Batch Time=8.59375 +GPU:0 | Epoch: 12 | loss=11.662971496582031 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.83525276184082 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.7073974609375 | Batch Time=8.984375 +GPU:0 | Epoch: 12 | loss=11.934013366699219 | Batch Time=7.8125 +GPU:0 | Epoch: 12 | loss=11.678367614746094 | Batch Time=11.71875 +GPU:0 | Epoch: 12 | loss=11.775666236877441 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.88856315612793 | Batch Time=7.03125 +GPU:0 | Epoch: 12 | loss=11.783597946166992 | Batch Time=7.03125 +GPU:0 | Epoch: 12 | loss=11.709858894348145 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.678096771240234 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.727117538452148 | Batch Time=8.984375 +GPU:0 | Epoch: 12 | loss=11.772171020507812 | Batch Time=11.328125 +GPU:0 | Epoch: 12 | loss=11.90283203125 | Batch Time=8.59375 +GPU:0 | Epoch: 12 | loss=11.627546310424805 | Batch Time=7.8125 +GPU:0 | Epoch: 12 | loss=11.67750358581543 | Batch Time=10.546875 +GPU:0 | Epoch: 12 | loss=11.699668884277344 | Batch Time=8.203125 +GPU:0 | Epoch: 12 | loss=11.67624282836914 | Batch Time=8.59375 +GPU:0 | Epoch: 12 | loss=11.716503143310547 | Batch Time=11.71875 +GPU:0 | Epoch: 12 | loss=11.596920013427734 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.790670394897461 | Batch Time=10.546875 +GPU:0 | Epoch: 12 | loss=11.701913833618164 | Batch Time=14.453125 +GPU:0 | Epoch: 12 | loss=11.954137802124023 | Batch Time=6.25 +GPU:0 | Epoch: 12 | loss=11.749165534973145 | Batch Time=11.71875 +GPU:0 | Epoch: 12 | loss=11.73612117767334 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.757402420043945 | Batch Time=11.328125 +GPU:0 | Epoch: 12 | loss=11.663274765014648 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.626026153564453 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.615705490112305 | Batch Time=7.8125 +GPU:0 | Epoch: 12 | loss=11.779583930969238 | Batch Time=8.203125 +GPU:0 | Epoch: 12 | loss=11.798980712890625 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.545989990234375 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.755033493041992 | Batch Time=8.984375 +GPU:0 | Epoch: 12 | loss=11.705615043640137 | Batch Time=13.671875 +GPU:0 | Epoch: 12 | loss=11.668731689453125 | Batch Time=8.984375 +GPU:0 | Epoch: 12 | loss=11.758514404296875 | Batch Time=11.71875 +GPU:0 | Epoch: 12 | loss=11.694438934326172 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.839611053466797 | Batch Time=8.203125 +GPU:0 | Epoch: 12 | loss=11.703929901123047 | Batch Time=7.03125 +GPU:0 | Epoch: 12 | loss=11.699102401733398 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.768943786621094 | Batch Time=7.421875 +GPU:0 | Epoch: 12 | loss=11.899360656738281 | Batch Time=5.46875 +GPU:0 | Epoch: 12 | loss=11.8011474609375 | Batch Time=8.984375 +GPU:0 | Epoch: 12 | loss=11.625901222229004 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.653544425964355 | Batch Time=12.890625 +GPU:0 | Epoch: 12 | loss=11.746620178222656 | Batch Time=8.203125 +GPU:0 | Epoch: 12 | loss=11.820926666259766 | Batch Time=7.421875 +GPU:0 | Epoch: 12 | loss=11.637081146240234 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.62013053894043 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.687222480773926 | Batch Time=12.5 +GPU:0 | Epoch: 12 | loss=11.613987922668457 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.628406524658203 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.826817512512207 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.77912425994873 | Batch Time=11.71875 +GPU:0 | Epoch: 12 | loss=11.659473419189453 | Batch Time=8.984375 +GPU:0 | Epoch: 12 | loss=11.720189094543457 | Batch Time=11.71875 +GPU:0 | Epoch: 12 | loss=11.723184585571289 | Batch Time=10.546875 +GPU:0 | Epoch: 12 | loss=11.66442584991455 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.618332862854004 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.836301803588867 | Batch Time=8.984375 +GPU:0 | Epoch: 12 | loss=11.513492584228516 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.650099754333496 | Batch Time=10.546875 +GPU:0 | Epoch: 12 | loss=11.64887809753418 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.791032791137695 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.702958106994629 | Batch Time=11.71875 +GPU:0 | Epoch: 12 | loss=11.912534713745117 | Batch Time=7.421875 +GPU:0 | Epoch: 12 | loss=11.797943115234375 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.634256362915039 | Batch Time=12.5 +GPU:0 | Epoch: 12 | loss=11.694695472717285 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.559158325195312 | Batch Time=12.890625 +GPU:0 | Epoch: 12 | loss=11.478765487670898 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.623432159423828 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.719789505004883 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.739073753356934 | Batch Time=12.109375 +GPU:0 | Epoch: 12 | loss=11.52255630493164 | Batch Time=8.203125 +GPU:0 | Epoch: 12 | loss=11.668499946594238 | Batch Time=11.328125 +GPU:0 | Epoch: 12 | loss=11.801509857177734 | Batch Time=7.8125 +GPU:0 | Epoch: 12 | loss=11.56587028503418 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.637378692626953 | Batch Time=14.84375 +GPU:0 | Epoch: 12 | loss=11.77385139465332 | Batch Time=7.8125 +GPU:0 | Epoch: 12 | loss=11.659208297729492 | Batch Time=8.59375 +GPU:0 | Epoch: 12 | loss=11.567583084106445 | Batch Time=11.328125 +GPU:0 | Epoch: 12 | loss=11.527627944946289 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.701351165771484 | Batch Time=7.421875 +GPU:0 | Epoch: 12 | loss=11.7997465133667 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.647636413574219 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.532005310058594 | Batch Time=11.328125 +GPU:0 | Epoch: 12 | loss=11.67756462097168 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.661197662353516 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.44858169555664 | Batch Time=12.5 +GPU:0 | Epoch: 12 | loss=11.689250946044922 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.566953659057617 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.771133422851562 | Batch Time=8.59375 +GPU:0 | Epoch: 12 | loss=11.473381042480469 | Batch Time=12.109375 +GPU:0 | Epoch: 12 | loss=11.501920700073242 | Batch Time=8.59375 +GPU:0 | Epoch: 12 | loss=11.718181610107422 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.773642539978027 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.481786727905273 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.614982604980469 | Batch Time=10.546875 +GPU:0 | Epoch: 12 | loss=11.454570770263672 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.778510093688965 | Batch Time=7.421875 +GPU:0 | Epoch: 12 | loss=11.752843856811523 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.804512977600098 | Batch Time=9.375 +GPU:0 | Epoch: 12 | loss=11.530250549316406 | Batch Time=7.421875 +GPU:0 | Epoch: 12 | loss=11.53184700012207 | Batch Time=10.546875 +GPU:0 | Epoch: 12 | loss=11.637145042419434 | Batch Time=10.9375 +GPU:0 | Epoch: 12 | loss=11.576364517211914 | Batch Time=12.109375 +GPU:0 | Epoch: 12 | loss=11.658048629760742 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=11.615303039550781 | Batch Time=10.546875 +GPU:0 | Epoch: 12 | loss=11.85739517211914 | Batch Time=5.46875 +GPU:0 | Epoch: 12 | loss=11.440685272216797 | Batch Time=11.328125 +GPU:0 | Epoch: 12 | loss=11.566946029663086 | Batch Time=7.8125 +GPU:0 | Epoch: 12 | loss=11.69501781463623 | Batch Time=8.59375 +GPU:0 | Epoch: 12 | loss=11.67153549194336 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.684449195861816 | Batch Time=8.203125 +GPU:0 | Epoch: 12 | loss=11.457340240478516 | Batch Time=11.71875 +GPU:0 | Epoch: 12 | loss=11.613494873046875 | Batch Time=9.765625 +GPU:0 | Epoch: 12 | loss=11.546892166137695 | Batch Time=10.546875 +GPU:0 | Epoch: 12 | loss=11.58975887298584 | Batch Time=8.59375 +GPU:0 | Epoch: 12 | loss=11.88878059387207 | Batch Time=8.59375 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=140575932416, percent=45.4, used=109079613440, free=17708290048, active=186755928064, inactive=48073633792, buffers=438112256, cached=130342068224, shared=5661081600, slab=3223314432) +AFTER TRAIN LOOP: Epoch 12 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +STARTING TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +BEFORE TRAIN LOOP: Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=140576239616, percent=45.4, used=109079547904, free=17708597248, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +AFTER TRAIN LOOP: Epoch 12 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +STARTING TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +BEFORE TRAIN LOOP: Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=140576239616, percent=45.4, used=109079547904, free=17708597248, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +AFTER TRAIN LOOP: Epoch 12 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +STARTING TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +BEFORE TRAIN LOOP: Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +AFTER TRAIN LOOP: Epoch 12 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +BEFORE VAL LOOP: GPU: 0 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=140576247808, percent=45.4, used=109079539712, free=17708605440, active=186755731456, inactive=48073535488, buffers=438112256, cached=130341826560, shared=5660839936, slab=3223265280) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 12 | loss=4.5002760887146 | Batch Time=12.890625 +GPU:0 | Epoch: 12 | loss=5.66232967376709 | Batch Time=3.125 +GPU:0 | Epoch: 12 | loss=5.271624565124512 | Batch Time=15.625 +GPU:0 | Epoch: 12 | loss=4.750343322753906 | Batch Time=7.03125 +GPU:0 | Epoch: 12 | loss=6.125829696655273 | Batch Time=1.171875 +GPU:0 | Epoch: 12 | loss=5.238719463348389 | Batch Time=12.109375 +GPU:0 | Epoch: 12 | loss=6.049678325653076 | Batch Time=0.390625 +GPU:0 | Epoch: 12 | loss=5.75517463684082 | Batch Time=4.6875 +GPU:0 | Epoch: 12 | loss=5.346975326538086 | Batch Time=5.46875 +GPU:0 | Epoch: 12 | loss=6.10013484954834 | Batch Time=1.953125 +GPU:0 | Epoch: 12 | loss=5.737612724304199 | Batch Time=1.953125 +GPU:0 | Epoch: 12 | loss=4.905691623687744 | Batch Time=13.671875 +GPU:0 | Epoch: 12 | loss=5.829073905944824 | Batch Time=2.34375 +GPU:0 | Epoch: 12 | loss=5.3989787101745605 | Batch Time=7.03125 +GPU:0 | Epoch: 12 | loss=5.0951762199401855 | Batch Time=3.125 +GPU:0 | Epoch: 12 | loss=5.2423415184021 | Batch Time=6.25 +GPU:0 | Epoch: 12 | loss=5.200155258178711 | Batch Time=6.640625 +GPU:0 | Epoch: 12 | loss=4.467076301574707 | Batch Time=17.1875 +GPU:0 | Epoch: 12 | loss=4.078910827636719 | Batch Time=16.40625 +GPU:0 | Epoch: 12 | loss=4.6007161140441895 | Batch Time=14.84375 +Model top1 Accuracy: 8.162 +Acc before rounding: 8.162 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 12 | loss=4.5002760887146 | Batch Time=12.890625 +GPU:0 | Epoch: 12 | loss=5.66232967376709 | Batch Time=3.125 +GPU:0 | Epoch: 12 | loss=5.271624565124512 | Batch Time=15.625 +GPU:0 | Epoch: 12 | loss=4.750343322753906 | Batch Time=7.03125 +GPU:0 | Epoch: 12 | loss=6.125829696655273 | Batch Time=1.171875 +GPU:0 | Epoch: 12 | loss=5.238719463348389 | Batch Time=12.109375 +GPU:0 | Epoch: 12 | loss=6.049678325653076 | Batch Time=0.390625 +GPU:0 | Epoch: 12 | loss=5.75517463684082 | Batch Time=4.6875 +GPU:0 | Epoch: 12 | loss=5.346975326538086 | Batch Time=5.46875 +GPU:0 | Epoch: 12 | loss=6.10013484954834 | Batch Time=1.953125 +GPU:0 | Epoch: 12 | loss=5.737612724304199 | Batch Time=1.953125 +GPU:0 | Epoch: 12 | loss=4.905691623687744 | Batch Time=13.671875 +GPU:0 | Epoch: 12 | loss=5.829073905944824 | Batch Time=2.34375 +GPU:0 | Epoch: 12 | loss=5.3989787101745605 | Batch Time=7.03125 +GPU:0 | Epoch: 12 | loss=5.0951762199401855 | Batch Time=3.125 +GPU:0 | Epoch: 12 | loss=5.2423415184021 | Batch Time=6.25 +GPU:0 | Epoch: 12 | loss=5.200155258178711 | Batch Time=6.640625 +GPU:0 | Epoch: 12 | loss=4.467076301574707 | Batch Time=17.1875 +GPU:0 | Epoch: 12 | loss=4.078910827636719 | Batch Time=16.40625 +GPU:0 | Epoch: 12 | loss=4.6007161140441895 | Batch Time=14.84375 +Model top1 Accuracy: 8.162 +Acc after rounding: 8.162 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 12 | loss=5.564075469970703 | Batch Time=10.15625 +GPU:0 | Epoch: 12 | loss=5.389588356018066 | Batch Time=6.640625 +GPU:0 | Epoch: 12 | loss=5.388845920562744 | Batch Time=7.8125 +GPU:0 | Epoch: 12 | loss=5.662675380706787 | Batch Time=7.03125 +Model top1 Accuracy: 6.79 +Validation Acc after rounding: 6.79 +AFTER VAL LOOP: GPU: 0 | Epoch 12 | Memory Usage: svmem(total=257568083968, available=122744934400, percent=52.3, used=126910435328, free=5902602240, active=200639029248, inactive=45880582144, buffers=437862400, cached=124317184000, shared=5661241344, slab=3228999680) +Rounding model with scheme: naive +Model avg sparsity: 43.14558853130041 +GPU:0 | Epoch: 12 | Acc=8.162 | Epoch Time=16.868646836280824 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=122744950784, percent=52.3, used=126910496768, free=5902594048, active=200639037440, inactive=45880553472, buffers=437870592, cached=124317122560, shared=5661159424, slab=3229007872) +STARTING TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=122744950784, percent=52.3, used=126910496768, free=5902594048, active=200639037440, inactive=45880553472, buffers=437870592, cached=124317122560, shared=5661159424, slab=3229007872) +BEFORE TRAIN LOOP: Epoch 13 | Memory Usage: svmem(total=257568083968, available=122744950784, percent=52.3, used=126910496768, free=5902594048, active=200639037440, inactive=45880553472, buffers=437870592, cached=124317122560, shared=5661159424, slab=3229007872) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=122744950784, percent=52.3, used=126910496768, free=5902594048, active=200639037440, inactive=45880553472, buffers=437870592, cached=124317122560, shared=5661159424, slab=3229007872) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 13 | loss=11.646064758300781 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.566594123840332 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.484236717224121 | Batch Time=13.671875 +GPU:1 | Epoch: 13 | loss=11.487554550170898 | Batch Time=12.109375 +GPU:1 | Epoch: 13 | loss=11.558588027954102 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.70685863494873 | Batch Time=8.59375 +GPU:1 | Epoch: 13 | loss=11.72650146484375 | Batch Time=7.421875 +GPU:1 | Epoch: 13 | loss=11.37993049621582 | Batch Time=12.5 +GPU:1 | Epoch: 13 | loss=11.526647567749023 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.654328346252441 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.49608039855957 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.41130256652832 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.656599044799805 | Batch Time=7.8125 +GPU:1 | Epoch: 13 | loss=11.506141662597656 | Batch Time=12.890625 +GPU:1 | Epoch: 13 | loss=11.506086349487305 | Batch Time=14.453125 +GPU:1 | Epoch: 13 | loss=11.73933219909668 | Batch Time=8.59375 +GPU:1 | Epoch: 13 | loss=11.607564926147461 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.780843734741211 | Batch Time=5.859375 +GPU:1 | Epoch: 13 | loss=11.58780288696289 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.800703048706055 | Batch Time=7.421875 +GPU:1 | Epoch: 13 | loss=11.533903121948242 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.658918380737305 | Batch Time=8.203125 +GPU:1 | Epoch: 13 | loss=11.705831527709961 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.683825492858887 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.432104110717773 | Batch Time=12.890625 +GPU:1 | Epoch: 13 | loss=11.592716217041016 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.711517333984375 | Batch Time=8.203125 +GPU:1 | Epoch: 13 | loss=11.450132369995117 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.524393081665039 | Batch Time=8.59375 +GPU:1 | Epoch: 13 | loss=11.60639762878418 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.383790016174316 | Batch Time=12.890625 +GPU:1 | Epoch: 13 | loss=11.644234657287598 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.380456924438477 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.652875900268555 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.56962776184082 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.395633697509766 | Batch Time=12.5 +GPU:1 | Epoch: 13 | loss=11.504293441772461 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.559730529785156 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.66104507446289 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.33388614654541 | Batch Time=12.890625 +GPU:1 | Epoch: 13 | loss=11.469398498535156 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.418503761291504 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.420780181884766 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.515302658081055 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.500492095947266 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.530540466308594 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.284006118774414 | Batch Time=13.671875 +GPU:1 | Epoch: 13 | loss=11.51733684539795 | Batch Time=12.890625 +GPU:1 | Epoch: 13 | loss=11.607229232788086 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.47403335571289 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.48843765258789 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.565768241882324 | Batch Time=12.109375 +GPU:1 | Epoch: 13 | loss=11.534245491027832 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.375543594360352 | Batch Time=12.109375 +GPU:1 | Epoch: 13 | loss=11.346277236938477 | Batch Time=8.203125 +GPU:1 | Epoch: 13 | loss=11.565387725830078 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.51960563659668 | Batch Time=6.25 +GPU:1 | Epoch: 13 | loss=11.456499099731445 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.473152160644531 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.540023803710938 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.55190658569336 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.526151657104492 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.283702850341797 | Batch Time=13.28125 +GPU:1 | Epoch: 13 | loss=11.595247268676758 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.397586822509766 | Batch Time=13.671875 +GPU:1 | Epoch: 13 | loss=11.58084487915039 | Batch Time=7.8125 +GPU:1 | Epoch: 13 | loss=11.624174118041992 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.517955780029297 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.606428146362305 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.461431503295898 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.52408504486084 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.473801612854004 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.638018608093262 | Batch Time=8.59375 +GPU:1 | Epoch: 13 | loss=11.450078964233398 | Batch Time=8.203125 +GPU:1 | Epoch: 13 | loss=11.622686386108398 | Batch Time=12.109375 +GPU:1 | Epoch: 13 | loss=11.50414752960205 | Batch Time=13.671875 +GPU:1 | Epoch: 13 | loss=11.563554763793945 | Batch Time=8.203125 +GPU:1 | Epoch: 13 | loss=11.53436279296875 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.673181533813477 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.379693984985352 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.388721466064453 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.401268005371094 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.264338493347168 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.43069839477539 | Batch Time=12.5 +GPU:1 | Epoch: 13 | loss=11.474174499511719 | Batch Time=10.15625 +GPU:1 | Epoch: 13 | loss=11.45026969909668 | Batch Time=8.203125 +GPU:1 | Epoch: 13 | loss=11.539405822753906 | Batch Time=12.109375 +GPU:1 | Epoch: 13 | loss=11.336786270141602 | Batch Time=8.59375 +GPU:1 | Epoch: 13 | loss=11.38869857788086 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.330036163330078 | Batch Time=13.671875 +GPU:1 | Epoch: 13 | loss=11.37552261352539 | Batch Time=12.5 +GPU:1 | Epoch: 13 | loss=11.566083908081055 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.489346504211426 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.509424209594727 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.445929527282715 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.435625076293945 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.269035339355469 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.432031631469727 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.393014907836914 | Batch Time=13.28125 +GPU:1 | Epoch: 13 | loss=11.369247436523438 | Batch Time=12.5 +GPU:1 | Epoch: 13 | loss=11.28286361694336 | Batch Time=14.0625 +GPU:1 | Epoch: 13 | loss=11.41236686706543 | Batch Time=12.890625 +GPU:1 | Epoch: 13 | loss=11.334810256958008 | Batch Time=12.890625 +GPU:1 | Epoch: 13 | loss=11.25448226928711 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.327470779418945 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.295909881591797 | Batch Time=15.625 +GPU:1 | Epoch: 13 | loss=11.289138793945312 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.453619956970215 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.402141571044922 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.483499526977539 | Batch Time=11.71875 +GPU:1 | Epoch: 13 | loss=11.463371276855469 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.465194702148438 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.31517219543457 | Batch Time=11.328125 +GPU:1 | Epoch: 13 | loss=11.36594295501709 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.407875061035156 | Batch Time=12.5 +GPU:1 | Epoch: 13 | loss=11.309558868408203 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.367429733276367 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.323480606079102 | Batch Time=10.9375 +GPU:1 | Epoch: 13 | loss=11.459661483764648 | Batch Time=10.546875 +GPU:1 | Epoch: 13 | loss=11.488886833190918 | Batch Time=9.765625 +GPU:1 | Epoch: 13 | loss=11.463123321533203 | Batch Time=12.109375 +GPU:1 | Epoch: 13 | loss=11.097755432128906 | Batch Time=16.40625 +GPU:1 | Epoch: 13 | loss=11.44664192199707 | Batch Time=9.375 +GPU:1 | Epoch: 13 | loss=11.507715225219727 | Batch Time=8.984375 +GPU:1 | Epoch: 13 | loss=11.305644989013672 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.683298110961914 | Batch Time=10.546875 +GPU:2 | Epoch: 13 | loss=11.448020935058594 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.435236930847168 | Batch Time=12.890625 +GPU:2 | Epoch: 13 | loss=11.739595413208008 | Batch Time=8.203125 +GPU:2 | Epoch: 13 | loss=11.70374870300293 | Batch Time=10.546875 +GPU:2 | Epoch: 13 | loss=11.487347602844238 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.473505973815918 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.305570602416992 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.502840995788574 | Batch Time=15.234375 +GPU:2 | Epoch: 13 | loss=11.653615951538086 | Batch Time=9.765625 +GPU:2 | Epoch: 13 | loss=11.449999809265137 | Batch Time=8.984375 +GPU:2 | Epoch: 13 | loss=11.600446701049805 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.637537002563477 | Batch Time=9.765625 +GPU:2 | Epoch: 13 | loss=11.41364860534668 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.682039260864258 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.497389793395996 | Batch Time=13.28125 +GPU:2 | Epoch: 13 | loss=11.40269660949707 | Batch Time=12.5 +GPU:2 | Epoch: 13 | loss=11.439388275146484 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.52160930633545 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.394952774047852 | Batch Time=13.671875 +GPU:2 | Epoch: 13 | loss=11.581546783447266 | Batch Time=8.984375 +GPU:2 | Epoch: 13 | loss=11.713062286376953 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.501943588256836 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.559316635131836 | Batch Time=13.28125 +GPU:2 | Epoch: 13 | loss=11.696044921875 | Batch Time=8.203125 +GPU:2 | Epoch: 13 | loss=11.495246887207031 | Batch Time=11.328125 +GPU:2 | Epoch: 13 | loss=11.755215644836426 | Batch Time=8.203125 +GPU:2 | Epoch: 13 | loss=11.429966926574707 | Batch Time=11.328125 +GPU:2 | Epoch: 13 | loss=11.475515365600586 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.46014404296875 | Batch Time=11.328125 +GPU:2 | Epoch: 13 | loss=11.395135879516602 | Batch Time=9.765625 +GPU:2 | Epoch: 13 | loss=11.495591163635254 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.604032516479492 | Batch Time=8.203125 +GPU:2 | Epoch: 13 | loss=11.647147178649902 | Batch Time=9.765625 +GPU:2 | Epoch: 13 | loss=11.52745246887207 | Batch Time=11.328125 +GPU:2 | Epoch: 13 | loss=11.533676147460938 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.588546752929688 | Batch Time=6.640625 +GPU:2 | Epoch: 13 | loss=11.53293228149414 | Batch Time=13.28125 +GPU:2 | Epoch: 13 | loss=11.482906341552734 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.37353801727295 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.550325393676758 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.669368743896484 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.717438697814941 | Batch Time=8.203125 +GPU:2 | Epoch: 13 | loss=11.518372535705566 | Batch Time=10.546875 +GPU:2 | Epoch: 13 | loss=11.499027252197266 | Batch Time=7.8125 +GPU:2 | Epoch: 13 | loss=11.376192092895508 | Batch Time=13.671875 +GPU:2 | Epoch: 13 | loss=11.481283187866211 | Batch Time=13.28125 +GPU:2 | Epoch: 13 | loss=11.659123420715332 | Batch Time=8.984375 +GPU:2 | Epoch: 13 | loss=11.601692199707031 | Batch Time=9.765625 +GPU:2 | Epoch: 13 | loss=11.733075141906738 | Batch Time=7.421875 +GPU:2 | Epoch: 13 | loss=11.338871002197266 | Batch Time=12.5 +GPU:2 | Epoch: 13 | loss=11.63686466217041 | Batch Time=11.328125 +GPU:2 | Epoch: 13 | loss=11.598101615905762 | Batch Time=9.765625 +GPU:2 | Epoch: 13 | loss=11.412662506103516 | Batch Time=14.0625 +GPU:2 | Epoch: 13 | loss=11.518289566040039 | Batch Time=12.890625 +GPU:2 | Epoch: 13 | loss=11.576791763305664 | Batch Time=8.984375 +GPU:2 | Epoch: 13 | loss=11.40568733215332 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.580755233764648 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.409708023071289 | Batch Time=14.0625 +GPU:2 | Epoch: 13 | loss=11.378567695617676 | Batch Time=11.328125 +GPU:2 | Epoch: 13 | loss=11.52746868133545 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.232675552368164 | Batch Time=17.1875 +GPU:2 | Epoch: 13 | loss=11.481856346130371 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.454463958740234 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.56353759765625 | Batch Time=13.28125 +GPU:2 | Epoch: 13 | loss=11.461091995239258 | Batch Time=10.546875 +GPU:2 | Epoch: 13 | loss=11.461071014404297 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.337202072143555 | Batch Time=15.234375 +GPU:2 | Epoch: 13 | loss=11.718267440795898 | Batch Time=6.640625 +GPU:2 | Epoch: 13 | loss=11.429105758666992 | Batch Time=8.59375 +GPU:2 | Epoch: 13 | loss=11.363941192626953 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.47787857055664 | Batch Time=7.03125 +GPU:2 | Epoch: 13 | loss=11.515482902526855 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.538655281066895 | Batch Time=12.5 +GPU:2 | Epoch: 13 | loss=11.551811218261719 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.366405487060547 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.374757766723633 | Batch Time=7.8125 +GPU:2 | Epoch: 13 | loss=11.422784805297852 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.409015655517578 | Batch Time=10.546875 +GPU:2 | Epoch: 13 | loss=11.398754119873047 | Batch Time=13.28125 +GPU:2 | Epoch: 13 | loss=11.43687629699707 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.503307342529297 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.453495025634766 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.4580717086792 | Batch Time=11.328125 +GPU:2 | Epoch: 13 | loss=11.456466674804688 | Batch Time=10.9375 +GPU:2 | Epoch: 13 | loss=11.274259567260742 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.25467300415039 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.463068962097168 | Batch Time=8.984375 +GPU:2 | Epoch: 13 | loss=11.261331558227539 | Batch Time=12.5 +GPU:2 | Epoch: 13 | loss=11.49250602722168 | Batch Time=12.890625 +GPU:2 | Epoch: 13 | loss=11.476903915405273 | Batch Time=8.203125 +GPU:2 | Epoch: 13 | loss=11.630356788635254 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.180426597595215 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.343835830688477 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.38520622253418 | Batch Time=14.453125 +GPU:2 | Epoch: 13 | loss=11.290679931640625 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.371894836425781 | Batch Time=15.234375 +GPU:2 | Epoch: 13 | loss=11.344220161437988 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.47149658203125 | Batch Time=9.765625 +GPU:2 | Epoch: 13 | loss=11.407514572143555 | Batch Time=7.421875 +GPU:2 | Epoch: 13 | loss=11.398317337036133 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.376127243041992 | Batch Time=12.5 +GPU:2 | Epoch: 13 | loss=11.469443321228027 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.562828063964844 | Batch Time=8.59375 +GPU:2 | Epoch: 13 | loss=11.45462417602539 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.307998657226562 | Batch Time=16.40625 +GPU:2 | Epoch: 13 | loss=11.343462944030762 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.444828987121582 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.658588409423828 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.412557601928711 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.487762451171875 | Batch Time=8.984375 +GPU:2 | Epoch: 13 | loss=11.383716583251953 | Batch Time=11.328125 +GPU:2 | Epoch: 13 | loss=11.633565902709961 | Batch Time=8.203125 +GPU:2 | Epoch: 13 | loss=11.372292518615723 | Batch Time=11.71875 +GPU:2 | Epoch: 13 | loss=11.432669639587402 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.365678787231445 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.585376739501953 | Batch Time=8.203125 +GPU:2 | Epoch: 13 | loss=11.382513046264648 | Batch Time=8.59375 +GPU:2 | Epoch: 13 | loss=11.233549118041992 | Batch Time=12.109375 +GPU:2 | Epoch: 13 | loss=11.306690216064453 | Batch Time=16.40625 +GPU:2 | Epoch: 13 | loss=11.532646179199219 | Batch Time=8.984375 +GPU:2 | Epoch: 13 | loss=11.488292694091797 | Batch Time=9.375 +GPU:2 | Epoch: 13 | loss=11.412879943847656 | Batch Time=10.15625 +GPU:2 | Epoch: 13 | loss=11.400148391723633 | Batch Time=12.890625 +GPU:2 | Epoch: 13 | loss=11.28536605834961 | Batch Time=16.015625 +GPU:0 | Epoch: 13 | loss=11.727699279785156 | Batch Time=7.421875 +GPU:0 | Epoch: 13 | loss=11.47524356842041 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.53090763092041 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.304107666015625 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.443140029907227 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.623198509216309 | Batch Time=11.71875 +GPU:0 | Epoch: 13 | loss=11.567509651184082 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.55858039855957 | Batch Time=11.71875 +GPU:0 | Epoch: 13 | loss=11.654011726379395 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.671388626098633 | Batch Time=8.203125 +GPU:0 | Epoch: 13 | loss=11.50832462310791 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=11.712630271911621 | Batch Time=7.421875 +GPU:0 | Epoch: 13 | loss=11.531096458435059 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=11.476150512695312 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.6813325881958 | Batch Time=8.59375 +GPU:0 | Epoch: 13 | loss=11.577642440795898 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.514535903930664 | Batch Time=8.59375 +GPU:0 | Epoch: 13 | loss=11.550098419189453 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.527986526489258 | Batch Time=12.5 +GPU:0 | Epoch: 13 | loss=11.658063888549805 | Batch Time=7.8125 +GPU:0 | Epoch: 13 | loss=11.410537719726562 | Batch Time=7.8125 +GPU:0 | Epoch: 13 | loss=11.550626754760742 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.603154182434082 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.639470100402832 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.563066482543945 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.494319915771484 | Batch Time=13.28125 +GPU:0 | Epoch: 13 | loss=11.610758781433105 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.507354736328125 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.40286636352539 | Batch Time=12.890625 +GPU:0 | Epoch: 13 | loss=11.502269744873047 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.536846160888672 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.488717079162598 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.571172714233398 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.71856689453125 | Batch Time=7.8125 +GPU:0 | Epoch: 13 | loss=11.54930305480957 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.475830078125 | Batch Time=7.8125 +GPU:0 | Epoch: 13 | loss=11.524286270141602 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.607295989990234 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=11.55007553100586 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.420258522033691 | Batch Time=13.28125 +GPU:0 | Epoch: 13 | loss=11.436115264892578 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.595525741577148 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.589723587036133 | Batch Time=12.5 +GPU:0 | Epoch: 13 | loss=11.485262870788574 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.455781936645508 | Batch Time=12.5 +GPU:0 | Epoch: 13 | loss=11.409740447998047 | Batch Time=14.84375 +GPU:0 | Epoch: 13 | loss=11.560033798217773 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.561748504638672 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.494791030883789 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=11.503559112548828 | Batch Time=12.890625 +GPU:0 | Epoch: 13 | loss=11.524785041809082 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.28344440460205 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.553935050964355 | Batch Time=7.8125 +GPU:0 | Epoch: 13 | loss=11.47873592376709 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.449907302856445 | Batch Time=12.5 +GPU:0 | Epoch: 13 | loss=11.468080520629883 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.585935592651367 | Batch Time=11.71875 +GPU:0 | Epoch: 13 | loss=11.383708953857422 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.558758735656738 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.551047325134277 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.557106018066406 | Batch Time=8.203125 +GPU:0 | Epoch: 13 | loss=11.430116653442383 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.513361930847168 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.6189546585083 | Batch Time=8.59375 +GPU:0 | Epoch: 13 | loss=11.64957046508789 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=11.497819900512695 | Batch Time=12.890625 +GPU:0 | Epoch: 13 | loss=11.381820678710938 | Batch Time=11.71875 +GPU:0 | Epoch: 13 | loss=11.502933502197266 | Batch Time=8.59375 +GPU:0 | Epoch: 13 | loss=11.46741008758545 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.50811767578125 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.490594863891602 | Batch Time=8.59375 +GPU:0 | Epoch: 13 | loss=11.326693534851074 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.409920692443848 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.288630485534668 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.569181442260742 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=11.376267433166504 | Batch Time=11.71875 +GPU:0 | Epoch: 13 | loss=11.429489135742188 | Batch Time=7.8125 +GPU:0 | Epoch: 13 | loss=11.369061470031738 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.508090019226074 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=11.406749725341797 | Batch Time=8.203125 +GPU:0 | Epoch: 13 | loss=11.465669631958008 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.348894119262695 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=11.669662475585938 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.666040420532227 | Batch Time=8.203125 +GPU:0 | Epoch: 13 | loss=11.55061149597168 | Batch Time=13.28125 +GPU:0 | Epoch: 13 | loss=11.445096969604492 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.471534729003906 | Batch Time=7.03125 +GPU:0 | Epoch: 13 | loss=11.424970626831055 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=11.557310104370117 | Batch Time=7.03125 +GPU:0 | Epoch: 13 | loss=11.749526977539062 | Batch Time=7.421875 +GPU:0 | Epoch: 13 | loss=11.362394332885742 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.411567687988281 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.385100364685059 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=11.527555465698242 | Batch Time=11.71875 +GPU:0 | Epoch: 13 | loss=11.395885467529297 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.378103256225586 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=11.244003295898438 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.436299324035645 | Batch Time=8.203125 +GPU:0 | Epoch: 13 | loss=11.449604034423828 | Batch Time=7.8125 +GPU:0 | Epoch: 13 | loss=11.383171081542969 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=11.364029884338379 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.426610946655273 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.182899475097656 | Batch Time=14.84375 +GPU:0 | Epoch: 13 | loss=11.469398498535156 | Batch Time=12.5 +GPU:0 | Epoch: 13 | loss=11.399877548217773 | Batch Time=13.671875 +GPU:0 | Epoch: 13 | loss=11.397428512573242 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.323835372924805 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.327309608459473 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=11.377723693847656 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=11.162881851196289 | Batch Time=14.453125 +GPU:0 | Epoch: 13 | loss=11.286558151245117 | Batch Time=12.890625 +GPU:0 | Epoch: 13 | loss=11.278315544128418 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.287982940673828 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.428140640258789 | Batch Time=10.15625 +GPU:0 | Epoch: 13 | loss=11.464073181152344 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=11.332021713256836 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.526890754699707 | Batch Time=10.9375 +GPU:0 | Epoch: 13 | loss=11.314237594604492 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.49631118774414 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=11.299943923950195 | Batch Time=12.890625 +GPU:0 | Epoch: 13 | loss=11.544776916503906 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.236845970153809 | Batch Time=11.328125 +GPU:0 | Epoch: 13 | loss=11.405736923217773 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=11.518976211547852 | Batch Time=12.109375 +GPU:0 | Epoch: 13 | loss=11.261962890625 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.522978782653809 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.58283805847168 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.437175750732422 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.592535018920898 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.594099998474121 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.60120677947998 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.57093620300293 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.565526962280273 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.628103256225586 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.550071716308594 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.496674537658691 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.51902961730957 | Batch Time=13.28125 +GPU:3 | Epoch: 13 | loss=11.58050537109375 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.41664981842041 | Batch Time=11.71875 +GPU:3 | Epoch: 13 | loss=11.483970642089844 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.571603775024414 | Batch Time=13.28125 +GPU:3 | Epoch: 13 | loss=11.581103324890137 | Batch Time=9.375 +GPU:3 | Epoch: 13 | loss=11.475378036499023 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.521820068359375 | Batch Time=15.234375 +GPU:3 | Epoch: 13 | loss=11.599054336547852 | Batch Time=8.203125 +GPU:3 | Epoch: 13 | loss=11.51797866821289 | Batch Time=8.203125 +GPU:3 | Epoch: 13 | loss=11.68954086303711 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.64732551574707 | Batch Time=6.25 +GPU:3 | Epoch: 13 | loss=11.352211952209473 | Batch Time=13.28125 +GPU:3 | Epoch: 13 | loss=11.624797821044922 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.542444229125977 | Batch Time=9.375 +GPU:3 | Epoch: 13 | loss=11.468920707702637 | Batch Time=12.5 +GPU:3 | Epoch: 13 | loss=11.502622604370117 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.554069519042969 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.545154571533203 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.609760284423828 | Batch Time=12.109375 +GPU:3 | Epoch: 13 | loss=11.410431861877441 | Batch Time=9.375 +GPU:3 | Epoch: 13 | loss=11.484107971191406 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.617582321166992 | Batch Time=8.59375 +GPU:3 | Epoch: 13 | loss=11.608148574829102 | Batch Time=12.5 +GPU:3 | Epoch: 13 | loss=11.339027404785156 | Batch Time=10.15625 +GPU:3 | Epoch: 13 | loss=11.677955627441406 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.450910568237305 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.493070602416992 | Batch Time=11.71875 +GPU:3 | Epoch: 13 | loss=11.529583930969238 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.637523651123047 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.342182159423828 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.463479995727539 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.720551490783691 | Batch Time=8.203125 +GPU:3 | Epoch: 13 | loss=11.422565460205078 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.415678977966309 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.353229522705078 | Batch Time=11.71875 +GPU:3 | Epoch: 13 | loss=11.506380081176758 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.582893371582031 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.516403198242188 | Batch Time=12.890625 +GPU:3 | Epoch: 13 | loss=11.585268020629883 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.418675422668457 | Batch Time=13.671875 +GPU:3 | Epoch: 13 | loss=11.353090286254883 | Batch Time=14.84375 +GPU:3 | Epoch: 13 | loss=11.368419647216797 | Batch Time=12.890625 +GPU:3 | Epoch: 13 | loss=11.499856948852539 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.435188293457031 | Batch Time=13.28125 +GPU:3 | Epoch: 13 | loss=11.58714485168457 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.362281799316406 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.42795467376709 | Batch Time=10.15625 +GPU:3 | Epoch: 13 | loss=11.444173812866211 | Batch Time=15.625 +GPU:3 | Epoch: 13 | loss=11.445229530334473 | Batch Time=13.28125 +GPU:3 | Epoch: 13 | loss=11.521442413330078 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.497201919555664 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.427133560180664 | Batch Time=7.8125 +GPU:3 | Epoch: 13 | loss=11.422538757324219 | Batch Time=12.109375 +GPU:3 | Epoch: 13 | loss=11.487018585205078 | Batch Time=9.375 +GPU:3 | Epoch: 13 | loss=11.584659576416016 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.343873977661133 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.446372985839844 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.478957176208496 | Batch Time=8.59375 +GPU:3 | Epoch: 13 | loss=11.48619270324707 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.332197189331055 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.578962326049805 | Batch Time=12.5 +GPU:3 | Epoch: 13 | loss=11.48415756225586 | Batch Time=12.890625 +GPU:3 | Epoch: 13 | loss=11.59029483795166 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.54723072052002 | Batch Time=10.15625 +GPU:3 | Epoch: 13 | loss=11.463724136352539 | Batch Time=9.375 +GPU:3 | Epoch: 13 | loss=11.593902587890625 | Batch Time=7.421875 +GPU:3 | Epoch: 13 | loss=11.476295471191406 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.357391357421875 | Batch Time=8.984375 +GPU:3 | Epoch: 13 | loss=11.61069107055664 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.517107963562012 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.391122817993164 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.564706802368164 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.484127044677734 | Batch Time=7.8125 +GPU:3 | Epoch: 13 | loss=11.50800609588623 | Batch Time=12.890625 +GPU:3 | Epoch: 13 | loss=11.501192092895508 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.473254203796387 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.480508804321289 | Batch Time=7.03125 +GPU:3 | Epoch: 13 | loss=11.427947044372559 | Batch Time=10.15625 +GPU:3 | Epoch: 13 | loss=11.307544708251953 | Batch Time=8.203125 +GPU:3 | Epoch: 13 | loss=11.525129318237305 | Batch Time=8.59375 +GPU:3 | Epoch: 13 | loss=11.38687515258789 | Batch Time=12.109375 +GPU:3 | Epoch: 13 | loss=11.16108512878418 | Batch Time=15.234375 +GPU:3 | Epoch: 13 | loss=11.484432220458984 | Batch Time=13.671875 +GPU:3 | Epoch: 13 | loss=11.242698669433594 | Batch Time=12.5 +GPU:3 | Epoch: 13 | loss=11.503196716308594 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.300554275512695 | Batch Time=12.109375 +GPU:3 | Epoch: 13 | loss=11.36454963684082 | Batch Time=8.59375 +GPU:3 | Epoch: 13 | loss=11.429314613342285 | Batch Time=14.84375 +GPU:3 | Epoch: 13 | loss=11.413362503051758 | Batch Time=11.71875 +GPU:3 | Epoch: 13 | loss=11.415056228637695 | Batch Time=12.5 +GPU:3 | Epoch: 13 | loss=11.309488296508789 | Batch Time=11.71875 +GPU:3 | Epoch: 13 | loss=11.439521789550781 | Batch Time=9.765625 +GPU:3 | Epoch: 13 | loss=11.462915420532227 | Batch Time=9.375 +GPU:3 | Epoch: 13 | loss=11.590808868408203 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.408668518066406 | Batch Time=12.109375 +GPU:3 | Epoch: 13 | loss=11.334176063537598 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.442251205444336 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.393926620483398 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.29443073272705 | Batch Time=12.890625 +GPU:3 | Epoch: 13 | loss=11.397666931152344 | Batch Time=11.328125 +GPU:3 | Epoch: 13 | loss=11.588762283325195 | Batch Time=9.375 +GPU:3 | Epoch: 13 | loss=11.436836242675781 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.49277114868164 | Batch Time=12.5 +GPU:3 | Epoch: 13 | loss=11.286182403564453 | Batch Time=13.28125 +GPU:3 | Epoch: 13 | loss=11.583796501159668 | Batch Time=10.9375 +GPU:3 | Epoch: 13 | loss=11.367454528808594 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.31283950805664 | Batch Time=14.84375 +GPU:3 | Epoch: 13 | loss=11.319306373596191 | Batch Time=10.546875 +GPU:3 | Epoch: 13 | loss=11.26357650756836 | Batch Time=11.71875 +GPU:3 | Epoch: 13 | loss=11.266443252563477 | Batch Time=13.28125 +GPU:3 | Epoch: 13 | loss=11.389734268188477 | Batch Time=9.375 +GPU:3 | Epoch: 13 | loss=11.583335876464844 | Batch Time=8.203125 +GPU:3 | Epoch: 13 | loss=11.651327133178711 | Batch Time=7.03125 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030666240, percent=48.4, used=116624711680, free=17358999552, active=182720311296, inactive=52378001408, buffers=415137792, cached=123169234944, shared=5661196288, slab=3231035392) +AFTER TRAIN LOOP: Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +BEFORE VAL LOOP: GPU: 0 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030666240, percent=48.4, used=116624711680, free=17358999552, active=182720311296, inactive=52378001408, buffers=415137792, cached=123169234944, shared=5661196288, slab=3231035392) +AFTER TRAIN LOOP: Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +STARTING TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +BEFORE TRAIN LOOP: Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030150144, percent=48.4, used=116625227776, free=17358483456, active=182720311296, inactive=52378001408, buffers=415137792, cached=123169234944, shared=5661196288, slab=3231035392) +AFTER TRAIN LOOP: Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +STARTING TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +BEFORE TRAIN LOOP: Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030150144, percent=48.4, used=116625227776, free=17358483456, active=182720311296, inactive=52378001408, buffers=415137792, cached=123169234944, shared=5661196288, slab=3231035392) +AFTER TRAIN LOOP: Epoch 13 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +STARTING TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +BEFORE TRAIN LOOP: Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=133030985728, percent=48.4, used=116624510976, free=17359273984, active=182720237568, inactive=52377993216, buffers=415137792, cached=123169161216, shared=5661114368, slab=3231019008) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 13 | loss=3.7641992568969727 | Batch Time=21.09375 +GPU:0 | Epoch: 13 | loss=4.6970367431640625 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=4.434534549713135 | Batch Time=21.875 +GPU:0 | Epoch: 13 | loss=5.005231857299805 | Batch Time=7.03125 +GPU:0 | Epoch: 13 | loss=4.8069868087768555 | Batch Time=6.25 +GPU:0 | Epoch: 13 | loss=5.027432918548584 | Batch Time=2.34375 +GPU:0 | Epoch: 13 | loss=5.059717655181885 | Batch Time=5.859375 +GPU:0 | Epoch: 13 | loss=5.273460865020752 | Batch Time=1.953125 +GPU:0 | Epoch: 13 | loss=5.5565032958984375 | Batch Time=0.78125 +GPU:0 | Epoch: 13 | loss=6.085434913635254 | Batch Time=1.5625 +GPU:0 | Epoch: 13 | loss=5.837414264678955 | Batch Time=1.5625 +GPU:0 | Epoch: 13 | loss=4.754918575286865 | Batch Time=12.890625 +GPU:0 | Epoch: 13 | loss=5.95582914352417 | Batch Time=1.953125 +GPU:0 | Epoch: 13 | loss=5.127518177032471 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=5.191589832305908 | Batch Time=3.515625 +GPU:0 | Epoch: 13 | loss=5.509829044342041 | Batch Time=7.421875 +GPU:0 | Epoch: 13 | loss=4.910893440246582 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=4.437804698944092 | Batch Time=14.0625 +GPU:0 | Epoch: 13 | loss=4.457239627838135 | Batch Time=13.671875 +GPU:0 | Epoch: 13 | loss=3.5381078720092773 | Batch Time=32.421875 +Model top1 Accuracy: 10.038 +Acc before rounding: 10.038 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 13 | loss=3.7641992568969727 | Batch Time=21.09375 +GPU:0 | Epoch: 13 | loss=4.6970367431640625 | Batch Time=10.546875 +GPU:0 | Epoch: 13 | loss=4.434534549713135 | Batch Time=21.875 +GPU:0 | Epoch: 13 | loss=5.005231857299805 | Batch Time=7.03125 +GPU:0 | Epoch: 13 | loss=4.8069868087768555 | Batch Time=6.25 +GPU:0 | Epoch: 13 | loss=5.027432918548584 | Batch Time=2.34375 +GPU:0 | Epoch: 13 | loss=5.059717655181885 | Batch Time=5.859375 +GPU:0 | Epoch: 13 | loss=5.273460865020752 | Batch Time=1.953125 +GPU:0 | Epoch: 13 | loss=5.5565032958984375 | Batch Time=0.78125 +GPU:0 | Epoch: 13 | loss=6.085434913635254 | Batch Time=1.5625 +GPU:0 | Epoch: 13 | loss=5.837414264678955 | Batch Time=1.5625 +GPU:0 | Epoch: 13 | loss=4.754918575286865 | Batch Time=12.890625 +GPU:0 | Epoch: 13 | loss=5.95582914352417 | Batch Time=1.953125 +GPU:0 | Epoch: 13 | loss=5.127518177032471 | Batch Time=8.984375 +GPU:0 | Epoch: 13 | loss=5.191589832305908 | Batch Time=3.515625 +GPU:0 | Epoch: 13 | loss=5.509829044342041 | Batch Time=7.421875 +GPU:0 | Epoch: 13 | loss=4.910893440246582 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=4.437804698944092 | Batch Time=14.0625 +GPU:0 | Epoch: 13 | loss=4.457239627838135 | Batch Time=13.671875 +GPU:0 | Epoch: 13 | loss=3.5381078720092773 | Batch Time=32.421875 +Model top1 Accuracy: 10.038 +Acc after rounding: 10.038 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 13 | loss=5.222412109375 | Batch Time=8.203125 +GPU:0 | Epoch: 13 | loss=5.149709224700928 | Batch Time=9.375 +GPU:0 | Epoch: 13 | loss=4.955284118652344 | Batch Time=9.765625 +GPU:0 | Epoch: 13 | loss=5.25376033782959 | Batch Time=6.25 +Model top1 Accuracy: 8.77 +Validation Acc after rounding: 8.77 +AFTER VAL LOOP: GPU: 0 | Epoch 13 | Memory Usage: svmem(total=257568083968, available=115197788160, percent=55.3, used=134457458688, free=5439594496, active=195331158016, inactive=51574083584, buffers=416841728, cached=117254189056, shared=5661347840, slab=3236155392) +Rounding model with scheme: naive +Model avg sparsity: 42.60185705860329 +GPU:0 | Epoch: 13 | Acc=10.038 | Epoch Time=17.359584307670595 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=115197788160, percent=55.3, used=134457458688, free=5439594496, active=195331158016, inactive=51574083584, buffers=416952320, cached=117254078464, shared=5661347840, slab=3236155392) +STARTING TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=115197788160, percent=55.3, used=134457458688, free=5439594496, active=195331158016, inactive=51574083584, buffers=416952320, cached=117254078464, shared=5661347840, slab=3236155392) +BEFORE TRAIN LOOP: Epoch 14 | Memory Usage: svmem(total=257568083968, available=115197788160, percent=55.3, used=134457458688, free=5439594496, active=195331158016, inactive=51574083584, buffers=416952320, cached=117254078464, shared=5661347840, slab=3236155392) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=115197788160, percent=55.3, used=134457458688, free=5439594496, active=195331158016, inactive=51574083584, buffers=416952320, cached=117254078464, shared=5661347840, slab=3236155392) +GPU:1 | Epoch: 14 | loss=11.595048904418945 | Batch Time=10.15625 +GPU:1 | Epoch: 14 | loss=11.29932975769043 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.407668113708496 | Batch Time=13.28125 +GPU:1 | Epoch: 14 | loss=11.255706787109375 | Batch Time=14.84375 +GPU:1 | Epoch: 14 | loss=11.513740539550781 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.408658981323242 | Batch Time=10.15625 +GPU:1 | Epoch: 14 | loss=11.464665412902832 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.028762817382812 | Batch Time=16.015625 +GPU:1 | Epoch: 14 | loss=11.399417877197266 | Batch Time=12.5 +GPU:1 | Epoch: 14 | loss=11.396976470947266 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.302791595458984 | Batch Time=12.5 +GPU:1 | Epoch: 14 | loss=11.3342924118042 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.39057731628418 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.412757873535156 | Batch Time=10.546875 +GPU:1 | Epoch: 14 | loss=11.210911750793457 | Batch Time=10.546875 +GPU:1 | Epoch: 14 | loss=11.458639144897461 | Batch Time=7.8125 +GPU:1 | Epoch: 14 | loss=11.364740371704102 | Batch Time=12.5 +GPU:1 | Epoch: 14 | loss=11.184283256530762 | Batch Time=10.546875 +GPU:1 | Epoch: 14 | loss=11.56399154663086 | Batch Time=10.15625 +GPU:1 | Epoch: 14 | loss=11.329968452453613 | Batch Time=7.8125 +GPU:1 | Epoch: 14 | loss=11.340368270874023 | Batch Time=10.15625 +GPU:1 | Epoch: 14 | loss=11.324960708618164 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.43739128112793 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.530538558959961 | Batch Time=9.375 +GPU:1 | Epoch: 14 | loss=11.437043190002441 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.231582641601562 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.369514465332031 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.566530227661133 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.430017471313477 | Batch Time=13.28125 +GPU:1 | Epoch: 14 | loss=11.350173950195312 | Batch Time=14.84375 +GPU:1 | Epoch: 14 | loss=11.432034492492676 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.370702743530273 | Batch Time=8.203125 +GPU:1 | Epoch: 14 | loss=11.238686561584473 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.278188705444336 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.354978561401367 | Batch Time=8.984375 +GPU:1 | Epoch: 14 | loss=11.282703399658203 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.303912162780762 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.331110000610352 | Batch Time=9.375 +GPU:1 | Epoch: 14 | loss=11.1044921875 | Batch Time=14.0625 +GPU:1 | Epoch: 14 | loss=11.258604049682617 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.254770278930664 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.330196380615234 | Batch Time=7.8125 +GPU:1 | Epoch: 14 | loss=11.374710083007812 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.279102325439453 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.300640106201172 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.29483413696289 | Batch Time=13.671875 +GPU:1 | Epoch: 14 | loss=11.37251091003418 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.324149131774902 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.338567733764648 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.237359046936035 | Batch Time=6.25 +GPU:1 | Epoch: 14 | loss=11.301826477050781 | Batch Time=8.984375 +GPU:1 | Epoch: 14 | loss=11.564844131469727 | Batch Time=8.984375 +GPU:1 | Epoch: 14 | loss=11.39476490020752 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.355043411254883 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.439903259277344 | Batch Time=7.03125 +GPU:1 | Epoch: 14 | loss=11.348724365234375 | Batch Time=14.0625 +GPU:1 | Epoch: 14 | loss=11.166139602661133 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.216058731079102 | Batch Time=13.28125 +GPU:1 | Epoch: 14 | loss=11.452699661254883 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.258169174194336 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.434759140014648 | Batch Time=8.203125 +GPU:1 | Epoch: 14 | loss=11.487260818481445 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.278739929199219 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.221156120300293 | Batch Time=12.5 +GPU:1 | Epoch: 14 | loss=11.406367301940918 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.339237213134766 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.382720947265625 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.33758544921875 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.204557418823242 | Batch Time=14.453125 +GPU:1 | Epoch: 14 | loss=11.187768936157227 | Batch Time=14.0625 +GPU:1 | Epoch: 14 | loss=11.278846740722656 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.364952087402344 | Batch Time=6.25 +GPU:1 | Epoch: 14 | loss=11.207763671875 | Batch Time=13.671875 +GPU:1 | Epoch: 14 | loss=11.229705810546875 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.425716400146484 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.310808181762695 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.158296585083008 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.313610076904297 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.346966743469238 | Batch Time=9.375 +GPU:1 | Epoch: 14 | loss=11.39018440246582 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.231388092041016 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.465116500854492 | Batch Time=6.25 +GPU:1 | Epoch: 14 | loss=11.360219955444336 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.187618255615234 | Batch Time=13.28125 +GPU:1 | Epoch: 14 | loss=11.430692672729492 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.312105178833008 | Batch Time=10.15625 +GPU:1 | Epoch: 14 | loss=11.259780883789062 | Batch Time=7.8125 +GPU:1 | Epoch: 14 | loss=11.45661735534668 | Batch Time=9.375 +GPU:1 | Epoch: 14 | loss=11.166369438171387 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.172622680664062 | Batch Time=13.671875 +GPU:1 | Epoch: 14 | loss=11.076433181762695 | Batch Time=13.671875 +GPU:1 | Epoch: 14 | loss=11.363992691040039 | Batch Time=9.375 +GPU:1 | Epoch: 14 | loss=11.226947784423828 | Batch Time=10.9375 +GPU:1 | Epoch: 14 | loss=11.236539840698242 | Batch Time=13.671875 +GPU:1 | Epoch: 14 | loss=11.19770336151123 | Batch Time=8.984375 +GPU:1 | Epoch: 14 | loss=11.357114791870117 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.036539077758789 | Batch Time=14.453125 +GPU:1 | Epoch: 14 | loss=11.190411567687988 | Batch Time=13.28125 +GPU:1 | Epoch: 14 | loss=11.2694091796875 | Batch Time=10.546875 +GPU:1 | Epoch: 14 | loss=11.130911827087402 | Batch Time=10.15625 +GPU:1 | Epoch: 14 | loss=11.265047073364258 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.17886734008789 | Batch Time=10.546875 +GPU:1 | Epoch: 14 | loss=11.135858535766602 | Batch Time=13.671875 +GPU:1 | Epoch: 14 | loss=11.064373016357422 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.169120788574219 | Batch Time=10.15625 +GPU:1 | Epoch: 14 | loss=11.11413288116455 | Batch Time=13.28125 +GPU:1 | Epoch: 14 | loss=11.291878700256348 | Batch Time=7.421875 +GPU:1 | Epoch: 14 | loss=11.224546432495117 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.365687370300293 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.144115447998047 | Batch Time=11.71875 +GPU:1 | Epoch: 14 | loss=11.347381591796875 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.121818542480469 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.235808372497559 | Batch Time=17.1875 +GPU:1 | Epoch: 14 | loss=11.25139045715332 | Batch Time=12.109375 +GPU:1 | Epoch: 14 | loss=11.100205421447754 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.218184471130371 | Batch Time=11.328125 +GPU:1 | Epoch: 14 | loss=11.117155075073242 | Batch Time=14.0625 +GPU:1 | Epoch: 14 | loss=11.422182083129883 | Batch Time=9.765625 +GPU:1 | Epoch: 14 | loss=11.128995895385742 | Batch Time=14.0625 +GPU:1 | Epoch: 14 | loss=11.196605682373047 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.201314926147461 | Batch Time=9.375 +GPU:1 | Epoch: 14 | loss=11.212197303771973 | Batch Time=12.890625 +GPU:1 | Epoch: 14 | loss=11.203784942626953 | Batch Time=14.453125 +GPU:1 | Epoch: 14 | loss=11.120182991027832 | Batch Time=10.9375 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:3 | Epoch: 14 | loss=11.387748718261719 | Batch Time=8.59375 +GPU:3 | Epoch: 14 | loss=11.468635559082031 | Batch Time=8.59375 +GPU:3 | Epoch: 14 | loss=11.398402214050293 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.386985778808594 | Batch Time=12.5 +GPU:3 | Epoch: 14 | loss=11.325706481933594 | Batch Time=11.328125 +GPU:3 | Epoch: 14 | loss=11.37870979309082 | Batch Time=9.375 +GPU:3 | Epoch: 14 | loss=11.325291633605957 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.304317474365234 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.383075714111328 | Batch Time=11.328125 +GPU:3 | Epoch: 14 | loss=11.38646125793457 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.378482818603516 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.512059211730957 | Batch Time=12.890625 +GPU:3 | Epoch: 14 | loss=11.057853698730469 | Batch Time=14.453125 +GPU:3 | Epoch: 14 | loss=11.297952651977539 | Batch Time=14.453125 +GPU:3 | Epoch: 14 | loss=11.206696510314941 | Batch Time=9.765625 +GPU:3 | Epoch: 14 | loss=11.390316009521484 | Batch Time=16.015625 +GPU:3 | Epoch: 14 | loss=11.478644371032715 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.443865776062012 | Batch Time=8.59375 +GPU:3 | Epoch: 14 | loss=11.4425048828125 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.295736312866211 | Batch Time=11.328125 +GPU:3 | Epoch: 14 | loss=11.274404525756836 | Batch Time=13.28125 +GPU:3 | Epoch: 14 | loss=11.265199661254883 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.311899185180664 | Batch Time=11.328125 +GPU:3 | Epoch: 14 | loss=11.420595169067383 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.614173889160156 | Batch Time=8.59375 +GPU:3 | Epoch: 14 | loss=11.198423385620117 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.46611213684082 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.373771667480469 | Batch Time=9.375 +GPU:3 | Epoch: 14 | loss=11.212308883666992 | Batch Time=13.671875 +GPU:3 | Epoch: 14 | loss=11.183209419250488 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.461617469787598 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.252557754516602 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.431760787963867 | Batch Time=12.890625 +GPU:3 | Epoch: 14 | loss=11.39608383178711 | Batch Time=13.28125 +GPU:3 | Epoch: 14 | loss=11.336531639099121 | Batch Time=8.59375 +GPU:3 | Epoch: 14 | loss=11.16604995727539 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.363476753234863 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.194084167480469 | Batch Time=11.328125 +GPU:3 | Epoch: 14 | loss=11.26611328125 | Batch Time=13.671875 +GPU:3 | Epoch: 14 | loss=11.46076774597168 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.200502395629883 | Batch Time=14.0625 +GPU:3 | Epoch: 14 | loss=11.433744430541992 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.292259216308594 | Batch Time=12.5 +GPU:3 | Epoch: 14 | loss=11.26416015625 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.390039443969727 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.194733619689941 | Batch Time=12.5 +GPU:3 | Epoch: 14 | loss=11.25854778289795 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.452922821044922 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.38172721862793 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.36237907409668 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.42278003692627 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.428645133972168 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.406517028808594 | Batch Time=9.375 +GPU:3 | Epoch: 14 | loss=11.113621711730957 | Batch Time=15.234375 +GPU:3 | Epoch: 14 | loss=11.301240921020508 | Batch Time=13.671875 +GPU:3 | Epoch: 14 | loss=11.252462387084961 | Batch Time=8.984375 +GPU:3 | Epoch: 14 | loss=11.35031509399414 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.324722290039062 | Batch Time=8.203125 +GPU:3 | Epoch: 14 | loss=11.268766403198242 | Batch Time=14.0625 +GPU:3 | Epoch: 14 | loss=11.1633882522583 | Batch Time=14.0625 +GPU:3 | Epoch: 14 | loss=11.287994384765625 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.084522247314453 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.125749588012695 | Batch Time=14.453125 +GPU:3 | Epoch: 14 | loss=11.284412384033203 | Batch Time=13.671875 +GPU:3 | Epoch: 14 | loss=11.286527633666992 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.38734245300293 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.21989631652832 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.198427200317383 | Batch Time=12.5 +GPU:3 | Epoch: 14 | loss=11.395084381103516 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.007030487060547 | Batch Time=13.28125 +GPU:3 | Epoch: 14 | loss=11.359546661376953 | Batch Time=12.890625 +GPU:3 | Epoch: 14 | loss=11.213631629943848 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.249031066894531 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.269390106201172 | Batch Time=12.5 +GPU:3 | Epoch: 14 | loss=11.471355438232422 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.471654891967773 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.221961975097656 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.24477767944336 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.292057991027832 | Batch Time=9.765625 +GPU:3 | Epoch: 14 | loss=11.383079528808594 | Batch Time=8.203125 +GPU:3 | Epoch: 14 | loss=11.269670486450195 | Batch Time=9.375 +GPU:3 | Epoch: 14 | loss=11.217315673828125 | Batch Time=14.0625 +GPU:3 | Epoch: 14 | loss=11.201658248901367 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.066949844360352 | Batch Time=8.984375 +GPU:3 | Epoch: 14 | loss=11.16297721862793 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.160161972045898 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.209555625915527 | Batch Time=13.671875 +GPU:3 | Epoch: 14 | loss=11.286733627319336 | Batch Time=9.375 +GPU:3 | Epoch: 14 | loss=11.13458251953125 | Batch Time=12.890625 +GPU:3 | Epoch: 14 | loss=11.241788864135742 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.19361686706543 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.352115631103516 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.475427627563477 | Batch Time=9.375 +GPU:3 | Epoch: 14 | loss=11.23046875 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.42813491821289 | Batch Time=6.640625 +GPU:3 | Epoch: 14 | loss=11.203330039978027 | Batch Time=10.9375 +GPU:3 | Epoch: 14 | loss=11.26699447631836 | Batch Time=15.625 +GPU:3 | Epoch: 14 | loss=11.304725646972656 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.36229419708252 | Batch Time=7.8125 +GPU:3 | Epoch: 14 | loss=11.219657897949219 | Batch Time=12.890625 +GPU:3 | Epoch: 14 | loss=11.189353942871094 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.159721374511719 | Batch Time=13.28125 +GPU:3 | Epoch: 14 | loss=11.181242942810059 | Batch Time=12.5 +GPU:3 | Epoch: 14 | loss=11.142560005187988 | Batch Time=12.5 +GPU:3 | Epoch: 14 | loss=11.22484016418457 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.286928176879883 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.125741958618164 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.11893081665039 | Batch Time=13.671875 +GPU:3 | Epoch: 14 | loss=11.221500396728516 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.091917991638184 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.31043815612793 | Batch Time=9.765625 +GPU:3 | Epoch: 14 | loss=11.393600463867188 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.370988845825195 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.390785217285156 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.152997016906738 | Batch Time=12.890625 +GPU:3 | Epoch: 14 | loss=11.238452911376953 | Batch Time=11.71875 +GPU:3 | Epoch: 14 | loss=11.241607666015625 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.15173053741455 | Batch Time=12.109375 +GPU:3 | Epoch: 14 | loss=11.108083724975586 | Batch Time=12.890625 +GPU:3 | Epoch: 14 | loss=11.179214477539062 | Batch Time=11.328125 +GPU:3 | Epoch: 14 | loss=11.124673843383789 | Batch Time=10.546875 +GPU:3 | Epoch: 14 | loss=11.21640396118164 | Batch Time=12.890625 +GPU:3 | Epoch: 14 | loss=11.145362854003906 | Batch Time=11.328125 +GPU:3 | Epoch: 14 | loss=11.312480926513672 | Batch Time=10.15625 +GPU:3 | Epoch: 14 | loss=11.07949447631836 | Batch Time=14.84375 +GPU:2 | Epoch: 14 | loss=11.397233963012695 | Batch Time=7.8125 +GPU:2 | Epoch: 14 | loss=11.374404907226562 | Batch Time=8.984375 +GPU:2 | Epoch: 14 | loss=11.281778335571289 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.452125549316406 | Batch Time=9.375 +GPU:2 | Epoch: 14 | loss=11.501335144042969 | Batch Time=10.546875 +GPU:2 | Epoch: 14 | loss=11.322712898254395 | Batch Time=12.5 +GPU:2 | Epoch: 14 | loss=11.336074829101562 | Batch Time=12.109375 +GPU:2 | Epoch: 14 | loss=11.421682357788086 | Batch Time=7.421875 +GPU:2 | Epoch: 14 | loss=11.320642471313477 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.186460494995117 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.556519508361816 | Batch Time=11.328125 +GPU:2 | Epoch: 14 | loss=11.566864967346191 | Batch Time=10.15625 +GPU:2 | Epoch: 14 | loss=11.40125846862793 | Batch Time=10.546875 +GPU:2 | Epoch: 14 | loss=11.49819564819336 | Batch Time=8.203125 +GPU:2 | Epoch: 14 | loss=11.449972152709961 | Batch Time=8.984375 +GPU:2 | Epoch: 14 | loss=11.458080291748047 | Batch Time=10.546875 +GPU:2 | Epoch: 14 | loss=11.314864158630371 | Batch Time=12.890625 +GPU:2 | Epoch: 14 | loss=11.319968223571777 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.247024536132812 | Batch Time=8.59375 +GPU:2 | Epoch: 14 | loss=11.440643310546875 | Batch Time=9.375 +GPU:2 | Epoch: 14 | loss=11.675813674926758 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.363424301147461 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.282819747924805 | Batch Time=10.15625 +GPU:2 | Epoch: 14 | loss=11.295141220092773 | Batch Time=8.203125 +GPU:2 | Epoch: 14 | loss=11.218828201293945 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.21820068359375 | Batch Time=12.890625 +GPU:2 | Epoch: 14 | loss=11.520502090454102 | Batch Time=8.203125 +GPU:2 | Epoch: 14 | loss=11.501091957092285 | Batch Time=7.421875 +GPU:2 | Epoch: 14 | loss=11.44304084777832 | Batch Time=11.328125 +GPU:2 | Epoch: 14 | loss=11.188937187194824 | Batch Time=14.84375 +GPU:2 | Epoch: 14 | loss=11.237410545349121 | Batch Time=12.5 +GPU:2 | Epoch: 14 | loss=11.311153411865234 | Batch Time=10.15625 +GPU:2 | Epoch: 14 | loss=11.644869804382324 | Batch Time=5.46875 +GPU:2 | Epoch: 14 | loss=11.20632553100586 | Batch Time=8.59375 +GPU:2 | Epoch: 14 | loss=11.207027435302734 | Batch Time=12.890625 +GPU:2 | Epoch: 14 | loss=11.352032661437988 | Batch Time=11.328125 +GPU:2 | Epoch: 14 | loss=11.376253128051758 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.340511322021484 | Batch Time=11.328125 +GPU:2 | Epoch: 14 | loss=11.378410339355469 | Batch Time=12.109375 +GPU:2 | Epoch: 14 | loss=11.456010818481445 | Batch Time=10.15625 +GPU:2 | Epoch: 14 | loss=11.322135925292969 | Batch Time=8.984375 +GPU:2 | Epoch: 14 | loss=11.47950553894043 | Batch Time=7.8125 +GPU:2 | Epoch: 14 | loss=11.327825546264648 | Batch Time=11.328125 +GPU:2 | Epoch: 14 | loss=11.304132461547852 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.473381042480469 | Batch Time=8.59375 +GPU:2 | Epoch: 14 | loss=11.361104965209961 | Batch Time=7.8125 +GPU:2 | Epoch: 14 | loss=11.25082778930664 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.114374160766602 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.161849975585938 | Batch Time=13.671875 +GPU:2 | Epoch: 14 | loss=11.612299919128418 | Batch Time=5.46875 +GPU:2 | Epoch: 14 | loss=11.321127891540527 | Batch Time=10.546875 +GPU:2 | Epoch: 14 | loss=11.287707328796387 | Batch Time=12.109375 +GPU:2 | Epoch: 14 | loss=11.286977767944336 | Batch Time=9.765625 +GPU:2 | Epoch: 14 | loss=11.254091262817383 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.236639976501465 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.207579612731934 | Batch Time=12.5 +GPU:2 | Epoch: 14 | loss=11.493875503540039 | Batch Time=8.984375 +GPU:2 | Epoch: 14 | loss=11.20661735534668 | Batch Time=9.375 +GPU:2 | Epoch: 14 | loss=11.305977821350098 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.51231575012207 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.297597885131836 | Batch Time=12.890625 +GPU:2 | Epoch: 14 | loss=11.270177841186523 | Batch Time=12.109375 +GPU:2 | Epoch: 14 | loss=11.394956588745117 | Batch Time=8.984375 +GPU:2 | Epoch: 14 | loss=11.300010681152344 | Batch Time=11.328125 +GPU:2 | Epoch: 14 | loss=11.056488037109375 | Batch Time=14.0625 +GPU:2 | Epoch: 14 | loss=11.139793395996094 | Batch Time=15.234375 +GPU:2 | Epoch: 14 | loss=11.281944274902344 | Batch Time=11.328125 +GPU:2 | Epoch: 14 | loss=11.253469467163086 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.215038299560547 | Batch Time=14.84375 +GPU:2 | Epoch: 14 | loss=11.345760345458984 | Batch Time=10.546875 +GPU:2 | Epoch: 14 | loss=11.308534622192383 | Batch Time=9.765625 +GPU:2 | Epoch: 14 | loss=11.337503433227539 | Batch Time=9.765625 +GPU:2 | Epoch: 14 | loss=11.262126922607422 | Batch Time=8.59375 +GPU:2 | Epoch: 14 | loss=11.199211120605469 | Batch Time=14.84375 +GPU:2 | Epoch: 14 | loss=11.347929000854492 | Batch Time=9.375 +GPU:2 | Epoch: 14 | loss=11.45864200592041 | Batch Time=10.15625 +GPU:2 | Epoch: 14 | loss=11.216797828674316 | Batch Time=15.625 +GPU:2 | Epoch: 14 | loss=11.271429061889648 | Batch Time=10.15625 +GPU:2 | Epoch: 14 | loss=11.115976333618164 | Batch Time=13.28125 +GPU:2 | Epoch: 14 | loss=11.261674880981445 | Batch Time=12.5 +GPU:2 | Epoch: 14 | loss=11.37134838104248 | Batch Time=9.765625 +GPU:2 | Epoch: 14 | loss=11.07632827758789 | Batch Time=17.1875 +GPU:2 | Epoch: 14 | loss=11.423460006713867 | Batch Time=12.109375 +GPU:2 | Epoch: 14 | loss=11.315905570983887 | Batch Time=9.375 +GPU:2 | Epoch: 14 | loss=11.204694747924805 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.390629768371582 | Batch Time=7.421875 +GPU:2 | Epoch: 14 | loss=11.26763916015625 | Batch Time=12.5 +GPU:2 | Epoch: 14 | loss=11.112251281738281 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.03176498413086 | Batch Time=14.453125 +GPU:2 | Epoch: 14 | loss=11.488283157348633 | Batch Time=9.765625 +GPU:2 | Epoch: 14 | loss=11.35137939453125 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.210823059082031 | Batch Time=10.15625 +GPU:2 | Epoch: 14 | loss=11.334120750427246 | Batch Time=12.5 +GPU:2 | Epoch: 14 | loss=11.274812698364258 | Batch Time=12.890625 +GPU:2 | Epoch: 14 | loss=11.289405822753906 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.339149475097656 | Batch Time=12.5 +GPU:2 | Epoch: 14 | loss=11.001649856567383 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.34180736541748 | Batch Time=7.8125 +GPU:2 | Epoch: 14 | loss=11.296319961547852 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.484841346740723 | Batch Time=5.859375 +GPU:2 | Epoch: 14 | loss=11.287885665893555 | Batch Time=9.765625 +GPU:2 | Epoch: 14 | loss=11.303213119506836 | Batch Time=14.0625 +GPU:2 | Epoch: 14 | loss=11.299776077270508 | Batch Time=11.328125 +GPU:2 | Epoch: 14 | loss=11.213948249816895 | Batch Time=12.890625 +GPU:2 | Epoch: 14 | loss=11.24256706237793 | Batch Time=10.546875 +GPU:2 | Epoch: 14 | loss=11.251081466674805 | Batch Time=12.890625 +GPU:2 | Epoch: 14 | loss=11.3253173828125 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.395692825317383 | Batch Time=8.203125 +GPU:2 | Epoch: 14 | loss=11.064996719360352 | Batch Time=13.671875 +GPU:2 | Epoch: 14 | loss=11.446264266967773 | Batch Time=10.9375 +GPU:2 | Epoch: 14 | loss=11.274309158325195 | Batch Time=7.421875 +GPU:2 | Epoch: 14 | loss=11.407983779907227 | Batch Time=8.59375 +GPU:2 | Epoch: 14 | loss=11.196432113647461 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.193258285522461 | Batch Time=14.0625 +GPU:2 | Epoch: 14 | loss=11.21352767944336 | Batch Time=12.890625 +GPU:2 | Epoch: 14 | loss=11.320850372314453 | Batch Time=9.765625 +GPU:2 | Epoch: 14 | loss=11.233041763305664 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.38674545288086 | Batch Time=11.71875 +GPU:2 | Epoch: 14 | loss=11.388345718383789 | Batch Time=12.109375 +GPU:2 | Epoch: 14 | loss=11.151945114135742 | Batch Time=10.546875 +GPU:2 | Epoch: 14 | loss=11.162372589111328 | Batch Time=10.546875 +GPU:2 | Epoch: 14 | loss=11.337716102600098 | Batch Time=10.15625 +GPU:2 | Epoch: 14 | loss=11.114843368530273 | Batch Time=14.0625 +GPU:2 | Epoch: 14 | loss=11.197946548461914 | Batch Time=8.984375 +GPU:2 | Epoch: 14 | loss=11.109946250915527 | Batch Time=14.453125 +GPU:0 | Epoch: 14 | loss=11.322103500366211 | Batch Time=14.453125 +GPU:0 | Epoch: 14 | loss=11.502609252929688 | Batch Time=9.375 +GPU:0 | Epoch: 14 | loss=11.371585845947266 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.246273040771484 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.358016967773438 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.457112312316895 | Batch Time=8.984375 +GPU:0 | Epoch: 14 | loss=11.374998092651367 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.188756942749023 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.376605987548828 | Batch Time=9.375 +GPU:0 | Epoch: 14 | loss=11.292381286621094 | Batch Time=14.0625 +GPU:0 | Epoch: 14 | loss=11.518999099731445 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.486258506774902 | Batch Time=7.421875 +GPU:0 | Epoch: 14 | loss=11.473104476928711 | Batch Time=8.59375 +GPU:0 | Epoch: 14 | loss=11.57802963256836 | Batch Time=8.203125 +GPU:0 | Epoch: 14 | loss=11.631340980529785 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.191160202026367 | Batch Time=13.28125 +GPU:0 | Epoch: 14 | loss=11.305968284606934 | Batch Time=10.546875 +GPU:0 | Epoch: 14 | loss=11.545804977416992 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.309484481811523 | Batch Time=12.890625 +GPU:0 | Epoch: 14 | loss=11.386005401611328 | Batch Time=10.546875 +GPU:0 | Epoch: 14 | loss=11.476667404174805 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.339495658874512 | Batch Time=14.0625 +GPU:0 | Epoch: 14 | loss=11.392526626586914 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.33564567565918 | Batch Time=11.71875 +GPU:0 | Epoch: 14 | loss=11.383005142211914 | Batch Time=8.984375 +GPU:0 | Epoch: 14 | loss=11.432870864868164 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=11.078250885009766 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.225976943969727 | Batch Time=11.71875 +GPU:0 | Epoch: 14 | loss=11.179449081420898 | Batch Time=12.890625 +GPU:0 | Epoch: 14 | loss=11.39692497253418 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.380489349365234 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.435495376586914 | Batch Time=8.59375 +GPU:0 | Epoch: 14 | loss=11.410266876220703 | Batch Time=14.0625 +GPU:0 | Epoch: 14 | loss=11.52618408203125 | Batch Time=5.859375 +GPU:0 | Epoch: 14 | loss=11.278066635131836 | Batch Time=12.5 +GPU:0 | Epoch: 14 | loss=11.136608123779297 | Batch Time=17.96875 +GPU:0 | Epoch: 14 | loss=11.30318832397461 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.433815002441406 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.124994277954102 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.317447662353516 | Batch Time=8.59375 +GPU:0 | Epoch: 14 | loss=11.248567581176758 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.291950225830078 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.22383975982666 | Batch Time=15.234375 +GPU:0 | Epoch: 14 | loss=11.341552734375 | Batch Time=8.59375 +GPU:0 | Epoch: 14 | loss=11.291142463684082 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=11.249469757080078 | Batch Time=8.59375 +GPU:0 | Epoch: 14 | loss=11.234073638916016 | Batch Time=14.0625 +GPU:0 | Epoch: 14 | loss=11.28459358215332 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.390287399291992 | Batch Time=12.890625 +GPU:0 | Epoch: 14 | loss=11.371912002563477 | Batch Time=10.546875 +GPU:0 | Epoch: 14 | loss=11.272722244262695 | Batch Time=11.71875 +GPU:0 | Epoch: 14 | loss=11.330252647399902 | Batch Time=8.59375 +GPU:0 | Epoch: 14 | loss=11.369708061218262 | Batch Time=11.71875 +GPU:0 | Epoch: 14 | loss=11.305066108703613 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.204299926757812 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.105294227600098 | Batch Time=12.5 +GPU:0 | Epoch: 14 | loss=11.522176742553711 | Batch Time=7.421875 +GPU:0 | Epoch: 14 | loss=11.292890548706055 | Batch Time=11.328125 +GPU:0 | Epoch: 14 | loss=11.24203109741211 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=11.30322265625 | Batch Time=11.71875 +GPU:0 | Epoch: 14 | loss=11.215372085571289 | Batch Time=9.375 +GPU:0 | Epoch: 14 | loss=11.427785873413086 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.216293334960938 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=11.279603004455566 | Batch Time=14.453125 +GPU:0 | Epoch: 14 | loss=11.180540084838867 | Batch Time=13.28125 +GPU:0 | Epoch: 14 | loss=11.268987655639648 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=11.123160362243652 | Batch Time=13.28125 +GPU:0 | Epoch: 14 | loss=11.248811721801758 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.247203826904297 | Batch Time=10.546875 +GPU:0 | Epoch: 14 | loss=11.306467056274414 | Batch Time=11.71875 +GPU:0 | Epoch: 14 | loss=11.373405456542969 | Batch Time=9.375 +GPU:0 | Epoch: 14 | loss=11.189313888549805 | Batch Time=12.5 +GPU:0 | Epoch: 14 | loss=11.199979782104492 | Batch Time=12.890625 +GPU:0 | Epoch: 14 | loss=11.43657112121582 | Batch Time=13.28125 +GPU:0 | Epoch: 14 | loss=11.299457550048828 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.320497512817383 | Batch Time=6.640625 +GPU:0 | Epoch: 14 | loss=11.307668685913086 | Batch Time=9.375 +GPU:0 | Epoch: 14 | loss=11.300642013549805 | Batch Time=8.203125 +GPU:0 | Epoch: 14 | loss=11.40765380859375 | Batch Time=11.71875 +GPU:0 | Epoch: 14 | loss=11.384804725646973 | Batch Time=16.40625 +GPU:0 | Epoch: 14 | loss=11.356710433959961 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=10.966686248779297 | Batch Time=17.1875 +GPU:0 | Epoch: 14 | loss=11.261945724487305 | Batch Time=16.40625 +GPU:0 | Epoch: 14 | loss=11.15601921081543 | Batch Time=12.5 +GPU:0 | Epoch: 14 | loss=11.159303665161133 | Batch Time=13.28125 +GPU:0 | Epoch: 14 | loss=11.215188980102539 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.30832576751709 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.111608505249023 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.285028457641602 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.295108795166016 | Batch Time=10.9375 +GPU:0 | Epoch: 14 | loss=11.208667755126953 | Batch Time=10.546875 +GPU:0 | Epoch: 14 | loss=11.360298156738281 | Batch Time=12.5 +GPU:0 | Epoch: 14 | loss=11.385459899902344 | Batch Time=11.71875 +GPU:0 | Epoch: 14 | loss=11.220067977905273 | Batch Time=14.453125 +GPU:0 | Epoch: 14 | loss=11.293315887451172 | Batch Time=8.984375 +GPU:0 | Epoch: 14 | loss=11.155080795288086 | Batch Time=13.671875 +GPU:0 | Epoch: 14 | loss=11.298410415649414 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.165239334106445 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=11.20718002319336 | Batch Time=13.28125 +GPU:0 | Epoch: 14 | loss=11.344854354858398 | Batch Time=11.328125 +GPU:0 | Epoch: 14 | loss=11.154786109924316 | Batch Time=14.0625 +GPU:0 | Epoch: 14 | loss=11.268009185791016 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.112472534179688 | Batch Time=15.625 +GPU:0 | Epoch: 14 | loss=11.42751407623291 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.188261032104492 | Batch Time=8.59375 +GPU:0 | Epoch: 14 | loss=11.343936920166016 | Batch Time=8.203125 +GPU:0 | Epoch: 14 | loss=11.117986679077148 | Batch Time=14.453125 +GPU:0 | Epoch: 14 | loss=11.270859718322754 | Batch Time=11.328125 +GPU:0 | Epoch: 14 | loss=11.076061248779297 | Batch Time=12.890625 +GPU:0 | Epoch: 14 | loss=11.202754974365234 | Batch Time=13.28125 +GPU:0 | Epoch: 14 | loss=11.300455093383789 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=10.970728874206543 | Batch Time=14.0625 +GPU:0 | Epoch: 14 | loss=11.301716804504395 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=11.260919570922852 | Batch Time=9.375 +GPU:0 | Epoch: 14 | loss=11.271227836608887 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=10.982760429382324 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=11.380420684814453 | Batch Time=10.546875 +GPU:0 | Epoch: 14 | loss=11.17398738861084 | Batch Time=11.328125 +GPU:0 | Epoch: 14 | loss=11.421163558959961 | Batch Time=4.6875 +GPU:0 | Epoch: 14 | loss=11.138044357299805 | Batch Time=14.84375 +GPU:0 | Epoch: 14 | loss=11.117719650268555 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=11.343430519104004 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=11.16215705871582 | Batch Time=12.890625 +GPU:0 | Epoch: 14 | loss=11.131730079650879 | Batch Time=12.5 +GPU:0 | Epoch: 14 | loss=11.170110702514648 | Batch Time=10.15625 +GPU:1 | Epoch: 14 | loss=11.076478958129883 | Batch Time=13.671875 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=125489451008, percent=51.3, used=124165865472, free=17265840128, active=183824789504, inactive=51298119680, buffers=410378240, cached=115726000128, shared=5661310976, slab=3233030144) +AFTER TRAIN LOOP: Epoch 14 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +STARTING TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +BEFORE TRAIN LOOP: Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=125489451008, percent=51.3, used=124165865472, free=17265840128, active=183824789504, inactive=51298119680, buffers=410378240, cached=115726000128, shared=5661310976, slab=3233030144) +AFTER TRAIN LOOP: Epoch 14 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +STARTING TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +BEFORE TRAIN LOOP: Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=125489451008, percent=51.3, used=124165865472, free=17265840128, active=183824789504, inactive=51298119680, buffers=410378240, cached=115726000128, shared=5661310976, slab=3233030144) +AFTER TRAIN LOOP: Epoch 14 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +STARTING TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +BEFORE TRAIN LOOP: Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=125489967104, percent=51.3, used=124165349376, free=17266356224, active=183824789504, inactive=51298119680, buffers=410378240, cached=115726000128, shared=5661310976, slab=3233030144) +AFTER TRAIN LOOP: Epoch 14 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +BEFORE VAL LOOP: GPU: 0 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=125490257920, percent=51.3, used=124165361664, free=17266647040, active=183824171008, inactive=51297943552, buffers=410378240, cached=115725697024, shared=5660983296, slab=3233013760) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 14 | loss=4.5324177742004395 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=5.6552252769470215 | Batch Time=5.46875 +GPU:0 | Epoch: 14 | loss=5.404822826385498 | Batch Time=14.84375 +GPU:0 | Epoch: 14 | loss=5.205772399902344 | Batch Time=7.03125 +GPU:0 | Epoch: 14 | loss=6.562313079833984 | Batch Time=0.78125 +GPU:0 | Epoch: 14 | loss=5.381644248962402 | Batch Time=5.078125 +GPU:0 | Epoch: 14 | loss=5.330379009246826 | Batch Time=2.734375 +GPU:0 | Epoch: 14 | loss=5.992331027984619 | Batch Time=3.515625 +GPU:0 | Epoch: 14 | loss=5.606138706207275 | Batch Time=3.125 +GPU:0 | Epoch: 14 | loss=6.4462571144104 | Batch Time=0.390625 +GPU:0 | Epoch: 14 | loss=5.99070405960083 | Batch Time=0.78125 +GPU:0 | Epoch: 14 | loss=5.435708045959473 | Batch Time=6.640625 +GPU:0 | Epoch: 14 | loss=5.962281227111816 | Batch Time=3.125 +GPU:0 | Epoch: 14 | loss=5.3130364418029785 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=5.016754627227783 | Batch Time=5.46875 +GPU:0 | Epoch: 14 | loss=5.346332550048828 | Batch Time=10.546875 +GPU:0 | Epoch: 14 | loss=5.596469402313232 | Batch Time=1.5625 +GPU:0 | Epoch: 14 | loss=4.705103874206543 | Batch Time=17.1875 +GPU:0 | Epoch: 14 | loss=4.166418075561523 | Batch Time=15.625 +GPU:0 | Epoch: 14 | loss=4.224815845489502 | Batch Time=23.4375 +Model top1 Accuracy: 8.546 +Acc before rounding: 8.546 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 14 | loss=4.5324177742004395 | Batch Time=12.109375 +GPU:0 | Epoch: 14 | loss=5.6552252769470215 | Batch Time=5.46875 +GPU:0 | Epoch: 14 | loss=5.404822826385498 | Batch Time=14.84375 +GPU:0 | Epoch: 14 | loss=5.205772399902344 | Batch Time=7.03125 +GPU:0 | Epoch: 14 | loss=6.562313079833984 | Batch Time=0.78125 +GPU:0 | Epoch: 14 | loss=5.381644248962402 | Batch Time=5.078125 +GPU:0 | Epoch: 14 | loss=5.330379009246826 | Batch Time=2.734375 +GPU:0 | Epoch: 14 | loss=5.992331027984619 | Batch Time=3.515625 +GPU:0 | Epoch: 14 | loss=5.606138706207275 | Batch Time=3.125 +GPU:0 | Epoch: 14 | loss=6.4462571144104 | Batch Time=0.390625 +GPU:0 | Epoch: 14 | loss=5.99070405960083 | Batch Time=0.78125 +GPU:0 | Epoch: 14 | loss=5.435708045959473 | Batch Time=6.640625 +GPU:0 | Epoch: 14 | loss=5.962281227111816 | Batch Time=3.125 +GPU:0 | Epoch: 14 | loss=5.3130364418029785 | Batch Time=10.15625 +GPU:0 | Epoch: 14 | loss=5.016754627227783 | Batch Time=5.46875 +GPU:0 | Epoch: 14 | loss=5.346332550048828 | Batch Time=10.546875 +GPU:0 | Epoch: 14 | loss=5.596469402313232 | Batch Time=1.5625 +GPU:0 | Epoch: 14 | loss=4.705103874206543 | Batch Time=17.1875 +GPU:0 | Epoch: 14 | loss=4.166418075561523 | Batch Time=15.625 +GPU:0 | Epoch: 14 | loss=4.224815845489502 | Batch Time=23.4375 +Model top1 Accuracy: 8.546 +Acc after rounding: 8.546 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 14 | loss=5.668009281158447 | Batch Time=9.765625 +GPU:0 | Epoch: 14 | loss=5.522944927215576 | Batch Time=5.46875 +GPU:0 | Epoch: 14 | loss=5.441335201263428 | Batch Time=8.203125 +GPU:0 | Epoch: 14 | loss=5.690935134887695 | Batch Time=7.03125 +Model top1 Accuracy: 7.53 +Validation Acc after rounding: 7.53 +AFTER VAL LOOP: GPU: 0 | Epoch 14 | Memory Usage: svmem(total=257568083968, available=107704733696, percent=58.2, used=141950427136, free=5428809728, active=201361330176, inactive=45480906752, buffers=411693056, cached=109777154048, shared=5661458432, slab=3235999744) +Rounding model with scheme: naive +Model avg sparsity: 42.0551388720761 +GPU:0 | Epoch: 14 | Acc=8.546 | Epoch Time=17.06425609588623 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=107704741888, percent=58.2, used=141950504960, free=5428809728, active=201361268736, inactive=45480873984, buffers=411697152, cached=109777072128, shared=5661376512, slab=3235999744) +STARTING TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=107704741888, percent=58.2, used=141950504960, free=5428809728, active=201361268736, inactive=45480873984, buffers=411697152, cached=109777072128, shared=5661376512, slab=3235999744) +BEFORE TRAIN LOOP: Epoch 15 | Memory Usage: svmem(total=257568083968, available=107704741888, percent=58.2, used=141950504960, free=5428809728, active=201361268736, inactive=45480873984, buffers=411697152, cached=109777072128, shared=5661376512, slab=3235999744) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=107704741888, percent=58.2, used=141950504960, free=5428809728, active=201361268736, inactive=45480873984, buffers=411697152, cached=109777072128, shared=5661376512, slab=3235999744) +GPU:1 | Epoch: 15 | loss=11.163223266601562 | Batch Time=13.671875 +GPU:1 | Epoch: 15 | loss=11.146480560302734 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=11.218185424804688 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=11.257705688476562 | Batch Time=9.375 +GPU:1 | Epoch: 15 | loss=11.130060195922852 | Batch Time=9.375 +GPU:1 | Epoch: 15 | loss=11.169973373413086 | Batch Time=13.671875 +GPU:1 | Epoch: 15 | loss=11.326393127441406 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=11.316322326660156 | Batch Time=10.546875 +GPU:1 | Epoch: 15 | loss=11.176278114318848 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=11.227046966552734 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.029321670532227 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=11.204939842224121 | Batch Time=10.546875 +GPU:1 | Epoch: 15 | loss=11.218269348144531 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=11.345195770263672 | Batch Time=10.15625 +GPU:1 | Epoch: 15 | loss=11.207134246826172 | Batch Time=8.59375 +GPU:1 | Epoch: 15 | loss=11.24371337890625 | Batch Time=9.375 +GPU:1 | Epoch: 15 | loss=11.100318908691406 | Batch Time=16.40625 +GPU:1 | Epoch: 15 | loss=11.277741432189941 | Batch Time=10.546875 +GPU:1 | Epoch: 15 | loss=11.05296516418457 | Batch Time=10.15625 +GPU:1 | Epoch: 15 | loss=10.909900665283203 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=11.091558456420898 | Batch Time=16.40625 +GPU:1 | Epoch: 15 | loss=11.192670822143555 | Batch Time=9.375 +GPU:1 | Epoch: 15 | loss=11.093729019165039 | Batch Time=8.984375 +GPU:1 | Epoch: 15 | loss=11.323233604431152 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.055079460144043 | Batch Time=12.5 +GPU:1 | Epoch: 15 | loss=11.19005012512207 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=11.095550537109375 | Batch Time=10.546875 +GPU:1 | Epoch: 15 | loss=10.996675491333008 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=11.274328231811523 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=10.944536209106445 | Batch Time=14.84375 +GPU:1 | Epoch: 15 | loss=11.062829971313477 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=11.16145133972168 | Batch Time=10.546875 +GPU:1 | Epoch: 15 | loss=11.204225540161133 | Batch Time=13.28125 +GPU:1 | Epoch: 15 | loss=10.828619956970215 | Batch Time=14.0625 +GPU:1 | Epoch: 15 | loss=11.198554992675781 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=11.21955680847168 | Batch Time=9.375 +GPU:1 | Epoch: 15 | loss=11.179512977600098 | Batch Time=13.28125 +GPU:1 | Epoch: 15 | loss=11.176895141601562 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=11.186357498168945 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=11.20702075958252 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.10060977935791 | Batch Time=12.5 +GPU:1 | Epoch: 15 | loss=11.158016204833984 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=11.105733871459961 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=10.895784378051758 | Batch Time=13.671875 +GPU:1 | Epoch: 15 | loss=11.104964256286621 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=11.164651870727539 | Batch Time=16.015625 +GPU:1 | Epoch: 15 | loss=11.029167175292969 | Batch Time=12.890625 +GPU:1 | Epoch: 15 | loss=10.907320022583008 | Batch Time=15.234375 +GPU:1 | Epoch: 15 | loss=10.926522254943848 | Batch Time=13.671875 +GPU:1 | Epoch: 15 | loss=11.18683910369873 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=11.074546813964844 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=11.025687217712402 | Batch Time=12.890625 +GPU:1 | Epoch: 15 | loss=10.908632278442383 | Batch Time=15.625 +GPU:1 | Epoch: 15 | loss=10.966963768005371 | Batch Time=14.453125 +GPU:1 | Epoch: 15 | loss=11.152409553527832 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=11.314702987670898 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.176615715026855 | Batch Time=15.234375 +GPU:1 | Epoch: 15 | loss=11.086488723754883 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=10.895899772644043 | Batch Time=16.796875 +GPU:1 | Epoch: 15 | loss=11.201215744018555 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.146459579467773 | Batch Time=14.0625 +GPU:1 | Epoch: 15 | loss=11.133947372436523 | Batch Time=12.5 +GPU:1 | Epoch: 15 | loss=10.999488830566406 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=10.978135108947754 | Batch Time=15.625 +GPU:1 | Epoch: 15 | loss=11.074103355407715 | Batch Time=10.15625 +GPU:1 | Epoch: 15 | loss=11.148887634277344 | Batch Time=12.890625 +GPU:1 | Epoch: 15 | loss=11.209163665771484 | Batch Time=7.421875 +GPU:1 | Epoch: 15 | loss=11.175912857055664 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.080595016479492 | Batch Time=13.671875 +GPU:1 | Epoch: 15 | loss=11.061339378356934 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=11.152250289916992 | Batch Time=12.890625 +GPU:1 | Epoch: 15 | loss=11.089550018310547 | Batch Time=14.453125 +GPU:1 | Epoch: 15 | loss=11.141809463500977 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=11.254251480102539 | Batch Time=13.28125 +GPU:1 | Epoch: 15 | loss=11.106515884399414 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=11.263376235961914 | Batch Time=10.15625 +GPU:1 | Epoch: 15 | loss=11.060855865478516 | Batch Time=14.84375 +GPU:1 | Epoch: 15 | loss=11.185163497924805 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=10.93370246887207 | Batch Time=14.84375 +GPU:1 | Epoch: 15 | loss=10.95772647857666 | Batch Time=13.28125 +GPU:1 | Epoch: 15 | loss=11.233019828796387 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=10.97093391418457 | Batch Time=12.5 +GPU:1 | Epoch: 15 | loss=10.841808319091797 | Batch Time=14.0625 +GPU:1 | Epoch: 15 | loss=11.045819282531738 | Batch Time=12.890625 +GPU:1 | Epoch: 15 | loss=11.005777359008789 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=11.060701370239258 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.162484169006348 | Batch Time=10.15625 +GPU:1 | Epoch: 15 | loss=10.903875350952148 | Batch Time=14.0625 +GPU:1 | Epoch: 15 | loss=11.03689956665039 | Batch Time=13.28125 +GPU:1 | Epoch: 15 | loss=11.073113441467285 | Batch Time=12.890625 +GPU:1 | Epoch: 15 | loss=11.048114776611328 | Batch Time=13.28125 +GPU:1 | Epoch: 15 | loss=11.099178314208984 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=10.988174438476562 | Batch Time=14.84375 +GPU:1 | Epoch: 15 | loss=11.162508010864258 | Batch Time=12.5 +GPU:1 | Epoch: 15 | loss=10.873737335205078 | Batch Time=17.578125 +GPU:1 | Epoch: 15 | loss=10.946915626525879 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=10.88365364074707 | Batch Time=14.84375 +GPU:1 | Epoch: 15 | loss=11.08878231048584 | Batch Time=12.890625 +GPU:1 | Epoch: 15 | loss=10.926807403564453 | Batch Time=14.84375 +GPU:1 | Epoch: 15 | loss=11.32992172241211 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=10.859273910522461 | Batch Time=12.5 +GPU:1 | Epoch: 15 | loss=11.049494743347168 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=11.027853012084961 | Batch Time=12.5 +GPU:1 | Epoch: 15 | loss=11.076571464538574 | Batch Time=13.671875 +GPU:1 | Epoch: 15 | loss=11.063154220581055 | Batch Time=10.9375 +GPU:1 | Epoch: 15 | loss=10.873760223388672 | Batch Time=13.28125 +GPU:1 | Epoch: 15 | loss=11.073464393615723 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=11.09920883178711 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.191349029541016 | Batch Time=9.375 +GPU:1 | Epoch: 15 | loss=10.892257690429688 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=10.814888000488281 | Batch Time=13.671875 +GPU:1 | Epoch: 15 | loss=11.183833122253418 | Batch Time=11.328125 +GPU:1 | Epoch: 15 | loss=11.112343788146973 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=11.000354766845703 | Batch Time=16.015625 +GPU:1 | Epoch: 15 | loss=11.25390911102295 | Batch Time=10.15625 +GPU:1 | Epoch: 15 | loss=11.070812225341797 | Batch Time=9.765625 +GPU:1 | Epoch: 15 | loss=11.085512161254883 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=10.974128723144531 | Batch Time=11.71875 +GPU:1 | Epoch: 15 | loss=11.000800132751465 | Batch Time=14.0625 +GPU:1 | Epoch: 15 | loss=11.051371574401855 | Batch Time=14.0625 +GPU:1 | Epoch: 15 | loss=10.954906463623047 | Batch Time=13.28125 +GPU:1 | Epoch: 15 | loss=11.050954818725586 | Batch Time=12.109375 +GPU:1 | Epoch: 15 | loss=10.950862884521484 | Batch Time=13.671875 +GPU:1 | Epoch: 15 | loss=11.005191802978516 | Batch Time=8.59375 +GPU:2 | Epoch: 15 | loss=11.268616676330566 | Batch Time=13.28125 +GPU:2 | Epoch: 15 | loss=11.350278854370117 | Batch Time=9.375 +GPU:2 | Epoch: 15 | loss=11.230829238891602 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.135663986206055 | Batch Time=9.765625 +GPU:2 | Epoch: 15 | loss=11.054967880249023 | Batch Time=12.109375 +GPU:2 | Epoch: 15 | loss=11.191934585571289 | Batch Time=13.28125 +GPU:2 | Epoch: 15 | loss=11.142804145812988 | Batch Time=14.84375 +GPU:2 | Epoch: 15 | loss=11.287643432617188 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=10.895200729370117 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.118831634521484 | Batch Time=10.546875 +GPU:2 | Epoch: 15 | loss=11.301595687866211 | Batch Time=12.890625 +GPU:2 | Epoch: 15 | loss=11.292590141296387 | Batch Time=8.984375 +GPU:2 | Epoch: 15 | loss=11.061270713806152 | Batch Time=16.796875 +GPU:2 | Epoch: 15 | loss=11.182430267333984 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.130542755126953 | Batch Time=10.15625 +GPU:2 | Epoch: 15 | loss=11.23139476776123 | Batch Time=8.984375 +GPU:2 | Epoch: 15 | loss=11.04798698425293 | Batch Time=9.765625 +GPU:2 | Epoch: 15 | loss=11.176231384277344 | Batch Time=10.546875 +GPU:2 | Epoch: 15 | loss=11.055036544799805 | Batch Time=14.0625 +GPU:2 | Epoch: 15 | loss=11.384038925170898 | Batch Time=7.8125 +GPU:2 | Epoch: 15 | loss=11.259687423706055 | Batch Time=9.765625 +GPU:2 | Epoch: 15 | loss=11.215600967407227 | Batch Time=15.234375 +GPU:2 | Epoch: 15 | loss=11.312837600708008 | Batch Time=13.671875 +GPU:2 | Epoch: 15 | loss=11.091010093688965 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.330471992492676 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.026508331298828 | Batch Time=15.234375 +GPU:2 | Epoch: 15 | loss=11.056368827819824 | Batch Time=14.0625 +GPU:2 | Epoch: 15 | loss=10.958856582641602 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.122944831848145 | Batch Time=14.84375 +GPU:2 | Epoch: 15 | loss=11.07425594329834 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.269386291503906 | Batch Time=9.375 +GPU:2 | Epoch: 15 | loss=11.100181579589844 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.015190124511719 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.207094192504883 | Batch Time=10.546875 +GPU:2 | Epoch: 15 | loss=11.093341827392578 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.00674819946289 | Batch Time=12.109375 +GPU:2 | Epoch: 15 | loss=11.01061725616455 | Batch Time=10.9375 +GPU:2 | Epoch: 15 | loss=11.050125122070312 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.252969741821289 | Batch Time=10.15625 +GPU:2 | Epoch: 15 | loss=11.08027458190918 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=10.9840087890625 | Batch Time=12.890625 +GPU:2 | Epoch: 15 | loss=11.113779067993164 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.18010139465332 | Batch Time=8.984375 +GPU:2 | Epoch: 15 | loss=11.309432983398438 | Batch Time=8.984375 +GPU:2 | Epoch: 15 | loss=11.059144973754883 | Batch Time=14.453125 +GPU:2 | Epoch: 15 | loss=11.165193557739258 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.298772811889648 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.142688751220703 | Batch Time=13.671875 +GPU:2 | Epoch: 15 | loss=11.356452941894531 | Batch Time=10.9375 +GPU:2 | Epoch: 15 | loss=11.248587608337402 | Batch Time=13.671875 +GPU:2 | Epoch: 15 | loss=10.826005935668945 | Batch Time=16.796875 +GPU:2 | Epoch: 15 | loss=11.109811782836914 | Batch Time=15.625 +GPU:2 | Epoch: 15 | loss=11.121252059936523 | Batch Time=12.890625 +GPU:2 | Epoch: 15 | loss=11.308222770690918 | Batch Time=9.375 +GPU:2 | Epoch: 15 | loss=10.785850524902344 | Batch Time=13.671875 +GPU:2 | Epoch: 15 | loss=10.980201721191406 | Batch Time=14.453125 +GPU:2 | Epoch: 15 | loss=11.018943786621094 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.041204452514648 | Batch Time=14.0625 +GPU:2 | Epoch: 15 | loss=11.325384140014648 | Batch Time=9.765625 +GPU:2 | Epoch: 15 | loss=11.156537055969238 | Batch Time=10.9375 +GPU:2 | Epoch: 15 | loss=11.02548599243164 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.244842529296875 | Batch Time=9.765625 +GPU:2 | Epoch: 15 | loss=11.227882385253906 | Batch Time=12.109375 +GPU:2 | Epoch: 15 | loss=11.040428161621094 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.019428253173828 | Batch Time=12.890625 +GPU:2 | Epoch: 15 | loss=11.218554496765137 | Batch Time=10.9375 +GPU:2 | Epoch: 15 | loss=10.913484573364258 | Batch Time=14.84375 +GPU:2 | Epoch: 15 | loss=10.963737487792969 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=10.961984634399414 | Batch Time=16.015625 +GPU:2 | Epoch: 15 | loss=11.091242790222168 | Batch Time=10.15625 +GPU:2 | Epoch: 15 | loss=11.211526870727539 | Batch Time=12.109375 +GPU:2 | Epoch: 15 | loss=11.384956359863281 | Batch Time=6.640625 +GPU:2 | Epoch: 15 | loss=11.20977783203125 | Batch Time=12.109375 +GPU:2 | Epoch: 15 | loss=10.984603881835938 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.024288177490234 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.008317947387695 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.188775062561035 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=10.984100341796875 | Batch Time=13.671875 +GPU:2 | Epoch: 15 | loss=11.009015083312988 | Batch Time=14.84375 +GPU:2 | Epoch: 15 | loss=11.132816314697266 | Batch Time=8.203125 +GPU:2 | Epoch: 15 | loss=10.827136993408203 | Batch Time=16.796875 +GPU:2 | Epoch: 15 | loss=10.99907112121582 | Batch Time=9.375 +GPU:2 | Epoch: 15 | loss=10.943440437316895 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=11.036754608154297 | Batch Time=13.28125 +GPU:2 | Epoch: 15 | loss=11.115355491638184 | Batch Time=8.984375 +GPU:2 | Epoch: 15 | loss=11.057809829711914 | Batch Time=14.84375 +GPU:2 | Epoch: 15 | loss=11.115708351135254 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.083477973937988 | Batch Time=14.453125 +GPU:2 | Epoch: 15 | loss=11.030563354492188 | Batch Time=14.0625 +GPU:2 | Epoch: 15 | loss=11.248733520507812 | Batch Time=10.15625 +GPU:2 | Epoch: 15 | loss=10.954429626464844 | Batch Time=12.890625 +GPU:2 | Epoch: 15 | loss=11.222223281860352 | Batch Time=10.546875 +GPU:2 | Epoch: 15 | loss=10.714376449584961 | Batch Time=14.84375 +GPU:2 | Epoch: 15 | loss=11.010013580322266 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=10.807267189025879 | Batch Time=14.0625 +GPU:2 | Epoch: 15 | loss=11.202031135559082 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.25529670715332 | Batch Time=8.984375 +GPU:2 | Epoch: 15 | loss=11.121604919433594 | Batch Time=10.546875 +GPU:2 | Epoch: 15 | loss=10.85236930847168 | Batch Time=15.625 +GPU:2 | Epoch: 15 | loss=11.009074211120605 | Batch Time=12.109375 +GPU:2 | Epoch: 15 | loss=11.140911102294922 | Batch Time=8.203125 +GPU:2 | Epoch: 15 | loss=11.105598449707031 | Batch Time=8.59375 +GPU:2 | Epoch: 15 | loss=11.292558670043945 | Batch Time=8.59375 +GPU:2 | Epoch: 15 | loss=10.87578296661377 | Batch Time=13.28125 +GPU:2 | Epoch: 15 | loss=10.960704803466797 | Batch Time=12.890625 +GPU:2 | Epoch: 15 | loss=11.115673065185547 | Batch Time=12.109375 +GPU:2 | Epoch: 15 | loss=10.886030197143555 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.031871795654297 | Batch Time=12.109375 +GPU:2 | Epoch: 15 | loss=11.081602096557617 | Batch Time=10.15625 +GPU:2 | Epoch: 15 | loss=11.051556587219238 | Batch Time=13.28125 +GPU:2 | Epoch: 15 | loss=11.141161918640137 | Batch Time=10.546875 +GPU:2 | Epoch: 15 | loss=10.995729446411133 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.143106460571289 | Batch Time=10.15625 +GPU:2 | Epoch: 15 | loss=11.17017936706543 | Batch Time=11.328125 +GPU:2 | Epoch: 15 | loss=10.939805030822754 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.252386093139648 | Batch Time=10.15625 +GPU:2 | Epoch: 15 | loss=11.113580703735352 | Batch Time=10.546875 +GPU:2 | Epoch: 15 | loss=11.112195014953613 | Batch Time=12.5 +GPU:2 | Epoch: 15 | loss=11.125022888183594 | Batch Time=11.71875 +GPU:2 | Epoch: 15 | loss=11.199463844299316 | Batch Time=13.671875 +GPU:2 | Epoch: 15 | loss=11.175300598144531 | Batch Time=8.203125 +GPU:2 | Epoch: 15 | loss=11.030187606811523 | Batch Time=16.40625 +GPU:2 | Epoch: 15 | loss=10.957959175109863 | Batch Time=16.015625 +GPU:2 | Epoch: 15 | loss=11.06951904296875 | Batch Time=10.15625 +GPU:3 | Epoch: 15 | loss=11.144065856933594 | Batch Time=10.15625 +GPU:3 | Epoch: 15 | loss=11.02261734008789 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=11.226982116699219 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=11.149032592773438 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.261648178100586 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=10.86393928527832 | Batch Time=16.796875 +GPU:3 | Epoch: 15 | loss=11.167466163635254 | Batch Time=13.671875 +GPU:3 | Epoch: 15 | loss=11.290494918823242 | Batch Time=9.765625 +GPU:3 | Epoch: 15 | loss=11.059724807739258 | Batch Time=13.28125 +GPU:3 | Epoch: 15 | loss=11.261333465576172 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=11.233211517333984 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=11.154350280761719 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.249780654907227 | Batch Time=10.15625 +GPU:3 | Epoch: 15 | loss=11.24884033203125 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=11.154462814331055 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=11.473623275756836 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=10.968450546264648 | Batch Time=15.234375 +GPU:3 | Epoch: 15 | loss=11.048711776733398 | Batch Time=17.1875 +GPU:3 | Epoch: 15 | loss=11.248001098632812 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=11.183080673217773 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=11.310497283935547 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.088010787963867 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.087697982788086 | Batch Time=12.5 +GPU:3 | Epoch: 15 | loss=11.011295318603516 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=10.958218574523926 | Batch Time=13.671875 +GPU:3 | Epoch: 15 | loss=11.268295288085938 | Batch Time=8.203125 +GPU:3 | Epoch: 15 | loss=11.01643180847168 | Batch Time=15.234375 +GPU:3 | Epoch: 15 | loss=11.11775016784668 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=10.965094566345215 | Batch Time=13.28125 +GPU:3 | Epoch: 15 | loss=11.123106956481934 | Batch Time=14.0625 +GPU:3 | Epoch: 15 | loss=11.416431427001953 | Batch Time=7.8125 +GPU:3 | Epoch: 15 | loss=11.068279266357422 | Batch Time=12.5 +GPU:3 | Epoch: 15 | loss=11.201095581054688 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.126086235046387 | Batch Time=10.15625 +GPU:3 | Epoch: 15 | loss=11.16427993774414 | Batch Time=14.0625 +GPU:3 | Epoch: 15 | loss=11.199771881103516 | Batch Time=13.28125 +GPU:3 | Epoch: 15 | loss=11.257983207702637 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=10.945135116577148 | Batch Time=13.28125 +GPU:3 | Epoch: 15 | loss=10.943849563598633 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.247957229614258 | Batch Time=13.28125 +GPU:3 | Epoch: 15 | loss=11.049825668334961 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.011964797973633 | Batch Time=14.453125 +GPU:3 | Epoch: 15 | loss=11.245821952819824 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.048908233642578 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.230963706970215 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=11.110204696655273 | Batch Time=10.15625 +GPU:3 | Epoch: 15 | loss=11.2694730758667 | Batch Time=8.59375 +GPU:3 | Epoch: 15 | loss=11.509759902954102 | Batch Time=10.546875 +GPU:3 | Epoch: 15 | loss=11.31004524230957 | Batch Time=10.546875 +GPU:3 | Epoch: 15 | loss=11.06940746307373 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.441886901855469 | Batch Time=10.546875 +GPU:3 | Epoch: 15 | loss=11.0437593460083 | Batch Time=9.765625 +GPU:3 | Epoch: 15 | loss=11.066147804260254 | Batch Time=14.0625 +GPU:3 | Epoch: 15 | loss=11.113993644714355 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.044378280639648 | Batch Time=15.234375 +GPU:3 | Epoch: 15 | loss=11.113988876342773 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=11.118276596069336 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=11.051383018493652 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.176563262939453 | Batch Time=12.5 +GPU:3 | Epoch: 15 | loss=10.9169340133667 | Batch Time=14.453125 +GPU:3 | Epoch: 15 | loss=11.064666748046875 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=11.268192291259766 | Batch Time=9.765625 +GPU:3 | Epoch: 15 | loss=11.234701156616211 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=11.039460182189941 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.075543403625488 | Batch Time=14.0625 +GPU:3 | Epoch: 15 | loss=11.112242698669434 | Batch Time=9.375 +GPU:3 | Epoch: 15 | loss=11.035449028015137 | Batch Time=13.28125 +GPU:3 | Epoch: 15 | loss=11.066577911376953 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=11.07016372680664 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=10.952837944030762 | Batch Time=15.234375 +GPU:3 | Epoch: 15 | loss=11.084543228149414 | Batch Time=14.0625 +GPU:3 | Epoch: 15 | loss=11.113424301147461 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.216985702514648 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=11.101187705993652 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.048254013061523 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=11.28298282623291 | Batch Time=9.375 +GPU:3 | Epoch: 15 | loss=11.114859580993652 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.064315795898438 | Batch Time=16.40625 +GPU:3 | Epoch: 15 | loss=11.085256576538086 | Batch Time=10.546875 +GPU:3 | Epoch: 15 | loss=11.075858116149902 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.099279403686523 | Batch Time=12.5 +GPU:3 | Epoch: 15 | loss=11.021073341369629 | Batch Time=13.28125 +GPU:3 | Epoch: 15 | loss=10.968356132507324 | Batch Time=12.5 +GPU:3 | Epoch: 15 | loss=11.132448196411133 | Batch Time=8.59375 +GPU:3 | Epoch: 15 | loss=11.393442153930664 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=10.929722785949707 | Batch Time=12.5 +GPU:3 | Epoch: 15 | loss=11.19237995147705 | Batch Time=10.15625 +GPU:3 | Epoch: 15 | loss=10.94611930847168 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.220328330993652 | Batch Time=10.546875 +GPU:3 | Epoch: 15 | loss=11.255112648010254 | Batch Time=9.765625 +GPU:3 | Epoch: 15 | loss=11.099512100219727 | Batch Time=13.28125 +GPU:3 | Epoch: 15 | loss=11.30622386932373 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=11.061332702636719 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.174570083618164 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.224658966064453 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=11.072694778442383 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=10.993650436401367 | Batch Time=13.671875 +GPU:3 | Epoch: 15 | loss=10.964462280273438 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.11681079864502 | Batch Time=10.546875 +GPU:3 | Epoch: 15 | loss=11.093867301940918 | Batch Time=10.15625 +GPU:3 | Epoch: 15 | loss=11.094192504882812 | Batch Time=9.765625 +GPU:3 | Epoch: 15 | loss=11.133261680603027 | Batch Time=11.71875 +GPU:3 | Epoch: 15 | loss=11.233229637145996 | Batch Time=8.59375 +GPU:3 | Epoch: 15 | loss=11.051433563232422 | Batch Time=13.671875 +GPU:3 | Epoch: 15 | loss=11.207805633544922 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=11.046154022216797 | Batch Time=15.234375 +GPU:3 | Epoch: 15 | loss=11.15781307220459 | Batch Time=10.546875 +GPU:3 | Epoch: 15 | loss=11.169500350952148 | Batch Time=13.671875 +GPU:3 | Epoch: 15 | loss=11.0057373046875 | Batch Time=13.671875 +GPU:3 | Epoch: 15 | loss=11.196573257446289 | Batch Time=9.375 +GPU:3 | Epoch: 15 | loss=11.194958686828613 | Batch Time=8.984375 +GPU:3 | Epoch: 15 | loss=10.959466934204102 | Batch Time=12.5 +GPU:3 | Epoch: 15 | loss=11.043737411499023 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=10.993127822875977 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=10.892805099487305 | Batch Time=14.84375 +GPU:3 | Epoch: 15 | loss=11.205830574035645 | Batch Time=11.328125 +GPU:3 | Epoch: 15 | loss=10.928067207336426 | Batch Time=12.5 +GPU:3 | Epoch: 15 | loss=10.991695404052734 | Batch Time=12.109375 +GPU:3 | Epoch: 15 | loss=11.039346694946289 | Batch Time=14.84375 +GPU:3 | Epoch: 15 | loss=11.074014663696289 | Batch Time=10.9375 +GPU:3 | Epoch: 15 | loss=11.106921195983887 | Batch Time=12.890625 +GPU:3 | Epoch: 15 | loss=10.886425971984863 | Batch Time=15.625 +GPU:3 | Epoch: 15 | loss=10.896682739257812 | Batch Time=14.453125 +GPU:3 | Epoch: 15 | loss=10.900184631347656 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=11.260354042053223 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=10.91581916809082 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=11.251258850097656 | Batch Time=10.546875 +GPU:0 | Epoch: 15 | loss=11.059881210327148 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=11.19527816772461 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.255941390991211 | Batch Time=8.984375 +GPU:0 | Epoch: 15 | loss=11.127862930297852 | Batch Time=16.015625 +GPU:0 | Epoch: 15 | loss=11.03510570526123 | Batch Time=16.40625 +GPU:0 | Epoch: 15 | loss=11.156828880310059 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.122528076171875 | Batch Time=10.546875 +GPU:0 | Epoch: 15 | loss=10.950611114501953 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.240364074707031 | Batch Time=16.40625 +GPU:0 | Epoch: 15 | loss=11.238415718078613 | Batch Time=10.546875 +GPU:0 | Epoch: 15 | loss=11.103891372680664 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=11.224203109741211 | Batch Time=9.765625 +GPU:0 | Epoch: 15 | loss=11.146839141845703 | Batch Time=8.984375 +GPU:0 | Epoch: 15 | loss=11.180912971496582 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=11.122774124145508 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=11.28936767578125 | Batch Time=11.71875 +GPU:0 | Epoch: 15 | loss=11.130899429321289 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=11.183769226074219 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=11.18492317199707 | Batch Time=10.15625 +GPU:0 | Epoch: 15 | loss=11.306685447692871 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=11.262299537658691 | Batch Time=12.5 +GPU:0 | Epoch: 15 | loss=11.307093620300293 | Batch Time=9.375 +GPU:0 | Epoch: 15 | loss=11.190275192260742 | Batch Time=11.71875 +GPU:0 | Epoch: 15 | loss=11.357612609863281 | Batch Time=8.59375 +GPU:0 | Epoch: 15 | loss=11.142236709594727 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=10.994734764099121 | Batch Time=14.84375 +GPU:0 | Epoch: 15 | loss=11.070348739624023 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=11.017732620239258 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=11.070234298706055 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=11.101234436035156 | Batch Time=11.71875 +GPU:0 | Epoch: 15 | loss=11.178136825561523 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.060068130493164 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.010049819946289 | Batch Time=14.453125 +GPU:0 | Epoch: 15 | loss=10.958813667297363 | Batch Time=15.234375 +GPU:0 | Epoch: 15 | loss=11.202381134033203 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=11.129695892333984 | Batch Time=14.0625 +GPU:0 | Epoch: 15 | loss=11.180335998535156 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.029440879821777 | Batch Time=9.375 +GPU:0 | Epoch: 15 | loss=11.1026611328125 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=11.284042358398438 | Batch Time=9.375 +GPU:0 | Epoch: 15 | loss=11.217792510986328 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.176360130310059 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=11.053398132324219 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=10.826894760131836 | Batch Time=11.71875 +GPU:0 | Epoch: 15 | loss=11.213119506835938 | Batch Time=9.765625 +GPU:0 | Epoch: 15 | loss=10.966026306152344 | Batch Time=14.453125 +GPU:0 | Epoch: 15 | loss=10.868827819824219 | Batch Time=15.625 +GPU:0 | Epoch: 15 | loss=11.082193374633789 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=11.116399765014648 | Batch Time=14.0625 +GPU:0 | Epoch: 15 | loss=11.12845230102539 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.105868339538574 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=11.004546165466309 | Batch Time=10.546875 +GPU:0 | Epoch: 15 | loss=11.157907485961914 | Batch Time=10.546875 +GPU:0 | Epoch: 15 | loss=11.215265274047852 | Batch Time=9.375 +GPU:0 | Epoch: 15 | loss=11.19580078125 | Batch Time=9.375 +GPU:0 | Epoch: 15 | loss=11.017780303955078 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=11.05950927734375 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=10.923263549804688 | Batch Time=14.453125 +GPU:0 | Epoch: 15 | loss=11.070125579833984 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.036822319030762 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.224847793579102 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.058270454406738 | Batch Time=14.453125 +GPU:0 | Epoch: 15 | loss=11.134750366210938 | Batch Time=10.15625 +GPU:0 | Epoch: 15 | loss=11.191583633422852 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.244058609008789 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.202225685119629 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=10.889632225036621 | Batch Time=12.5 +GPU:0 | Epoch: 15 | loss=11.01531982421875 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=11.070919036865234 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=11.230497360229492 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=11.248234748840332 | Batch Time=9.765625 +GPU:0 | Epoch: 15 | loss=11.033388137817383 | Batch Time=15.234375 +GPU:0 | Epoch: 15 | loss=11.140226364135742 | Batch Time=9.765625 +GPU:0 | Epoch: 15 | loss=11.043234825134277 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=11.15934944152832 | Batch Time=8.59375 +GPU:0 | Epoch: 15 | loss=11.247186660766602 | Batch Time=10.546875 +GPU:0 | Epoch: 15 | loss=11.23469352722168 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=11.316375732421875 | Batch Time=8.984375 +GPU:0 | Epoch: 15 | loss=11.226649284362793 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.003475189208984 | Batch Time=11.71875 +GPU:0 | Epoch: 15 | loss=11.191469192504883 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.046306610107422 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.058130264282227 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=10.951733589172363 | Batch Time=12.5 +GPU:0 | Epoch: 15 | loss=11.285066604614258 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.227633476257324 | Batch Time=10.15625 +GPU:0 | Epoch: 15 | loss=11.006731033325195 | Batch Time=14.84375 +GPU:0 | Epoch: 15 | loss=10.757184982299805 | Batch Time=16.015625 +GPU:0 | Epoch: 15 | loss=11.114847183227539 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=11.002313613891602 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=11.058891296386719 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=11.054384231567383 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=11.011798858642578 | Batch Time=14.453125 +GPU:0 | Epoch: 15 | loss=11.123689651489258 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=10.97658920288086 | Batch Time=16.40625 +GPU:0 | Epoch: 15 | loss=11.120312690734863 | Batch Time=15.234375 +GPU:0 | Epoch: 15 | loss=11.104496002197266 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=10.989952087402344 | Batch Time=14.453125 +GPU:0 | Epoch: 15 | loss=11.138555526733398 | Batch Time=9.375 +GPU:0 | Epoch: 15 | loss=10.935961723327637 | Batch Time=14.0625 +GPU:0 | Epoch: 15 | loss=10.929647445678711 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.013603210449219 | Batch Time=11.71875 +GPU:0 | Epoch: 15 | loss=11.225502014160156 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=10.973791122436523 | Batch Time=14.84375 +GPU:0 | Epoch: 15 | loss=11.005481719970703 | Batch Time=13.671875 +GPU:0 | Epoch: 15 | loss=11.166053771972656 | Batch Time=9.375 +GPU:0 | Epoch: 15 | loss=10.941484451293945 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=11.022132873535156 | Batch Time=9.765625 +GPU:0 | Epoch: 15 | loss=11.105414390563965 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=11.193070411682129 | Batch Time=10.546875 +GPU:0 | Epoch: 15 | loss=11.109153747558594 | Batch Time=10.9375 +GPU:0 | Epoch: 15 | loss=11.066781997680664 | Batch Time=15.234375 +GPU:0 | Epoch: 15 | loss=10.937664031982422 | Batch Time=15.234375 +GPU:0 | Epoch: 15 | loss=10.9627103805542 | Batch Time=12.890625 +GPU:0 | Epoch: 15 | loss=11.187301635742188 | Batch Time=8.984375 +GPU:0 | Epoch: 15 | loss=10.895456314086914 | Batch Time=13.28125 +GPU:0 | Epoch: 15 | loss=10.987600326538086 | Batch Time=14.0625 +GPU:0 | Epoch: 15 | loss=11.167821884155273 | Batch Time=9.375 +GPU:0 | Epoch: 15 | loss=10.952290534973145 | Batch Time=12.109375 +GPU:0 | Epoch: 15 | loss=10.910810470581055 | Batch Time=10.546875 +GPU:0 | Epoch: 15 | loss=10.93470287322998 | Batch Time=14.84375 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 15 | loss=11.230133056640625 | Batch Time=8.59375 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +AFTER TRAIN LOOP: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +BEFORE VAL LOOP: GPU: 0 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +GPU:1 | Epoch: 15 | loss=11.010747909545898 | Batch Time=13.671875 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +AFTER TRAIN LOOP: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +BEFORE PRUNE: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +Pruning Model: +AFTER PRUNE: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +STARTING TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +BEFORE TRAIN LOOP: Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +GPU:3 | Epoch: 15 | loss=11.064921379089355 | Batch Time=11.71875 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +AFTER TRAIN LOOP: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +BEFORE PRUNE: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +Pruning Model: +AFTER PRUNE: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +STARTING TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +BEFORE TRAIN LOOP: Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +GPU:2 | Epoch: 15 | loss=10.801154136657715 | Batch Time=16.796875 +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=117940568064, percent=54.2, used=131714723840, free=17033093120, active=185010282496, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219959808) +AFTER TRAIN LOOP: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +BEFORE PRUNE: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714191360, free=17033625600, active=185010208768, inactive=50282672128, buffers=404230144, cached=108416036864, shared=5661425664, slab=3219951616) +Pruning Model: +AFTER PRUNE: Epoch 15 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +STARTING TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +BEFORE TRAIN LOOP: Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=117941100544, percent=54.2, used=131714453504, free=17032593408, active=185010196480, inactive=50283704320, buffers=404230144, cached=108416806912, shared=5661425664, slab=3219951616) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 15 | loss=4.1388936042785645 | Batch Time=23.046875 +GPU:0 | Epoch: 15 | loss=5.248618125915527 | Batch Time=7.03125 +GPU:0 | Epoch: 15 | loss=4.815436363220215 | Batch Time=22.65625 +GPU:0 | Epoch: 15 | loss=4.564565181732178 | Batch Time=8.59375 +GPU:0 | Epoch: 15 | loss=5.433211803436279 | Batch Time=3.90625 +GPU:0 | Epoch: 15 | loss=4.533936500549316 | Batch Time=10.15625 +GPU:0 | Epoch: 15 | loss=5.638082027435303 | Batch Time=3.515625 +GPU:0 | Epoch: 15 | loss=5.59310245513916 | Batch Time=3.515625 +GPU:0 | Epoch: 15 | loss=5.285001754760742 | Batch Time=6.25 +GPU:0 | Epoch: 15 | loss=5.850387096405029 | Batch Time=1.953125 +GPU:0 | Epoch: 15 | loss=5.514808654785156 | Batch Time=0.390625 +GPU:0 | Epoch: 15 | loss=4.744860649108887 | Batch Time=16.015625 +GPU:0 | Epoch: 15 | loss=5.417393684387207 | Batch Time=6.25 +GPU:0 | Epoch: 15 | loss=4.859984874725342 | Batch Time=12.5 +GPU:0 | Epoch: 15 | loss=5.133882522583008 | Batch Time=4.6875 +GPU:0 | Epoch: 15 | loss=5.27291202545166 | Batch Time=9.765625 +GPU:0 | Epoch: 15 | loss=4.891410827636719 | Batch Time=8.203125 +GPU:0 | Epoch: 15 | loss=4.08057165145874 | Batch Time=23.828125 +GPU:0 | Epoch: 15 | loss=4.885376930236816 | Batch Time=14.84375 +GPU:0 | Epoch: 15 | loss=4.168951511383057 | Batch Time=22.65625 +Model top1 Accuracy: 9.918 +Acc before rounding: 9.918 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 15 | loss=4.1388936042785645 | Batch Time=23.046875 +GPU:0 | Epoch: 15 | loss=5.248618125915527 | Batch Time=7.03125 +GPU:0 | Epoch: 15 | loss=4.815436363220215 | Batch Time=22.65625 +GPU:0 | Epoch: 15 | loss=4.564565181732178 | Batch Time=8.59375 +GPU:0 | Epoch: 15 | loss=5.433211803436279 | Batch Time=3.90625 +GPU:0 | Epoch: 15 | loss=4.533936500549316 | Batch Time=10.15625 +GPU:0 | Epoch: 15 | loss=5.638082027435303 | Batch Time=3.515625 +GPU:0 | Epoch: 15 | loss=5.59310245513916 | Batch Time=3.515625 +GPU:0 | Epoch: 15 | loss=5.285001754760742 | Batch Time=6.25 +GPU:0 | Epoch: 15 | loss=5.850387096405029 | Batch Time=1.953125 +GPU:0 | Epoch: 15 | loss=5.514808654785156 | Batch Time=0.390625 +GPU:0 | Epoch: 15 | loss=4.744860649108887 | Batch Time=16.015625 +GPU:0 | Epoch: 15 | loss=5.417393684387207 | Batch Time=6.25 +GPU:0 | Epoch: 15 | loss=4.859984874725342 | Batch Time=12.5 +GPU:0 | Epoch: 15 | loss=5.133882522583008 | Batch Time=4.6875 +GPU:0 | Epoch: 15 | loss=5.27291202545166 | Batch Time=9.765625 +GPU:0 | Epoch: 15 | loss=4.891410827636719 | Batch Time=8.203125 +GPU:0 | Epoch: 15 | loss=4.08057165145874 | Batch Time=23.828125 +GPU:0 | Epoch: 15 | loss=4.885376930236816 | Batch Time=14.84375 +GPU:0 | Epoch: 15 | loss=4.168951511383057 | Batch Time=22.65625 +Model top1 Accuracy: 9.918 +Acc after rounding: 9.918 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 15 | loss=5.272021770477295 | Batch Time=11.328125 +GPU:0 | Epoch: 15 | loss=5.278052806854248 | Batch Time=9.765625 +GPU:0 | Epoch: 15 | loss=5.108400821685791 | Batch Time=10.15625 +GPU:0 | Epoch: 15 | loss=5.277854919433594 | Batch Time=8.203125 +Model top1 Accuracy: 8.11 +Validation Acc after rounding: 8.11 +AFTER VAL LOOP: GPU: 0 | Epoch 15 | Memory Usage: svmem(total=257568083968, available=100152057856, percent=61.1, used=149502955520, free=5362253824, active=204532068352, inactive=42325102592, buffers=405078016, cached=102297796608, shared=5661573120, slab=3218710528) +BEFORE PRUNE: Epoch 15 | Memory Usage: svmem(total=257568083968, available=100152057856, percent=61.1, used=149502955520, free=5362253824, active=204532068352, inactive=42325102592, buffers=405078016, cached=102297796608, shared=5661573120, slab=3218710528) +Pruning Model: +AFTER PRUNE: Epoch 15 | Memory Usage: svmem(total=257568083968, available=100152057856, percent=61.1, used=149502955520, free=5362253824, active=204532068352, inactive=42325102592, buffers=405078016, cached=102297796608, shared=5661573120, slab=3218710528) +Rounding model with scheme: naive +Model avg sparsity: 41.51028133348723 +GPU:0 | Epoch: 15 | Acc=9.918 | Epoch Time=17.392623364925385 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=100152250368, percent=61.1, used=149502791680, free=5362253824, active=204532191232, inactive=42325082112, buffers=405315584, cached=102297722880, shared=5661491200, slab=3218710528) +STARTING TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=100152250368, percent=61.1, used=149502791680, free=5362253824, active=204532191232, inactive=42325082112, buffers=405315584, cached=102297722880, shared=5661491200, slab=3218710528) +BEFORE TRAIN LOOP: Epoch 16 | Memory Usage: svmem(total=257568083968, available=100152250368, percent=61.1, used=149502791680, free=5362253824, active=204532191232, inactive=42325082112, buffers=405315584, cached=102297722880, shared=5661491200, slab=3218710528) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=100152250368, percent=61.1, used=149502791680, free=5362253824, active=204532191232, inactive=42325082112, buffers=405315584, cached=102297722880, shared=5661491200, slab=3218710528) +GPU:2 | Epoch: 16 | loss=10.809133529663086 | Batch Time=16.796875 +GPU:2 | Epoch: 16 | loss=10.847816467285156 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=10.839188575744629 | Batch Time=12.890625 +GPU:2 | Epoch: 16 | loss=11.083984375 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=10.940139770507812 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=11.030373573303223 | Batch Time=10.9375 +GPU:2 | Epoch: 16 | loss=11.151387214660645 | Batch Time=9.375 +GPU:2 | Epoch: 16 | loss=10.792414665222168 | Batch Time=18.359375 +GPU:2 | Epoch: 16 | loss=11.102745056152344 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=10.967658996582031 | Batch Time=12.890625 +GPU:2 | Epoch: 16 | loss=11.095893859863281 | Batch Time=9.765625 +GPU:2 | Epoch: 16 | loss=10.772573471069336 | Batch Time=18.359375 +GPU:2 | Epoch: 16 | loss=11.199546813964844 | Batch Time=7.03125 +GPU:2 | Epoch: 16 | loss=11.223288536071777 | Batch Time=8.203125 +GPU:2 | Epoch: 16 | loss=11.02058219909668 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=11.010183334350586 | Batch Time=11.71875 +GPU:2 | Epoch: 16 | loss=10.858721733093262 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.968528747558594 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.867362022399902 | Batch Time=15.625 +GPU:2 | Epoch: 16 | loss=10.95229721069336 | Batch Time=9.765625 +GPU:2 | Epoch: 16 | loss=11.069201469421387 | Batch Time=10.15625 +GPU:2 | Epoch: 16 | loss=11.107182502746582 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.985076904296875 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.83953857421875 | Batch Time=13.28125 +GPU:2 | Epoch: 16 | loss=11.018302917480469 | Batch Time=11.71875 +GPU:2 | Epoch: 16 | loss=10.989408493041992 | Batch Time=15.625 +GPU:2 | Epoch: 16 | loss=10.953536033630371 | Batch Time=10.9375 +GPU:2 | Epoch: 16 | loss=11.15838623046875 | Batch Time=12.890625 +GPU:2 | Epoch: 16 | loss=10.851166725158691 | Batch Time=15.234375 +GPU:2 | Epoch: 16 | loss=11.016897201538086 | Batch Time=10.546875 +GPU:2 | Epoch: 16 | loss=11.095874786376953 | Batch Time=13.28125 +GPU:2 | Epoch: 16 | loss=10.999944686889648 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=10.739871978759766 | Batch Time=14.453125 +GPU:2 | Epoch: 16 | loss=10.806018829345703 | Batch Time=16.015625 +GPU:2 | Epoch: 16 | loss=11.040753364562988 | Batch Time=11.71875 +GPU:2 | Epoch: 16 | loss=11.013040542602539 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=10.981964111328125 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.899643898010254 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.855022430419922 | Batch Time=13.28125 +GPU:2 | Epoch: 16 | loss=11.048933029174805 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.997441291809082 | Batch Time=10.15625 +GPU:2 | Epoch: 16 | loss=11.03831672668457 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.70417594909668 | Batch Time=16.015625 +GPU:2 | Epoch: 16 | loss=10.865631103515625 | Batch Time=15.234375 +GPU:2 | Epoch: 16 | loss=10.997419357299805 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.923593521118164 | Batch Time=10.15625 +GPU:2 | Epoch: 16 | loss=10.93483829498291 | Batch Time=10.546875 +GPU:2 | Epoch: 16 | loss=10.987873077392578 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.874076843261719 | Batch Time=10.9375 +GPU:2 | Epoch: 16 | loss=11.056266784667969 | Batch Time=10.9375 +GPU:2 | Epoch: 16 | loss=10.847845077514648 | Batch Time=16.015625 +GPU:2 | Epoch: 16 | loss=10.723258018493652 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.804147720336914 | Batch Time=15.625 +GPU:2 | Epoch: 16 | loss=10.850729942321777 | Batch Time=15.234375 +GPU:2 | Epoch: 16 | loss=10.847488403320312 | Batch Time=14.453125 +GPU:2 | Epoch: 16 | loss=11.170299530029297 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=11.054759979248047 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.872039794921875 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.981901168823242 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=11.061355590820312 | Batch Time=9.375 +GPU:2 | Epoch: 16 | loss=11.219593048095703 | Batch Time=8.984375 +GPU:2 | Epoch: 16 | loss=10.993719100952148 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=11.043487548828125 | Batch Time=12.890625 +GPU:2 | Epoch: 16 | loss=10.95933723449707 | Batch Time=10.9375 +GPU:2 | Epoch: 16 | loss=10.857576370239258 | Batch Time=14.84375 +GPU:2 | Epoch: 16 | loss=10.96605110168457 | Batch Time=11.71875 +GPU:2 | Epoch: 16 | loss=10.84205436706543 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=11.071405410766602 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.81641674041748 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.817996978759766 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=11.026636123657227 | Batch Time=11.71875 +GPU:2 | Epoch: 16 | loss=10.971900939941406 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=10.823341369628906 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=10.798578262329102 | Batch Time=14.84375 +GPU:2 | Epoch: 16 | loss=10.845114707946777 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.953771591186523 | Batch Time=10.15625 +GPU:2 | Epoch: 16 | loss=10.689215660095215 | Batch Time=16.40625 +GPU:2 | Epoch: 16 | loss=11.063629150390625 | Batch Time=9.765625 +GPU:2 | Epoch: 16 | loss=10.993492126464844 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=10.840963363647461 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.901777267456055 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.794767379760742 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=11.03188705444336 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=10.967788696289062 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.854263305664062 | Batch Time=14.84375 +GPU:2 | Epoch: 16 | loss=10.766674041748047 | Batch Time=16.796875 +GPU:2 | Epoch: 16 | loss=11.031332015991211 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=11.013912200927734 | Batch Time=11.71875 +GPU:2 | Epoch: 16 | loss=10.745200157165527 | Batch Time=13.28125 +GPU:2 | Epoch: 16 | loss=10.92419147491455 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=10.87067985534668 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=11.011312484741211 | Batch Time=9.375 +GPU:2 | Epoch: 16 | loss=11.038248062133789 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.73589038848877 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.881576538085938 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=10.840063095092773 | Batch Time=11.71875 +GPU:2 | Epoch: 16 | loss=10.91229248046875 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=10.802595138549805 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.93618392944336 | Batch Time=12.890625 +GPU:2 | Epoch: 16 | loss=11.122076034545898 | Batch Time=8.59375 +GPU:2 | Epoch: 16 | loss=10.893777847290039 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.748897552490234 | Batch Time=16.015625 +GPU:2 | Epoch: 16 | loss=10.600847244262695 | Batch Time=17.96875 +GPU:2 | Epoch: 16 | loss=10.624120712280273 | Batch Time=16.015625 +GPU:2 | Epoch: 16 | loss=10.82693099975586 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.935507774353027 | Batch Time=11.328125 +GPU:2 | Epoch: 16 | loss=10.998189926147461 | Batch Time=8.984375 +GPU:2 | Epoch: 16 | loss=10.945052146911621 | Batch Time=10.546875 +GPU:2 | Epoch: 16 | loss=11.06375503540039 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.793042182922363 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.949892044067383 | Batch Time=12.890625 +GPU:2 | Epoch: 16 | loss=10.80914306640625 | Batch Time=12.890625 +GPU:2 | Epoch: 16 | loss=10.737507820129395 | Batch Time=13.671875 +GPU:2 | Epoch: 16 | loss=10.674235343933105 | Batch Time=15.234375 +GPU:2 | Epoch: 16 | loss=10.821990966796875 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.916229248046875 | Batch Time=15.234375 +GPU:2 | Epoch: 16 | loss=10.827146530151367 | Batch Time=14.453125 +GPU:2 | Epoch: 16 | loss=10.926933288574219 | Batch Time=15.234375 +GPU:2 | Epoch: 16 | loss=10.916004180908203 | Batch Time=12.890625 +GPU:2 | Epoch: 16 | loss=10.716960906982422 | Batch Time=16.40625 +GPU:2 | Epoch: 16 | loss=10.879127502441406 | Batch Time=14.0625 +GPU:2 | Epoch: 16 | loss=10.995553970336914 | Batch Time=12.109375 +GPU:2 | Epoch: 16 | loss=10.813152313232422 | Batch Time=12.5 +GPU:2 | Epoch: 16 | loss=10.815023422241211 | Batch Time=13.671875 +GPU:3 | Epoch: 16 | loss=10.954080581665039 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=10.916841506958008 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=11.061744689941406 | Batch Time=11.328125 +GPU:3 | Epoch: 16 | loss=11.037166595458984 | Batch Time=10.9375 +GPU:3 | Epoch: 16 | loss=10.765491485595703 | Batch Time=13.671875 +GPU:3 | Epoch: 16 | loss=10.937079429626465 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.882018089294434 | Batch Time=11.71875 +GPU:3 | Epoch: 16 | loss=11.063007354736328 | Batch Time=14.84375 +GPU:3 | Epoch: 16 | loss=10.996105194091797 | Batch Time=14.84375 +GPU:3 | Epoch: 16 | loss=11.319576263427734 | Batch Time=8.59375 +GPU:3 | Epoch: 16 | loss=10.896598815917969 | Batch Time=14.84375 +GPU:3 | Epoch: 16 | loss=11.0750093460083 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.884305953979492 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=10.74802017211914 | Batch Time=16.40625 +GPU:3 | Epoch: 16 | loss=11.017524719238281 | Batch Time=11.71875 +GPU:3 | Epoch: 16 | loss=10.934946060180664 | Batch Time=10.15625 +GPU:3 | Epoch: 16 | loss=11.004018783569336 | Batch Time=12.5 +GPU:3 | Epoch: 16 | loss=11.124507904052734 | Batch Time=10.15625 +GPU:3 | Epoch: 16 | loss=11.063712120056152 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.81770133972168 | Batch Time=15.625 +GPU:3 | Epoch: 16 | loss=10.87468147277832 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=10.998682022094727 | Batch Time=11.328125 +GPU:3 | Epoch: 16 | loss=11.156176567077637 | Batch Time=12.890625 +GPU:3 | Epoch: 16 | loss=10.922920227050781 | Batch Time=15.234375 +GPU:3 | Epoch: 16 | loss=10.79489517211914 | Batch Time=13.671875 +GPU:3 | Epoch: 16 | loss=10.909074783325195 | Batch Time=15.234375 +GPU:3 | Epoch: 16 | loss=11.178960800170898 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.880243301391602 | Batch Time=13.671875 +GPU:3 | Epoch: 16 | loss=10.966348648071289 | Batch Time=11.71875 +GPU:3 | Epoch: 16 | loss=11.145580291748047 | Batch Time=10.15625 +GPU:3 | Epoch: 16 | loss=10.822421073913574 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=11.004077911376953 | Batch Time=10.15625 +GPU:3 | Epoch: 16 | loss=10.972101211547852 | Batch Time=12.890625 +GPU:3 | Epoch: 16 | loss=11.018899917602539 | Batch Time=16.796875 +GPU:3 | Epoch: 16 | loss=11.109663009643555 | Batch Time=7.03125 +GPU:3 | Epoch: 16 | loss=10.994091033935547 | Batch Time=13.671875 +GPU:3 | Epoch: 16 | loss=10.94209098815918 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=10.930733680725098 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=10.859975814819336 | Batch Time=16.015625 +GPU:3 | Epoch: 16 | loss=10.880578994750977 | Batch Time=12.5 +GPU:3 | Epoch: 16 | loss=10.984321594238281 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.921823501586914 | Batch Time=11.71875 +GPU:3 | Epoch: 16 | loss=10.818373680114746 | Batch Time=11.71875 +GPU:3 | Epoch: 16 | loss=10.998043060302734 | Batch Time=11.328125 +GPU:3 | Epoch: 16 | loss=11.005440711975098 | Batch Time=15.625 +GPU:3 | Epoch: 16 | loss=10.847223281860352 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.798202514648438 | Batch Time=11.71875 +GPU:3 | Epoch: 16 | loss=10.804473876953125 | Batch Time=12.890625 +GPU:3 | Epoch: 16 | loss=10.84075927734375 | Batch Time=15.234375 +GPU:3 | Epoch: 16 | loss=10.936504364013672 | Batch Time=12.5 +GPU:3 | Epoch: 16 | loss=11.102487564086914 | Batch Time=9.375 +GPU:3 | Epoch: 16 | loss=10.889688491821289 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=11.103965759277344 | Batch Time=11.71875 +GPU:3 | Epoch: 16 | loss=10.999680519104004 | Batch Time=10.546875 +GPU:3 | Epoch: 16 | loss=11.027715682983398 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=11.026264190673828 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=10.986322402954102 | Batch Time=9.375 +GPU:3 | Epoch: 16 | loss=10.76736068725586 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=10.855852127075195 | Batch Time=11.328125 +GPU:3 | Epoch: 16 | loss=10.885377883911133 | Batch Time=15.625 +GPU:3 | Epoch: 16 | loss=10.974811553955078 | Batch Time=10.546875 +GPU:3 | Epoch: 16 | loss=10.971928596496582 | Batch Time=10.546875 +GPU:3 | Epoch: 16 | loss=10.888969421386719 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=10.856277465820312 | Batch Time=15.625 +GPU:3 | Epoch: 16 | loss=10.917987823486328 | Batch Time=10.9375 +GPU:3 | Epoch: 16 | loss=11.051873207092285 | Batch Time=10.9375 +GPU:3 | Epoch: 16 | loss=10.895759582519531 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.870722770690918 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.969246864318848 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=10.88752555847168 | Batch Time=11.328125 +GPU:3 | Epoch: 16 | loss=11.202857971191406 | Batch Time=9.765625 +GPU:3 | Epoch: 16 | loss=10.758432388305664 | Batch Time=14.84375 +GPU:3 | Epoch: 16 | loss=10.811029434204102 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=10.918987274169922 | Batch Time=12.890625 +GPU:3 | Epoch: 16 | loss=10.828764915466309 | Batch Time=15.625 +GPU:3 | Epoch: 16 | loss=10.87761116027832 | Batch Time=11.71875 +GPU:3 | Epoch: 16 | loss=10.912346839904785 | Batch Time=15.625 +GPU:3 | Epoch: 16 | loss=10.824214935302734 | Batch Time=12.890625 +GPU:3 | Epoch: 16 | loss=10.792513847351074 | Batch Time=15.234375 +GPU:3 | Epoch: 16 | loss=10.741470336914062 | Batch Time=17.1875 +GPU:3 | Epoch: 16 | loss=10.942407608032227 | Batch Time=10.9375 +GPU:3 | Epoch: 16 | loss=10.92668342590332 | Batch Time=9.375 +GPU:3 | Epoch: 16 | loss=10.91342830657959 | Batch Time=12.890625 +GPU:3 | Epoch: 16 | loss=10.817123413085938 | Batch Time=13.671875 +GPU:3 | Epoch: 16 | loss=11.018875122070312 | Batch Time=11.328125 +GPU:3 | Epoch: 16 | loss=10.960886001586914 | Batch Time=13.671875 +GPU:3 | Epoch: 16 | loss=10.81965446472168 | Batch Time=14.0625 +GPU:3 | Epoch: 16 | loss=10.883687019348145 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=10.859603881835938 | Batch Time=12.5 +GPU:3 | Epoch: 16 | loss=10.911117553710938 | Batch Time=15.625 +GPU:3 | Epoch: 16 | loss=10.93988037109375 | Batch Time=12.5 +GPU:3 | Epoch: 16 | loss=10.987197875976562 | Batch Time=9.765625 +GPU:3 | Epoch: 16 | loss=10.999401092529297 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=11.15052604675293 | Batch Time=10.546875 +GPU:3 | Epoch: 16 | loss=11.010150909423828 | Batch Time=8.984375 +GPU:3 | Epoch: 16 | loss=10.856959342956543 | Batch Time=11.328125 +GPU:3 | Epoch: 16 | loss=10.75154972076416 | Batch Time=13.671875 +GPU:3 | Epoch: 16 | loss=10.808070182800293 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.999937057495117 | Batch Time=10.15625 +GPU:3 | Epoch: 16 | loss=10.656436920166016 | Batch Time=17.578125 +GPU:3 | Epoch: 16 | loss=10.873506546020508 | Batch Time=9.765625 +GPU:3 | Epoch: 16 | loss=10.792058944702148 | Batch Time=14.0625 +GPU:3 | Epoch: 16 | loss=10.849364280700684 | Batch Time=10.546875 +GPU:3 | Epoch: 16 | loss=10.88199234008789 | Batch Time=15.625 +GPU:3 | Epoch: 16 | loss=10.909228324890137 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.85445785522461 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=11.105833053588867 | Batch Time=10.15625 +GPU:3 | Epoch: 16 | loss=10.917620658874512 | Batch Time=11.328125 +GPU:3 | Epoch: 16 | loss=10.79771614074707 | Batch Time=14.453125 +GPU:3 | Epoch: 16 | loss=10.893280029296875 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=10.77756118774414 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=10.990373611450195 | Batch Time=12.5 +GPU:3 | Epoch: 16 | loss=10.678299903869629 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=10.793733596801758 | Batch Time=15.234375 +GPU:3 | Epoch: 16 | loss=10.931211471557617 | Batch Time=10.546875 +GPU:3 | Epoch: 16 | loss=10.929487228393555 | Batch Time=12.890625 +GPU:3 | Epoch: 16 | loss=10.892292976379395 | Batch Time=12.109375 +GPU:3 | Epoch: 16 | loss=10.843148231506348 | Batch Time=10.9375 +GPU:3 | Epoch: 16 | loss=10.939401626586914 | Batch Time=12.890625 +GPU:3 | Epoch: 16 | loss=10.602885246276855 | Batch Time=13.28125 +GPU:3 | Epoch: 16 | loss=10.669727325439453 | Batch Time=12.5 +GPU:3 | Epoch: 16 | loss=10.966485977172852 | Batch Time=10.9375 +GPU:3 | Epoch: 16 | loss=10.697946548461914 | Batch Time=14.0625 +GPU:3 | Epoch: 16 | loss=10.823993682861328 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=11.110546112060547 | Batch Time=9.765625 +GPU:1 | Epoch: 16 | loss=10.945219993591309 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.811308860778809 | Batch Time=14.84375 +GPU:1 | Epoch: 16 | loss=11.077863693237305 | Batch Time=11.71875 +GPU:1 | Epoch: 16 | loss=10.968708038330078 | Batch Time=16.015625 +GPU:1 | Epoch: 16 | loss=10.869998931884766 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=10.805252075195312 | Batch Time=15.234375 +GPU:1 | Epoch: 16 | loss=11.00532341003418 | Batch Time=14.453125 +GPU:1 | Epoch: 16 | loss=10.782350540161133 | Batch Time=12.109375 +GPU:1 | Epoch: 16 | loss=11.023717880249023 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.917207717895508 | Batch Time=12.109375 +GPU:1 | Epoch: 16 | loss=11.065641403198242 | Batch Time=15.234375 +GPU:1 | Epoch: 16 | loss=10.914144515991211 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=11.144497871398926 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=11.024112701416016 | Batch Time=10.546875 +GPU:1 | Epoch: 16 | loss=10.86638069152832 | Batch Time=13.671875 +GPU:1 | Epoch: 16 | loss=11.295182228088379 | Batch Time=7.421875 +GPU:1 | Epoch: 16 | loss=11.107534408569336 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.852527618408203 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.960540771484375 | Batch Time=13.671875 +GPU:1 | Epoch: 16 | loss=10.978191375732422 | Batch Time=11.71875 +GPU:1 | Epoch: 16 | loss=10.824000358581543 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.948476791381836 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.952754974365234 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.999117851257324 | Batch Time=9.375 +GPU:1 | Epoch: 16 | loss=10.995476722717285 | Batch Time=8.984375 +GPU:1 | Epoch: 16 | loss=10.953471183776855 | Batch Time=12.109375 +GPU:1 | Epoch: 16 | loss=10.781140327453613 | Batch Time=12.890625 +GPU:1 | Epoch: 16 | loss=10.973655700683594 | Batch Time=10.546875 +GPU:1 | Epoch: 16 | loss=11.041126251220703 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=11.052129745483398 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=11.098226547241211 | Batch Time=9.765625 +GPU:1 | Epoch: 16 | loss=10.897016525268555 | Batch Time=15.234375 +GPU:1 | Epoch: 16 | loss=10.979963302612305 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=11.167379379272461 | Batch Time=14.453125 +GPU:1 | Epoch: 16 | loss=11.118993759155273 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.920516014099121 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=10.785255432128906 | Batch Time=17.578125 +GPU:1 | Epoch: 16 | loss=11.022871017456055 | Batch Time=10.15625 +GPU:1 | Epoch: 16 | loss=10.815374374389648 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=11.06375503540039 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=11.061059951782227 | Batch Time=9.765625 +GPU:1 | Epoch: 16 | loss=10.893503189086914 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.948240280151367 | Batch Time=14.453125 +GPU:1 | Epoch: 16 | loss=10.875151634216309 | Batch Time=12.890625 +GPU:1 | Epoch: 16 | loss=11.03006362915039 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.66196060180664 | Batch Time=18.75 +GPU:1 | Epoch: 16 | loss=10.92039680480957 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.762869834899902 | Batch Time=13.671875 +GPU:1 | Epoch: 16 | loss=10.674932479858398 | Batch Time=15.625 +GPU:1 | Epoch: 16 | loss=10.973363876342773 | Batch Time=10.546875 +GPU:1 | Epoch: 16 | loss=11.011154174804688 | Batch Time=14.84375 +GPU:1 | Epoch: 16 | loss=10.93526840209961 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.636520385742188 | Batch Time=18.75 +GPU:1 | Epoch: 16 | loss=10.91341781616211 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.922904968261719 | Batch Time=13.671875 +GPU:1 | Epoch: 16 | loss=10.892064094543457 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=10.86544418334961 | Batch Time=11.71875 +GPU:1 | Epoch: 16 | loss=11.106535911560059 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.888093948364258 | Batch Time=12.109375 +GPU:1 | Epoch: 16 | loss=11.17181396484375 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.710714340209961 | Batch Time=12.890625 +GPU:1 | Epoch: 16 | loss=10.885433197021484 | Batch Time=15.234375 +GPU:1 | Epoch: 16 | loss=10.854321479797363 | Batch Time=13.671875 +GPU:1 | Epoch: 16 | loss=10.84669303894043 | Batch Time=14.453125 +GPU:1 | Epoch: 16 | loss=10.859941482543945 | Batch Time=13.671875 +GPU:1 | Epoch: 16 | loss=10.862499237060547 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.753459930419922 | Batch Time=15.234375 +GPU:1 | Epoch: 16 | loss=10.772817611694336 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.773408889770508 | Batch Time=15.625 +GPU:1 | Epoch: 16 | loss=10.811285018920898 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=10.869524002075195 | Batch Time=14.84375 +GPU:1 | Epoch: 16 | loss=10.916908264160156 | Batch Time=14.453125 +GPU:1 | Epoch: 16 | loss=11.079605102539062 | Batch Time=9.765625 +GPU:1 | Epoch: 16 | loss=11.050243377685547 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.919394493103027 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=10.856328964233398 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.585798263549805 | Batch Time=19.140625 +GPU:1 | Epoch: 16 | loss=10.871111869812012 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=10.857083320617676 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=10.93792724609375 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.828547477722168 | Batch Time=12.890625 +GPU:1 | Epoch: 16 | loss=10.777494430541992 | Batch Time=14.84375 +GPU:1 | Epoch: 16 | loss=11.028251647949219 | Batch Time=10.546875 +GPU:1 | Epoch: 16 | loss=10.865974426269531 | Batch Time=11.328125 +GPU:1 | Epoch: 16 | loss=10.903446197509766 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.761743545532227 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.950562477111816 | Batch Time=15.234375 +GPU:1 | Epoch: 16 | loss=11.090360641479492 | Batch Time=12.109375 +GPU:1 | Epoch: 16 | loss=10.945938110351562 | Batch Time=9.765625 +GPU:1 | Epoch: 16 | loss=10.747617721557617 | Batch Time=15.625 +GPU:1 | Epoch: 16 | loss=10.759174346923828 | Batch Time=17.1875 +GPU:1 | Epoch: 16 | loss=10.919288635253906 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.928759574890137 | Batch Time=12.890625 +GPU:1 | Epoch: 16 | loss=10.83453369140625 | Batch Time=13.671875 +GPU:1 | Epoch: 16 | loss=10.735891342163086 | Batch Time=14.84375 +GPU:1 | Epoch: 16 | loss=11.105079650878906 | Batch Time=10.546875 +GPU:1 | Epoch: 16 | loss=10.83810806274414 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.725350379943848 | Batch Time=14.0625 +GPU:1 | Epoch: 16 | loss=10.877523422241211 | Batch Time=14.84375 +GPU:1 | Epoch: 16 | loss=11.072586059570312 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.863688468933105 | Batch Time=9.765625 +GPU:1 | Epoch: 16 | loss=10.79932689666748 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.843221664428711 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.954141616821289 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.770813941955566 | Batch Time=11.71875 +GPU:1 | Epoch: 16 | loss=10.934450149536133 | Batch Time=13.671875 +GPU:1 | Epoch: 16 | loss=11.125577926635742 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.814462661743164 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.845869064331055 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.864517211914062 | Batch Time=12.890625 +GPU:1 | Epoch: 16 | loss=11.06022834777832 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.737203598022461 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.813455581665039 | Batch Time=13.28125 +GPU:1 | Epoch: 16 | loss=10.904701232910156 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.712961196899414 | Batch Time=14.453125 +GPU:1 | Epoch: 16 | loss=10.715814590454102 | Batch Time=14.453125 +GPU:1 | Epoch: 16 | loss=10.839739799499512 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.773650169372559 | Batch Time=14.453125 +GPU:1 | Epoch: 16 | loss=10.717385292053223 | Batch Time=10.15625 +GPU:1 | Epoch: 16 | loss=10.965242385864258 | Batch Time=10.9375 +GPU:1 | Epoch: 16 | loss=10.737571716308594 | Batch Time=12.5 +GPU:1 | Epoch: 16 | loss=10.73829460144043 | Batch Time=12.890625 +GPU:1 | Epoch: 16 | loss=10.719537734985352 | Batch Time=17.1875 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 16 | loss=11.040619850158691 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=11.057727813720703 | Batch Time=9.765625 +GPU:0 | Epoch: 16 | loss=10.98253059387207 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=11.086677551269531 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=10.837959289550781 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.995135307312012 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.988490104675293 | Batch Time=11.328125 +GPU:0 | Epoch: 16 | loss=11.080379486083984 | Batch Time=9.765625 +GPU:0 | Epoch: 16 | loss=10.735296249389648 | Batch Time=14.84375 +GPU:0 | Epoch: 16 | loss=11.167129516601562 | Batch Time=10.9375 +GPU:0 | Epoch: 16 | loss=10.946285247802734 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=10.968952178955078 | Batch Time=9.375 +GPU:0 | Epoch: 16 | loss=11.136598587036133 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=11.189945220947266 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=10.82084846496582 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=11.07266616821289 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=10.997171401977539 | Batch Time=12.109375 +GPU:0 | Epoch: 16 | loss=11.054044723510742 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=10.85428237915039 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=11.151631355285645 | Batch Time=8.984375 +GPU:0 | Epoch: 16 | loss=11.024114608764648 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=10.984813690185547 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=11.111592292785645 | Batch Time=14.84375 +GPU:0 | Epoch: 16 | loss=10.971269607543945 | Batch Time=10.9375 +GPU:0 | Epoch: 16 | loss=11.061708450317383 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=10.922324180603027 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=11.042566299438477 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.999197006225586 | Batch Time=9.375 +GPU:0 | Epoch: 16 | loss=10.832219123840332 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=10.968711853027344 | Batch Time=12.109375 +GPU:0 | Epoch: 16 | loss=10.909602165222168 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=10.965978622436523 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=10.785591125488281 | Batch Time=17.578125 +GPU:0 | Epoch: 16 | loss=11.024690628051758 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=11.088549613952637 | Batch Time=10.9375 +GPU:0 | Epoch: 16 | loss=11.12551498413086 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=11.04867935180664 | Batch Time=10.9375 +GPU:0 | Epoch: 16 | loss=11.037193298339844 | Batch Time=15.625 +GPU:0 | Epoch: 16 | loss=11.023733139038086 | Batch Time=9.375 +GPU:0 | Epoch: 16 | loss=10.944438934326172 | Batch Time=8.984375 +GPU:0 | Epoch: 16 | loss=11.052480697631836 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=10.921884536743164 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=10.846534729003906 | Batch Time=11.328125 +GPU:0 | Epoch: 16 | loss=10.962575912475586 | Batch Time=12.109375 +GPU:0 | Epoch: 16 | loss=11.059213638305664 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=11.042501449584961 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=11.04947280883789 | Batch Time=10.15625 +GPU:0 | Epoch: 16 | loss=10.800649642944336 | Batch Time=10.9375 +GPU:0 | Epoch: 16 | loss=10.906777381896973 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=11.132518768310547 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=10.784465789794922 | Batch Time=16.40625 +GPU:0 | Epoch: 16 | loss=11.025056838989258 | Batch Time=15.625 +GPU:0 | Epoch: 16 | loss=10.741498947143555 | Batch Time=17.1875 +GPU:0 | Epoch: 16 | loss=10.819141387939453 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.923677444458008 | Batch Time=8.984375 +GPU:0 | Epoch: 16 | loss=11.049345016479492 | Batch Time=9.765625 +GPU:0 | Epoch: 16 | loss=10.752148628234863 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=10.869977951049805 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=11.085064888000488 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=11.006563186645508 | Batch Time=10.9375 +GPU:0 | Epoch: 16 | loss=10.991559982299805 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=10.8590669631958 | Batch Time=10.15625 +GPU:0 | Epoch: 16 | loss=10.802553176879883 | Batch Time=14.453125 +GPU:0 | Epoch: 16 | loss=10.768003463745117 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=10.884092330932617 | Batch Time=11.328125 +GPU:0 | Epoch: 16 | loss=10.820838928222656 | Batch Time=10.9375 +GPU:0 | Epoch: 16 | loss=10.97237491607666 | Batch Time=12.890625 +GPU:0 | Epoch: 16 | loss=10.993305206298828 | Batch Time=10.15625 +GPU:0 | Epoch: 16 | loss=11.024398803710938 | Batch Time=9.375 +GPU:0 | Epoch: 16 | loss=10.814809799194336 | Batch Time=18.75 +GPU:0 | Epoch: 16 | loss=10.955138206481934 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=10.830509185791016 | Batch Time=10.15625 +GPU:0 | Epoch: 16 | loss=10.745768547058105 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.820554733276367 | Batch Time=14.84375 +GPU:0 | Epoch: 16 | loss=10.853032112121582 | Batch Time=11.328125 +GPU:0 | Epoch: 16 | loss=10.954723358154297 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=10.93871021270752 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=10.812261581420898 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.7433500289917 | Batch Time=18.75 +GPU:0 | Epoch: 16 | loss=10.79459285736084 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=10.825572967529297 | Batch Time=12.890625 +GPU:0 | Epoch: 16 | loss=11.124635696411133 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=10.921747207641602 | Batch Time=16.40625 +GPU:0 | Epoch: 16 | loss=10.819496154785156 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=10.832164764404297 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=10.768135070800781 | Batch Time=14.84375 +GPU:0 | Epoch: 16 | loss=11.172809600830078 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=10.820619583129883 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.979772567749023 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=10.87270736694336 | Batch Time=15.234375 +GPU:0 | Epoch: 16 | loss=10.82221794128418 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=10.969005584716797 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=10.982589721679688 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=11.055927276611328 | Batch Time=11.328125 +GPU:0 | Epoch: 16 | loss=11.001422882080078 | Batch Time=12.890625 +GPU:0 | Epoch: 16 | loss=10.85731315612793 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.93209457397461 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=10.981035232543945 | Batch Time=9.765625 +GPU:0 | Epoch: 16 | loss=10.74267864227295 | Batch Time=16.015625 +GPU:0 | Epoch: 16 | loss=10.638179779052734 | Batch Time=17.1875 +GPU:0 | Epoch: 16 | loss=10.766132354736328 | Batch Time=17.1875 +GPU:0 | Epoch: 16 | loss=11.029256820678711 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=10.79616928100586 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=11.013690948486328 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=10.874011993408203 | Batch Time=11.328125 +GPU:0 | Epoch: 16 | loss=10.871597290039062 | Batch Time=10.9375 +GPU:0 | Epoch: 16 | loss=11.042661666870117 | Batch Time=12.109375 +GPU:0 | Epoch: 16 | loss=10.867667198181152 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=11.014453887939453 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.962418556213379 | Batch Time=14.453125 +GPU:0 | Epoch: 16 | loss=10.723764419555664 | Batch Time=15.234375 +GPU:0 | Epoch: 16 | loss=10.869401931762695 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=10.914535522460938 | Batch Time=12.109375 +GPU:0 | Epoch: 16 | loss=10.81393051147461 | Batch Time=14.0625 +GPU:0 | Epoch: 16 | loss=10.989665031433105 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=10.804349899291992 | Batch Time=14.453125 +GPU:0 | Epoch: 16 | loss=10.990213394165039 | Batch Time=12.109375 +GPU:0 | Epoch: 16 | loss=10.857516288757324 | Batch Time=13.671875 +GPU:0 | Epoch: 16 | loss=10.915726661682129 | Batch Time=9.765625 +GPU:0 | Epoch: 16 | loss=10.91387939453125 | Batch Time=10.15625 +GPU:0 | Epoch: 16 | loss=10.859895706176758 | Batch Time=10.15625 +GPU:0 | Epoch: 16 | loss=10.738380432128906 | Batch Time=16.015625 +GPU:0 | Epoch: 16 | loss=10.735658645629883 | Batch Time=14.84375 +GPU:0 | Epoch: 16 | loss=10.979799270629883 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=10.758079528808594 | Batch Time=12.890625 +GPU:1 | Epoch: 16 | loss=10.823670387268066 | Batch Time=11.328125 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=110391906304, percent=57.1, used=139263291392, free=17311084544, active=184581226496, inactive=50386571264, buffers=402038784, cached=100591669248, shared=5661540352, slab=3192639488) +AFTER TRAIN LOOP: Epoch 16 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +STARTING TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +BEFORE TRAIN LOOP: Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=110391906304, percent=57.1, used=139263291392, free=17311084544, active=184581226496, inactive=50386571264, buffers=402038784, cached=100591669248, shared=5661540352, slab=3192639488) +AFTER TRAIN LOOP: Epoch 16 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +BEFORE VAL LOOP: GPU: 0 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +GPU:3 | Epoch: 16 | loss=11.020856857299805 | Batch Time=11.328125 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=110391906304, percent=57.1, used=139263291392, free=17311084544, active=184581226496, inactive=50386571264, buffers=402038784, cached=100591669248, shared=5661540352, slab=3192639488) +AFTER TRAIN LOOP: Epoch 16 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +STARTING TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +BEFORE TRAIN LOOP: Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +GPU:2 | Epoch: 16 | loss=10.721294403076172 | Batch Time=17.1875 +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=110391906304, percent=57.1, used=139263291392, free=17311084544, active=184581226496, inactive=50386571264, buffers=402038784, cached=100591669248, shared=5661540352, slab=3192639488) +AFTER TRAIN LOOP: Epoch 16 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +STARTING TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +BEFORE TRAIN LOOP: Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=110392745984, percent=57.1, used=139262697472, free=17311916032, active=184580861952, inactive=50386481152, buffers=402038784, cached=100591431680, shared=5661294592, slab=3192590336) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 16 | loss=3.783663034439087 | Batch Time=24.609375 +GPU:0 | Epoch: 16 | loss=4.8223652839660645 | Batch Time=7.03125 +GPU:0 | Epoch: 16 | loss=4.793120384216309 | Batch Time=18.359375 +GPU:0 | Epoch: 16 | loss=4.451131343841553 | Batch Time=8.203125 +GPU:0 | Epoch: 16 | loss=5.64849853515625 | Batch Time=1.953125 +GPU:0 | Epoch: 16 | loss=4.879901885986328 | Batch Time=7.421875 +GPU:0 | Epoch: 16 | loss=4.7145233154296875 | Batch Time=4.6875 +GPU:0 | Epoch: 16 | loss=5.811066627502441 | Batch Time=3.90625 +GPU:0 | Epoch: 16 | loss=5.302881240844727 | Batch Time=3.515625 +GPU:0 | Epoch: 16 | loss=6.102051258087158 | Batch Time=0.78125 +GPU:0 | Epoch: 16 | loss=5.4763875007629395 | Batch Time=1.5625 +GPU:0 | Epoch: 16 | loss=4.870032787322998 | Batch Time=12.890625 +GPU:0 | Epoch: 16 | loss=4.985108852386475 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=4.511333465576172 | Batch Time=16.40625 +GPU:0 | Epoch: 16 | loss=4.844858169555664 | Batch Time=4.6875 +GPU:0 | Epoch: 16 | loss=4.955650329589844 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=4.768664836883545 | Batch Time=8.203125 +GPU:0 | Epoch: 16 | loss=3.763244152069092 | Batch Time=23.828125 +GPU:0 | Epoch: 16 | loss=4.257292747497559 | Batch Time=16.40625 +GPU:0 | Epoch: 16 | loss=3.677156448364258 | Batch Time=26.5625 +Model top1 Accuracy: 12.05 +Acc before rounding: 12.05 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 16 | loss=3.783663034439087 | Batch Time=24.609375 +GPU:0 | Epoch: 16 | loss=4.8223652839660645 | Batch Time=7.03125 +GPU:0 | Epoch: 16 | loss=4.793120384216309 | Batch Time=18.359375 +GPU:0 | Epoch: 16 | loss=4.451131343841553 | Batch Time=8.203125 +GPU:0 | Epoch: 16 | loss=5.64849853515625 | Batch Time=1.953125 +GPU:0 | Epoch: 16 | loss=4.879901885986328 | Batch Time=7.421875 +GPU:0 | Epoch: 16 | loss=4.7145233154296875 | Batch Time=4.6875 +GPU:0 | Epoch: 16 | loss=5.811066627502441 | Batch Time=3.90625 +GPU:0 | Epoch: 16 | loss=5.302881240844727 | Batch Time=3.515625 +GPU:0 | Epoch: 16 | loss=6.102051258087158 | Batch Time=0.78125 +GPU:0 | Epoch: 16 | loss=5.4763875007629395 | Batch Time=1.5625 +GPU:0 | Epoch: 16 | loss=4.870032787322998 | Batch Time=12.890625 +GPU:0 | Epoch: 16 | loss=4.985108852386475 | Batch Time=13.28125 +GPU:0 | Epoch: 16 | loss=4.511333465576172 | Batch Time=16.40625 +GPU:0 | Epoch: 16 | loss=4.844858169555664 | Batch Time=4.6875 +GPU:0 | Epoch: 16 | loss=4.955650329589844 | Batch Time=12.5 +GPU:0 | Epoch: 16 | loss=4.768664836883545 | Batch Time=8.203125 +GPU:0 | Epoch: 16 | loss=3.763244152069092 | Batch Time=23.828125 +GPU:0 | Epoch: 16 | loss=4.257292747497559 | Batch Time=16.40625 +GPU:0 | Epoch: 16 | loss=3.677156448364258 | Batch Time=26.5625 +Model top1 Accuracy: 12.05 +Acc after rounding: 12.05 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 16 | loss=4.9959235191345215 | Batch Time=10.546875 +GPU:0 | Epoch: 16 | loss=5.1004557609558105 | Batch Time=11.328125 +GPU:0 | Epoch: 16 | loss=5.006574630737305 | Batch Time=11.71875 +GPU:0 | Epoch: 16 | loss=5.1177778244018555 | Batch Time=8.59375 +Model top1 Accuracy: 10.26 +Validation Acc after rounding: 10.26 +AFTER VAL LOOP: GPU: 0 | Epoch 16 | Memory Usage: svmem(total=257568083968, available=92555804672, percent=64.1, used=157099257856, free=5251473408, active=205448450048, inactive=41466339328, buffers=408084480, cached=94809268224, shared=5661687808, slab=3197108224) +Rounding model with scheme: naive +Model avg sparsity: 40.96479038283425 +GPU:0 | Epoch: 16 | Acc=12.05 | Epoch Time=17.285814809799195 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=92555812864, percent=64.1, used=157099249664, free=5251481600, active=205448437760, inactive=41466339328, buffers=408088576, cached=94809264128, shared=5661687808, slab=3197100032) +STARTING TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=92555812864, percent=64.1, used=157099249664, free=5251481600, active=205448437760, inactive=41466339328, buffers=408088576, cached=94809264128, shared=5661687808, slab=3197100032) +BEFORE TRAIN LOOP: Epoch 17 | Memory Usage: svmem(total=257568083968, available=92555812864, percent=64.1, used=157099249664, free=5251481600, active=205448437760, inactive=41466339328, buffers=408088576, cached=94809264128, shared=5661687808, slab=3197100032) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=92555812864, percent=64.1, used=157099249664, free=5251481600, active=205448437760, inactive=41466339328, buffers=408088576, cached=94809264128, shared=5661687808, slab=3197100032) +GPU:3 | Epoch: 17 | loss=10.896085739135742 | Batch Time=14.84375 +GPU:3 | Epoch: 17 | loss=10.916549682617188 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.859640121459961 | Batch Time=6.25 +GPU:3 | Epoch: 17 | loss=10.8284273147583 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.802379608154297 | Batch Time=10.9375 +GPU:3 | Epoch: 17 | loss=10.720758438110352 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.837007522583008 | Batch Time=15.234375 +GPU:3 | Epoch: 17 | loss=10.871967315673828 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.913111686706543 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.844764709472656 | Batch Time=16.015625 +GPU:3 | Epoch: 17 | loss=10.722110748291016 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.77806568145752 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.883246421813965 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=11.0191650390625 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.655019760131836 | Batch Time=16.015625 +GPU:3 | Epoch: 17 | loss=10.707090377807617 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.957342147827148 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.960160255432129 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.955652236938477 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.891136169433594 | Batch Time=13.671875 +GPU:3 | Epoch: 17 | loss=10.741142272949219 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.816555976867676 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=11.102824211120605 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.822261810302734 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.7897367477417 | Batch Time=13.671875 +GPU:3 | Epoch: 17 | loss=10.620216369628906 | Batch Time=15.234375 +GPU:3 | Epoch: 17 | loss=10.739286422729492 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.900418281555176 | Batch Time=10.9375 +GPU:3 | Epoch: 17 | loss=10.804091453552246 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.871119499206543 | Batch Time=8.203125 +GPU:3 | Epoch: 17 | loss=10.684338569641113 | Batch Time=10.9375 +GPU:3 | Epoch: 17 | loss=10.635272026062012 | Batch Time=15.625 +GPU:3 | Epoch: 17 | loss=10.947467803955078 | Batch Time=16.015625 +GPU:3 | Epoch: 17 | loss=10.802898406982422 | Batch Time=11.71875 +GPU:3 | Epoch: 17 | loss=10.80118179321289 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.79284954071045 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.800580978393555 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.952384948730469 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.816953659057617 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=11.063687324523926 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.658967971801758 | Batch Time=14.84375 +GPU:3 | Epoch: 17 | loss=10.748725891113281 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.972785949707031 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.75277042388916 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.771041870117188 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.969441413879395 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.703165054321289 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.858838081359863 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.82071304321289 | Batch Time=11.71875 +GPU:3 | Epoch: 17 | loss=10.715814590454102 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.832160949707031 | Batch Time=10.546875 +GPU:3 | Epoch: 17 | loss=10.593315124511719 | Batch Time=14.84375 +GPU:3 | Epoch: 17 | loss=10.525918006896973 | Batch Time=16.40625 +GPU:3 | Epoch: 17 | loss=10.67248249053955 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.801816940307617 | Batch Time=14.453125 +GPU:3 | Epoch: 17 | loss=10.688352584838867 | Batch Time=17.1875 +GPU:3 | Epoch: 17 | loss=10.647693634033203 | Batch Time=13.671875 +GPU:3 | Epoch: 17 | loss=10.681074142456055 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.823835372924805 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.724689483642578 | Batch Time=14.453125 +GPU:3 | Epoch: 17 | loss=10.885141372680664 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.784570693969727 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.764973640441895 | Batch Time=14.84375 +GPU:3 | Epoch: 17 | loss=10.800497055053711 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.928323745727539 | Batch Time=10.15625 +GPU:3 | Epoch: 17 | loss=10.724714279174805 | Batch Time=15.234375 +GPU:3 | Epoch: 17 | loss=10.870798110961914 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.860700607299805 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.590005874633789 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.731022834777832 | Batch Time=17.1875 +GPU:3 | Epoch: 17 | loss=10.713974952697754 | Batch Time=14.84375 +GPU:3 | Epoch: 17 | loss=10.891241073608398 | Batch Time=10.9375 +GPU:3 | Epoch: 17 | loss=10.724201202392578 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.87388801574707 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.82862377166748 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.687557220458984 | Batch Time=15.234375 +GPU:3 | Epoch: 17 | loss=10.649446487426758 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.709999084472656 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.865331649780273 | Batch Time=11.71875 +GPU:3 | Epoch: 17 | loss=10.625995635986328 | Batch Time=14.453125 +GPU:3 | Epoch: 17 | loss=10.563785552978516 | Batch Time=18.75 +GPU:3 | Epoch: 17 | loss=10.756515502929688 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.628579139709473 | Batch Time=16.40625 +GPU:3 | Epoch: 17 | loss=10.869462013244629 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.880006790161133 | Batch Time=11.328125 +GPU:3 | Epoch: 17 | loss=10.994722366333008 | Batch Time=10.546875 +GPU:3 | Epoch: 17 | loss=10.593036651611328 | Batch Time=15.234375 +GPU:3 | Epoch: 17 | loss=10.809856414794922 | Batch Time=10.15625 +GPU:3 | Epoch: 17 | loss=10.754117965698242 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.721513748168945 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.577835083007812 | Batch Time=13.671875 +GPU:3 | Epoch: 17 | loss=10.827521324157715 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.658894538879395 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.828478813171387 | Batch Time=15.625 +GPU:3 | Epoch: 17 | loss=10.624628067016602 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.562094688415527 | Batch Time=12.5 +GPU:3 | Epoch: 17 | loss=10.722600936889648 | Batch Time=15.234375 +GPU:3 | Epoch: 17 | loss=10.6699857711792 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.65206241607666 | Batch Time=15.234375 +GPU:3 | Epoch: 17 | loss=10.63235855102539 | Batch Time=15.625 +GPU:3 | Epoch: 17 | loss=10.639017105102539 | Batch Time=10.15625 +GPU:3 | Epoch: 17 | loss=10.784980773925781 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.592856407165527 | Batch Time=14.453125 +GPU:3 | Epoch: 17 | loss=10.752872467041016 | Batch Time=10.546875 +GPU:3 | Epoch: 17 | loss=10.606287956237793 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.717477798461914 | Batch Time=12.109375 +GPU:3 | Epoch: 17 | loss=10.797718048095703 | Batch Time=7.8125 +GPU:3 | Epoch: 17 | loss=10.690140724182129 | Batch Time=16.40625 +GPU:3 | Epoch: 17 | loss=10.715686798095703 | Batch Time=12.890625 +GPU:3 | Epoch: 17 | loss=10.603577613830566 | Batch Time=17.578125 +GPU:3 | Epoch: 17 | loss=10.667638778686523 | Batch Time=13.28125 +GPU:3 | Epoch: 17 | loss=10.779031753540039 | Batch Time=13.671875 +GPU:3 | Epoch: 17 | loss=10.81576919555664 | Batch Time=14.0625 +GPU:3 | Epoch: 17 | loss=10.772029876708984 | Batch Time=14.453125 +GPU:3 | Epoch: 17 | loss=10.454635620117188 | Batch Time=16.015625 +GPU:3 | Epoch: 17 | loss=10.525169372558594 | Batch Time=16.015625 +GPU:3 | Epoch: 17 | loss=10.706916809082031 | Batch Time=15.234375 +GPU:3 | Epoch: 17 | loss=10.74586296081543 | Batch Time=16.015625 +GPU:3 | Epoch: 17 | loss=10.370624542236328 | Batch Time=17.578125 +GPU:3 | Epoch: 17 | loss=10.87596321105957 | Batch Time=10.9375 +GPU:3 | Epoch: 17 | loss=10.470925331115723 | Batch Time=14.453125 +GPU:3 | Epoch: 17 | loss=10.556117057800293 | Batch Time=16.796875 +GPU:3 | Epoch: 17 | loss=10.55411434173584 | Batch Time=15.625 +GPU:3 | Epoch: 17 | loss=10.462987899780273 | Batch Time=16.796875 +GPU:1 | Epoch: 17 | loss=10.891390800476074 | Batch Time=9.765625 +GPU:1 | Epoch: 17 | loss=10.726930618286133 | Batch Time=17.578125 +GPU:1 | Epoch: 17 | loss=10.888273239135742 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.758682250976562 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.913728713989258 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=11.020630836486816 | Batch Time=11.328125 +GPU:1 | Epoch: 17 | loss=10.695283889770508 | Batch Time=16.015625 +GPU:1 | Epoch: 17 | loss=10.782093048095703 | Batch Time=9.765625 +GPU:1 | Epoch: 17 | loss=10.875503540039062 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.907434463500977 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.758634567260742 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.8329439163208 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.972990036010742 | Batch Time=11.328125 +GPU:1 | Epoch: 17 | loss=10.725972175598145 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.883893966674805 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.761131286621094 | Batch Time=16.40625 +GPU:1 | Epoch: 17 | loss=10.976298332214355 | Batch Time=11.328125 +GPU:1 | Epoch: 17 | loss=10.641793251037598 | Batch Time=16.796875 +GPU:1 | Epoch: 17 | loss=10.778120040893555 | Batch Time=16.40625 +GPU:1 | Epoch: 17 | loss=10.8192777633667 | Batch Time=13.28125 +GPU:1 | Epoch: 17 | loss=10.906339645385742 | Batch Time=10.546875 +GPU:1 | Epoch: 17 | loss=11.096912384033203 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.719321250915527 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.704885482788086 | Batch Time=11.71875 +GPU:1 | Epoch: 17 | loss=10.772645950317383 | Batch Time=17.1875 +GPU:1 | Epoch: 17 | loss=10.831111907958984 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.812407493591309 | Batch Time=16.015625 +GPU:1 | Epoch: 17 | loss=10.854386329650879 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.879666328430176 | Batch Time=13.28125 +GPU:1 | Epoch: 17 | loss=10.76762866973877 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.900510787963867 | Batch Time=15.234375 +GPU:1 | Epoch: 17 | loss=10.694271087646484 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.790327072143555 | Batch Time=14.84375 +GPU:1 | Epoch: 17 | loss=10.896315574645996 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.783906936645508 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.811854362487793 | Batch Time=14.84375 +GPU:1 | Epoch: 17 | loss=10.767532348632812 | Batch Time=13.28125 +GPU:1 | Epoch: 17 | loss=10.699576377868652 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.706892967224121 | Batch Time=14.0625 +GPU:1 | Epoch: 17 | loss=10.914495468139648 | Batch Time=14.453125 +GPU:1 | Epoch: 17 | loss=10.964242935180664 | Batch Time=10.546875 +GPU:1 | Epoch: 17 | loss=10.907703399658203 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.76956558227539 | Batch Time=14.453125 +GPU:1 | Epoch: 17 | loss=10.634631156921387 | Batch Time=15.234375 +GPU:1 | Epoch: 17 | loss=10.979656219482422 | Batch Time=13.28125 +GPU:1 | Epoch: 17 | loss=10.763179779052734 | Batch Time=10.546875 +GPU:1 | Epoch: 17 | loss=10.73448371887207 | Batch Time=16.796875 +GPU:1 | Epoch: 17 | loss=10.748517036437988 | Batch Time=14.453125 +GPU:1 | Epoch: 17 | loss=10.91482162475586 | Batch Time=9.765625 +GPU:1 | Epoch: 17 | loss=10.841804504394531 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.878089904785156 | Batch Time=11.328125 +GPU:1 | Epoch: 17 | loss=10.673650741577148 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.693878173828125 | Batch Time=17.1875 +GPU:1 | Epoch: 17 | loss=10.735128402709961 | Batch Time=11.328125 +GPU:1 | Epoch: 17 | loss=10.73671817779541 | Batch Time=15.234375 +GPU:1 | Epoch: 17 | loss=10.988336563110352 | Batch Time=8.984375 +GPU:1 | Epoch: 17 | loss=10.477766036987305 | Batch Time=18.359375 +GPU:1 | Epoch: 17 | loss=11.044471740722656 | Batch Time=8.59375 +GPU:1 | Epoch: 17 | loss=10.750399589538574 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.809894561767578 | Batch Time=14.453125 +GPU:1 | Epoch: 17 | loss=10.744132995605469 | Batch Time=14.0625 +GPU:1 | Epoch: 17 | loss=10.866385459899902 | Batch Time=14.0625 +GPU:1 | Epoch: 17 | loss=10.75327205657959 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.856637954711914 | Batch Time=11.71875 +GPU:1 | Epoch: 17 | loss=10.664153099060059 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.953539848327637 | Batch Time=9.765625 +GPU:1 | Epoch: 17 | loss=10.90400505065918 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.754084587097168 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.69804573059082 | Batch Time=13.28125 +GPU:1 | Epoch: 17 | loss=10.804435729980469 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.549304962158203 | Batch Time=14.0625 +GPU:1 | Epoch: 17 | loss=10.71919059753418 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.668240547180176 | Batch Time=11.71875 +GPU:1 | Epoch: 17 | loss=10.593225479125977 | Batch Time=15.234375 +GPU:1 | Epoch: 17 | loss=10.634806632995605 | Batch Time=15.625 +GPU:1 | Epoch: 17 | loss=10.855180740356445 | Batch Time=11.328125 +GPU:1 | Epoch: 17 | loss=10.7852783203125 | Batch Time=11.71875 +GPU:1 | Epoch: 17 | loss=10.795867919921875 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.68458366394043 | Batch Time=13.28125 +GPU:1 | Epoch: 17 | loss=10.56869888305664 | Batch Time=15.234375 +GPU:1 | Epoch: 17 | loss=10.464082717895508 | Batch Time=15.625 +GPU:1 | Epoch: 17 | loss=10.542024612426758 | Batch Time=16.796875 +GPU:1 | Epoch: 17 | loss=10.849863052368164 | Batch Time=9.765625 +GPU:1 | Epoch: 17 | loss=10.658618927001953 | Batch Time=14.0625 +GPU:1 | Epoch: 17 | loss=10.793083190917969 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.538705825805664 | Batch Time=15.625 +GPU:1 | Epoch: 17 | loss=10.733899116516113 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.642040252685547 | Batch Time=14.0625 +GPU:1 | Epoch: 17 | loss=10.630796432495117 | Batch Time=17.578125 +GPU:1 | Epoch: 17 | loss=10.808366775512695 | Batch Time=9.765625 +GPU:1 | Epoch: 17 | loss=10.668954849243164 | Batch Time=15.234375 +GPU:1 | Epoch: 17 | loss=10.72055721282959 | Batch Time=11.71875 +GPU:1 | Epoch: 17 | loss=10.569814682006836 | Batch Time=17.1875 +GPU:1 | Epoch: 17 | loss=10.718724250793457 | Batch Time=16.796875 +GPU:1 | Epoch: 17 | loss=10.70058822631836 | Batch Time=14.84375 +GPU:1 | Epoch: 17 | loss=10.637378692626953 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.574140548706055 | Batch Time=15.234375 +GPU:1 | Epoch: 17 | loss=10.927872657775879 | Batch Time=12.109375 +GPU:1 | Epoch: 17 | loss=10.683294296264648 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.637712478637695 | Batch Time=15.234375 +GPU:1 | Epoch: 17 | loss=10.961681365966797 | Batch Time=13.671875 +GPU:1 | Epoch: 17 | loss=10.469063758850098 | Batch Time=19.140625 +GPU:1 | Epoch: 17 | loss=10.736211776733398 | Batch Time=10.546875 +GPU:1 | Epoch: 17 | loss=10.570915222167969 | Batch Time=11.71875 +GPU:1 | Epoch: 17 | loss=10.7788667678833 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.631959915161133 | Batch Time=15.625 +GPU:1 | Epoch: 17 | loss=10.84084701538086 | Batch Time=7.421875 +GPU:1 | Epoch: 17 | loss=10.785232543945312 | Batch Time=12.5 +GPU:1 | Epoch: 17 | loss=10.667072296142578 | Batch Time=14.453125 +GPU:1 | Epoch: 17 | loss=10.614664077758789 | Batch Time=17.1875 +GPU:1 | Epoch: 17 | loss=10.843442916870117 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.668121337890625 | Batch Time=18.75 +GPU:1 | Epoch: 17 | loss=10.578449249267578 | Batch Time=14.84375 +GPU:1 | Epoch: 17 | loss=10.573333740234375 | Batch Time=17.1875 +GPU:1 | Epoch: 17 | loss=10.73233413696289 | Batch Time=14.0625 +GPU:1 | Epoch: 17 | loss=10.892208099365234 | Batch Time=8.984375 +GPU:1 | Epoch: 17 | loss=10.782581329345703 | Batch Time=14.0625 +GPU:1 | Epoch: 17 | loss=10.633522033691406 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.559868812561035 | Batch Time=16.796875 +GPU:1 | Epoch: 17 | loss=10.497852325439453 | Batch Time=15.625 +GPU:1 | Epoch: 17 | loss=10.520344734191895 | Batch Time=17.1875 +GPU:1 | Epoch: 17 | loss=10.634469985961914 | Batch Time=12.890625 +GPU:1 | Epoch: 17 | loss=10.902008056640625 | Batch Time=10.546875 +GPU:1 | Epoch: 17 | loss=10.764528274536133 | Batch Time=11.328125 +GPU:0 | Epoch: 17 | loss=10.876609802246094 | Batch Time=10.546875 +GPU:0 | Epoch: 17 | loss=10.664817810058594 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.587045669555664 | Batch Time=18.75 +GPU:0 | Epoch: 17 | loss=10.931941986083984 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=10.612994194030762 | Batch Time=17.578125 +GPU:0 | Epoch: 17 | loss=10.52242660522461 | Batch Time=16.796875 +GPU:0 | Epoch: 17 | loss=10.804945945739746 | Batch Time=14.453125 +GPU:0 | Epoch: 17 | loss=11.03923225402832 | Batch Time=8.984375 +GPU:0 | Epoch: 17 | loss=10.799782752990723 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.753491401672363 | Batch Time=12.5 +GPU:0 | Epoch: 17 | loss=10.970300674438477 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=10.776844024658203 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.649517059326172 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=10.549196243286133 | Batch Time=15.625 +GPU:0 | Epoch: 17 | loss=10.813859939575195 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=10.506832122802734 | Batch Time=18.75 +GPU:0 | Epoch: 17 | loss=10.850641250610352 | Batch Time=10.546875 +GPU:0 | Epoch: 17 | loss=10.529792785644531 | Batch Time=17.578125 +GPU:0 | Epoch: 17 | loss=11.015523910522461 | Batch Time=11.328125 +GPU:0 | Epoch: 17 | loss=10.729110717773438 | Batch Time=12.5 +GPU:0 | Epoch: 17 | loss=10.862995147705078 | Batch Time=12.5 +GPU:0 | Epoch: 17 | loss=11.011672973632812 | Batch Time=11.328125 +GPU:0 | Epoch: 17 | loss=10.759318351745605 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.897083282470703 | Batch Time=10.15625 +GPU:0 | Epoch: 17 | loss=10.746933937072754 | Batch Time=15.625 +GPU:0 | Epoch: 17 | loss=10.660259246826172 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.739744186401367 | Batch Time=10.546875 +GPU:0 | Epoch: 17 | loss=10.649370193481445 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.839086532592773 | Batch Time=15.234375 +GPU:0 | Epoch: 17 | loss=10.823795318603516 | Batch Time=9.765625 +GPU:0 | Epoch: 17 | loss=10.858699798583984 | Batch Time=12.5 +GPU:0 | Epoch: 17 | loss=10.998922348022461 | Batch Time=9.765625 +GPU:0 | Epoch: 17 | loss=10.868889808654785 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.838152885437012 | Batch Time=11.328125 +GPU:0 | Epoch: 17 | loss=10.914094924926758 | Batch Time=11.71875 +GPU:0 | Epoch: 17 | loss=10.90694522857666 | Batch Time=11.71875 +GPU:0 | Epoch: 17 | loss=10.923656463623047 | Batch Time=9.765625 +GPU:0 | Epoch: 17 | loss=10.918027877807617 | Batch Time=11.328125 +GPU:0 | Epoch: 17 | loss=10.700332641601562 | Batch Time=12.890625 +GPU:0 | Epoch: 17 | loss=10.71613883972168 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=10.893209457397461 | Batch Time=12.890625 +GPU:0 | Epoch: 17 | loss=10.757457733154297 | Batch Time=9.765625 +GPU:0 | Epoch: 17 | loss=10.628568649291992 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=10.716846466064453 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=10.772567749023438 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.63156509399414 | Batch Time=18.359375 +GPU:0 | Epoch: 17 | loss=10.65044116973877 | Batch Time=14.453125 +GPU:0 | Epoch: 17 | loss=10.871223449707031 | Batch Time=12.5 +GPU:0 | Epoch: 17 | loss=10.725910186767578 | Batch Time=12.890625 +GPU:0 | Epoch: 17 | loss=10.925466537475586 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=10.830646514892578 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.71342658996582 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=10.751741409301758 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.76050090789795 | Batch Time=10.15625 +GPU:0 | Epoch: 17 | loss=10.615774154663086 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.642748832702637 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.546924591064453 | Batch Time=19.53125 +GPU:0 | Epoch: 17 | loss=10.814583778381348 | Batch Time=12.890625 +GPU:0 | Epoch: 17 | loss=10.696357727050781 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.662558555603027 | Batch Time=15.234375 +GPU:0 | Epoch: 17 | loss=10.779394149780273 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.628593444824219 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.810699462890625 | Batch Time=17.1875 +GPU:0 | Epoch: 17 | loss=10.766118049621582 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.757258415222168 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.720748901367188 | Batch Time=12.5 +GPU:0 | Epoch: 17 | loss=10.791500091552734 | Batch Time=10.9375 +GPU:0 | Epoch: 17 | loss=10.563371658325195 | Batch Time=17.96875 +GPU:0 | Epoch: 17 | loss=10.697993278503418 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=11.011512756347656 | Batch Time=11.328125 +GPU:0 | Epoch: 17 | loss=10.693612098693848 | Batch Time=16.796875 +GPU:0 | Epoch: 17 | loss=10.829346656799316 | Batch Time=15.625 +GPU:0 | Epoch: 17 | loss=10.69483757019043 | Batch Time=14.453125 +GPU:0 | Epoch: 17 | loss=10.663042068481445 | Batch Time=15.234375 +GPU:0 | Epoch: 17 | loss=10.699176788330078 | Batch Time=11.71875 +GPU:0 | Epoch: 17 | loss=10.655370712280273 | Batch Time=16.796875 +GPU:0 | Epoch: 17 | loss=10.80685806274414 | Batch Time=12.5 +GPU:0 | Epoch: 17 | loss=10.900951385498047 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.67381477355957 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.78436279296875 | Batch Time=10.546875 +GPU:0 | Epoch: 17 | loss=10.581628799438477 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.574819564819336 | Batch Time=17.578125 +GPU:0 | Epoch: 17 | loss=10.8495454788208 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.709328651428223 | Batch Time=17.578125 +GPU:0 | Epoch: 17 | loss=10.643909454345703 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=10.677505493164062 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.719602584838867 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.574356079101562 | Batch Time=18.359375 +GPU:0 | Epoch: 17 | loss=10.764814376831055 | Batch Time=14.453125 +GPU:0 | Epoch: 17 | loss=10.661806106567383 | Batch Time=14.84375 +GPU:0 | Epoch: 17 | loss=10.62987995147705 | Batch Time=14.453125 +GPU:0 | Epoch: 17 | loss=10.609525680541992 | Batch Time=12.5 +GPU:0 | Epoch: 17 | loss=10.800405502319336 | Batch Time=10.15625 +GPU:0 | Epoch: 17 | loss=10.571184158325195 | Batch Time=15.234375 +GPU:0 | Epoch: 17 | loss=10.589387893676758 | Batch Time=16.796875 +GPU:0 | Epoch: 17 | loss=10.783613204956055 | Batch Time=10.546875 +GPU:0 | Epoch: 17 | loss=10.81631851196289 | Batch Time=11.328125 +GPU:0 | Epoch: 17 | loss=10.66602611541748 | Batch Time=11.71875 +GPU:0 | Epoch: 17 | loss=10.70278549194336 | Batch Time=14.453125 +GPU:0 | Epoch: 17 | loss=10.750896453857422 | Batch Time=16.015625 +GPU:0 | Epoch: 17 | loss=10.822233200073242 | Batch Time=10.15625 +GPU:0 | Epoch: 17 | loss=10.77449893951416 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.908822059631348 | Batch Time=10.546875 +GPU:0 | Epoch: 17 | loss=10.84282112121582 | Batch Time=12.890625 +GPU:0 | Epoch: 17 | loss=10.68663501739502 | Batch Time=15.234375 +GPU:0 | Epoch: 17 | loss=10.607294082641602 | Batch Time=14.453125 +GPU:0 | Epoch: 17 | loss=10.555215835571289 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.550228118896484 | Batch Time=17.578125 +GPU:0 | Epoch: 17 | loss=10.555152893066406 | Batch Time=16.015625 +GPU:0 | Epoch: 17 | loss=10.669479370117188 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=10.815614700317383 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=10.509456634521484 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=10.677118301391602 | Batch Time=16.40625 +GPU:0 | Epoch: 17 | loss=10.688783645629883 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=10.7783203125 | Batch Time=10.9375 +GPU:0 | Epoch: 17 | loss=10.649775505065918 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=10.715681076049805 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.828343391418457 | Batch Time=10.15625 +GPU:0 | Epoch: 17 | loss=10.658562660217285 | Batch Time=15.234375 +GPU:0 | Epoch: 17 | loss=10.566560745239258 | Batch Time=13.671875 +GPU:0 | Epoch: 17 | loss=10.753768920898438 | Batch Time=16.015625 +GPU:0 | Epoch: 17 | loss=10.760377883911133 | Batch Time=12.890625 +GPU:0 | Epoch: 17 | loss=10.563039779663086 | Batch Time=14.453125 +GPU:0 | Epoch: 17 | loss=10.539571762084961 | Batch Time=16.015625 +GPU:2 | Epoch: 17 | loss=10.904256820678711 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.757009506225586 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=11.022024154663086 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.721006393432617 | Batch Time=15.625 +GPU:2 | Epoch: 17 | loss=10.5904541015625 | Batch Time=16.40625 +GPU:2 | Epoch: 17 | loss=10.815526008605957 | Batch Time=10.9375 +GPU:2 | Epoch: 17 | loss=10.64731216430664 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.830963134765625 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=11.014023780822754 | Batch Time=8.59375 +GPU:2 | Epoch: 17 | loss=10.952407836914062 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.828924179077148 | Batch Time=14.84375 +GPU:2 | Epoch: 17 | loss=10.778823852539062 | Batch Time=17.96875 +GPU:2 | Epoch: 17 | loss=10.628962516784668 | Batch Time=14.84375 +GPU:2 | Epoch: 17 | loss=10.760979652404785 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.626792907714844 | Batch Time=19.921875 +GPU:2 | Epoch: 17 | loss=10.816118240356445 | Batch Time=10.15625 +GPU:2 | Epoch: 17 | loss=10.903570175170898 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.741748809814453 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.776044845581055 | Batch Time=11.328125 +GPU:2 | Epoch: 17 | loss=10.902430534362793 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.730512619018555 | Batch Time=15.625 +GPU:2 | Epoch: 17 | loss=10.826220512390137 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.936525344848633 | Batch Time=12.109375 +GPU:2 | Epoch: 17 | loss=11.018594741821289 | Batch Time=12.109375 +GPU:2 | Epoch: 17 | loss=10.792959213256836 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.701837539672852 | Batch Time=16.015625 +GPU:2 | Epoch: 17 | loss=10.807132720947266 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.740930557250977 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.861640930175781 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.838272094726562 | Batch Time=15.234375 +GPU:2 | Epoch: 17 | loss=10.663469314575195 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.83353042602539 | Batch Time=14.84375 +GPU:2 | Epoch: 17 | loss=10.556181907653809 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.835867881774902 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.762687683105469 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.725117683410645 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.724437713623047 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.84988021850586 | Batch Time=14.0625 +GPU:2 | Epoch: 17 | loss=10.624467849731445 | Batch Time=16.796875 +GPU:2 | Epoch: 17 | loss=10.667680740356445 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.626802444458008 | Batch Time=16.015625 +GPU:2 | Epoch: 17 | loss=10.947811126708984 | Batch Time=9.765625 +GPU:2 | Epoch: 17 | loss=10.747127532958984 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.737770080566406 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.684667587280273 | Batch Time=16.40625 +GPU:2 | Epoch: 17 | loss=10.715654373168945 | Batch Time=14.84375 +GPU:2 | Epoch: 17 | loss=10.773008346557617 | Batch Time=14.0625 +GPU:2 | Epoch: 17 | loss=10.72413444519043 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.994500160217285 | Batch Time=8.984375 +GPU:2 | Epoch: 17 | loss=10.801077842712402 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.764697074890137 | Batch Time=14.0625 +GPU:2 | Epoch: 17 | loss=10.656143188476562 | Batch Time=14.84375 +GPU:2 | Epoch: 17 | loss=10.735845565795898 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.840246200561523 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.651758193969727 | Batch Time=18.359375 +GPU:2 | Epoch: 17 | loss=11.092645645141602 | Batch Time=10.546875 +GPU:2 | Epoch: 17 | loss=10.94558334350586 | Batch Time=10.546875 +GPU:2 | Epoch: 17 | loss=10.754549026489258 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.73094367980957 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.660953521728516 | Batch Time=14.0625 +GPU:2 | Epoch: 17 | loss=10.740705490112305 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.73238754272461 | Batch Time=15.234375 +GPU:2 | Epoch: 17 | loss=10.694416046142578 | Batch Time=18.75 +GPU:2 | Epoch: 17 | loss=10.807257652282715 | Batch Time=15.234375 +GPU:2 | Epoch: 17 | loss=10.647007942199707 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.798360824584961 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.6524658203125 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.777872085571289 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.615507125854492 | Batch Time=15.625 +GPU:2 | Epoch: 17 | loss=10.845943450927734 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.719332695007324 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.977629661560059 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.653268814086914 | Batch Time=15.234375 +GPU:2 | Epoch: 17 | loss=10.663763046264648 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.69791030883789 | Batch Time=16.015625 +GPU:2 | Epoch: 17 | loss=10.840217590332031 | Batch Time=11.328125 +GPU:2 | Epoch: 17 | loss=10.966445922851562 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.793678283691406 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.46863079071045 | Batch Time=19.140625 +GPU:2 | Epoch: 17 | loss=10.711907386779785 | Batch Time=16.015625 +GPU:2 | Epoch: 17 | loss=10.563953399658203 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.327651023864746 | Batch Time=18.359375 +GPU:2 | Epoch: 17 | loss=10.758939743041992 | Batch Time=15.625 +GPU:2 | Epoch: 17 | loss=10.782456398010254 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.782691955566406 | Batch Time=12.109375 +GPU:2 | Epoch: 17 | loss=10.680185317993164 | Batch Time=11.328125 +GPU:2 | Epoch: 17 | loss=10.704879760742188 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.839834213256836 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.585533142089844 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.645442962646484 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.705507278442383 | Batch Time=16.015625 +GPU:2 | Epoch: 17 | loss=10.877741813659668 | Batch Time=11.328125 +GPU:2 | Epoch: 17 | loss=10.636651992797852 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.762787818908691 | Batch Time=11.71875 +GPU:2 | Epoch: 17 | loss=10.452409744262695 | Batch Time=17.96875 +GPU:2 | Epoch: 17 | loss=10.753555297851562 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.739633560180664 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.87043571472168 | Batch Time=12.890625 +GPU:2 | Epoch: 17 | loss=10.667550086975098 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.692244529724121 | Batch Time=17.578125 +GPU:2 | Epoch: 17 | loss=10.733598709106445 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.55919075012207 | Batch Time=14.84375 +GPU:2 | Epoch: 17 | loss=10.755659103393555 | Batch Time=15.234375 +GPU:2 | Epoch: 17 | loss=10.577518463134766 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.703330993652344 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.58012580871582 | Batch Time=15.234375 +GPU:2 | Epoch: 17 | loss=10.674198150634766 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.502408027648926 | Batch Time=17.1875 +GPU:2 | Epoch: 17 | loss=10.50368881225586 | Batch Time=14.84375 +GPU:2 | Epoch: 17 | loss=10.65069580078125 | Batch Time=13.671875 +GPU:2 | Epoch: 17 | loss=10.465057373046875 | Batch Time=18.359375 +GPU:2 | Epoch: 17 | loss=10.644813537597656 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.65926742553711 | Batch Time=12.109375 +GPU:2 | Epoch: 17 | loss=10.658792495727539 | Batch Time=16.40625 +GPU:2 | Epoch: 17 | loss=10.500682830810547 | Batch Time=12.109375 +GPU:2 | Epoch: 17 | loss=10.72454833984375 | Batch Time=14.453125 +GPU:2 | Epoch: 17 | loss=10.709774017333984 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.735332489013672 | Batch Time=15.625 +GPU:2 | Epoch: 17 | loss=10.700736999511719 | Batch Time=14.84375 +GPU:2 | Epoch: 17 | loss=10.753406524658203 | Batch Time=12.5 +GPU:2 | Epoch: 17 | loss=10.488045692443848 | Batch Time=16.40625 +GPU:2 | Epoch: 17 | loss=10.620431900024414 | Batch Time=17.578125 +GPU:2 | Epoch: 17 | loss=10.562807083129883 | Batch Time=13.28125 +GPU:2 | Epoch: 17 | loss=10.69652271270752 | Batch Time=15.625 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 17 | loss=10.707609176635742 | Batch Time=12.5 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=102847975424, percent=60.1, used=146807185408, free=17088552960, active=184842293248, inactive=50310303744, buffers=408420352, cached=93263925248, shared=5661655040, slab=3159580672) +AFTER TRAIN LOOP: Epoch 17 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +BEFORE VAL LOOP: GPU: 0 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +GPU:3 | Epoch: 17 | loss=10.79414176940918 | Batch Time=14.0625 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +AFTER TRAIN LOOP: Epoch 17 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +STARTING TRAINING: Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +BEFORE TRAIN LOOP: Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +GPU:2 | Epoch: 17 | loss=10.846989631652832 | Batch Time=8.984375 +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +AFTER TRAIN LOOP: Epoch 17 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +STARTING TRAINING: Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +BEFORE TRAIN LOOP: Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +GPU:1 | Epoch: 17 | loss=10.492459297180176 | Batch Time=17.1875 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +AFTER TRAIN LOOP: Epoch 17 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +STARTING TRAINING: Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +BEFORE TRAIN LOOP: Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=102848000000, percent=60.1, used=146807304192, free=17088577536, active=184842153984, inactive=50310242304, buffers=408420352, cached=93263781888, shared=5661511680, slab=3159556096) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 17 | loss=3.548027276992798 | Batch Time=25.0 +GPU:0 | Epoch: 17 | loss=4.7517170906066895 | Batch Time=10.546875 +GPU:0 | Epoch: 17 | loss=4.187586307525635 | Batch Time=18.75 +GPU:0 | Epoch: 17 | loss=3.9650630950927734 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=4.357652187347412 | Batch Time=7.03125 +GPU:0 | Epoch: 17 | loss=4.093540191650391 | Batch Time=10.15625 +GPU:0 | Epoch: 17 | loss=4.991823673248291 | Batch Time=6.25 +GPU:0 | Epoch: 17 | loss=4.689149379730225 | Batch Time=6.640625 +GPU:0 | Epoch: 17 | loss=5.334346294403076 | Batch Time=5.46875 +GPU:0 | Epoch: 17 | loss=6.12536096572876 | Batch Time=1.953125 +GPU:0 | Epoch: 17 | loss=5.776645660400391 | Batch Time=0.78125 +GPU:0 | Epoch: 17 | loss=4.828537464141846 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=5.623818397521973 | Batch Time=5.859375 +GPU:0 | Epoch: 17 | loss=4.585191249847412 | Batch Time=15.625 +GPU:0 | Epoch: 17 | loss=5.174492359161377 | Batch Time=4.296875 +GPU:0 | Epoch: 17 | loss=5.464189529418945 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=4.603853225708008 | Batch Time=16.015625 +GPU:0 | Epoch: 17 | loss=3.5295541286468506 | Batch Time=29.296875 +GPU:0 | Epoch: 17 | loss=4.264293670654297 | Batch Time=18.359375 +GPU:0 | Epoch: 17 | loss=3.2476048469543457 | Batch Time=30.078125 +Model top1 Accuracy: 13.43 +Acc before rounding: 13.43 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 17 | loss=3.548027276992798 | Batch Time=25.0 +GPU:0 | Epoch: 17 | loss=4.7517170906066895 | Batch Time=10.546875 +GPU:0 | Epoch: 17 | loss=4.187586307525635 | Batch Time=18.75 +GPU:0 | Epoch: 17 | loss=3.9650630950927734 | Batch Time=14.0625 +GPU:0 | Epoch: 17 | loss=4.357652187347412 | Batch Time=7.03125 +GPU:0 | Epoch: 17 | loss=4.093540191650391 | Batch Time=10.15625 +GPU:0 | Epoch: 17 | loss=4.991823673248291 | Batch Time=6.25 +GPU:0 | Epoch: 17 | loss=4.689149379730225 | Batch Time=6.640625 +GPU:0 | Epoch: 17 | loss=5.334346294403076 | Batch Time=5.46875 +GPU:0 | Epoch: 17 | loss=6.12536096572876 | Batch Time=1.953125 +GPU:0 | Epoch: 17 | loss=5.776645660400391 | Batch Time=0.78125 +GPU:0 | Epoch: 17 | loss=4.828537464141846 | Batch Time=13.28125 +GPU:0 | Epoch: 17 | loss=5.623818397521973 | Batch Time=5.859375 +GPU:0 | Epoch: 17 | loss=4.585191249847412 | Batch Time=15.625 +GPU:0 | Epoch: 17 | loss=5.174492359161377 | Batch Time=4.296875 +GPU:0 | Epoch: 17 | loss=5.464189529418945 | Batch Time=12.109375 +GPU:0 | Epoch: 17 | loss=4.603853225708008 | Batch Time=16.015625 +GPU:0 | Epoch: 17 | loss=3.5295541286468506 | Batch Time=29.296875 +GPU:0 | Epoch: 17 | loss=4.264293670654297 | Batch Time=18.359375 +GPU:0 | Epoch: 17 | loss=3.2476048469543457 | Batch Time=30.078125 +Model top1 Accuracy: 13.43 +Acc after rounding: 13.43 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 17 | loss=4.89275598526001 | Batch Time=11.71875 +GPU:0 | Epoch: 17 | loss=4.955215930938721 | Batch Time=10.15625 +GPU:0 | Epoch: 17 | loss=4.730183124542236 | Batch Time=15.234375 +GPU:0 | Epoch: 17 | loss=4.885677814483643 | Batch Time=9.375 +Model top1 Accuracy: 11.78 +Validation Acc after rounding: 11.78 +AFTER VAL LOOP: GPU: 0 | Epoch 17 | Memory Usage: svmem(total=257568083968, available=85032898560, percent=67.0, used=164622028800, free=5241155584, active=208915963904, inactive=37624520704, buffers=988524544, cached=86716375040, shared=5661802496, slab=3510804480) +Rounding model with scheme: naive +Model avg sparsity: 40.421934250420165 +GPU:0 | Epoch: 17 | Acc=13.43 | Epoch Time=17.505802953243254 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=85033156608, percent=67.0, used=164621852672, free=5241413632, active=208915898368, inactive=37624487936, buffers=988524544, cached=86716293120, shared=5661720576, slab=3510804480) +STARTING TRAINING: Epoch 18 | Memory Usage: svmem(total=257568083968, available=85033156608, percent=67.0, used=164621852672, free=5241413632, active=208915898368, inactive=37624487936, buffers=988524544, cached=86716293120, shared=5661720576, slab=3510804480) +BEFORE TRAIN LOOP: Epoch 18 | Memory Usage: svmem(total=257568083968, available=85033156608, percent=67.0, used=164621852672, free=5241413632, active=208915898368, inactive=37624487936, buffers=988524544, cached=86716293120, shared=5661720576, slab=3510804480) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=85033156608, percent=67.0, used=164621852672, free=5241413632, active=208915898368, inactive=37624487936, buffers=988524544, cached=86716293120, shared=5661720576, slab=3510804480) +GPU:1 | Epoch: 18 | loss=10.685943603515625 | Batch Time=12.109375 +GPU:1 | Epoch: 18 | loss=10.697699546813965 | Batch Time=12.5 +GPU:1 | Epoch: 18 | loss=10.730705261230469 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.56944465637207 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.8456449508667 | Batch Time=10.15625 +GPU:1 | Epoch: 18 | loss=10.686307907104492 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.764951705932617 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.727836608886719 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.670522689819336 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.651325225830078 | Batch Time=10.15625 +GPU:1 | Epoch: 18 | loss=10.529773712158203 | Batch Time=19.53125 +GPU:1 | Epoch: 18 | loss=10.833841323852539 | Batch Time=6.640625 +GPU:1 | Epoch: 18 | loss=10.412755966186523 | Batch Time=16.015625 +GPU:1 | Epoch: 18 | loss=10.477834701538086 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.669696807861328 | Batch Time=11.71875 +GPU:1 | Epoch: 18 | loss=10.756715774536133 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.74718189239502 | Batch Time=14.0625 +GPU:1 | Epoch: 18 | loss=10.465988159179688 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.470941543579102 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.615196228027344 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.751928329467773 | Batch Time=12.5 +GPU:1 | Epoch: 18 | loss=10.615457534790039 | Batch Time=12.890625 +GPU:1 | Epoch: 18 | loss=10.618284225463867 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.664337158203125 | Batch Time=17.1875 +GPU:1 | Epoch: 18 | loss=10.646993637084961 | Batch Time=11.328125 +GPU:1 | Epoch: 18 | loss=10.599672317504883 | Batch Time=12.890625 +GPU:1 | Epoch: 18 | loss=10.528270721435547 | Batch Time=17.578125 +GPU:1 | Epoch: 18 | loss=10.564879417419434 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.498137474060059 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.819378852844238 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.578448295593262 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.703042984008789 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.617086410522461 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.750072479248047 | Batch Time=9.765625 +GPU:1 | Epoch: 18 | loss=10.565767288208008 | Batch Time=11.328125 +GPU:1 | Epoch: 18 | loss=10.559730529785156 | Batch Time=16.015625 +GPU:1 | Epoch: 18 | loss=10.714824676513672 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.679915428161621 | Batch Time=15.625 +GPU:1 | Epoch: 18 | loss=10.734893798828125 | Batch Time=12.890625 +GPU:1 | Epoch: 18 | loss=10.594629287719727 | Batch Time=10.9375 +GPU:1 | Epoch: 18 | loss=10.585102081298828 | Batch Time=12.890625 +GPU:1 | Epoch: 18 | loss=10.589221000671387 | Batch Time=16.40625 +GPU:1 | Epoch: 18 | loss=10.4122896194458 | Batch Time=15.625 +GPU:1 | Epoch: 18 | loss=10.688882827758789 | Batch Time=12.5 +GPU:1 | Epoch: 18 | loss=10.681251525878906 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.80480670928955 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.600542068481445 | Batch Time=12.5 +GPU:1 | Epoch: 18 | loss=10.70069694519043 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.555532455444336 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.621923446655273 | Batch Time=12.109375 +GPU:1 | Epoch: 18 | loss=10.448165893554688 | Batch Time=16.015625 +GPU:1 | Epoch: 18 | loss=10.806453704833984 | Batch Time=12.5 +GPU:1 | Epoch: 18 | loss=10.478530883789062 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.489532470703125 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.724958419799805 | Batch Time=11.71875 +GPU:1 | Epoch: 18 | loss=10.81633186340332 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.607807159423828 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.52717399597168 | Batch Time=17.578125 +GPU:1 | Epoch: 18 | loss=10.618332862854004 | Batch Time=14.0625 +GPU:1 | Epoch: 18 | loss=10.45334243774414 | Batch Time=19.140625 +GPU:1 | Epoch: 18 | loss=10.485360145568848 | Batch Time=20.3125 +GPU:1 | Epoch: 18 | loss=10.612325668334961 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.584341049194336 | Batch Time=11.71875 +GPU:1 | Epoch: 18 | loss=10.680097579956055 | Batch Time=12.890625 +GPU:1 | Epoch: 18 | loss=10.651851654052734 | Batch Time=14.0625 +GPU:1 | Epoch: 18 | loss=10.502134323120117 | Batch Time=18.359375 +GPU:1 | Epoch: 18 | loss=10.458366394042969 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.639534950256348 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.316408157348633 | Batch Time=17.1875 +GPU:1 | Epoch: 18 | loss=10.498319625854492 | Batch Time=11.71875 +GPU:1 | Epoch: 18 | loss=10.634420394897461 | Batch Time=12.109375 +GPU:1 | Epoch: 18 | loss=10.589045524597168 | Batch Time=12.5 +GPU:1 | Epoch: 18 | loss=10.58609390258789 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.545221328735352 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.452720642089844 | Batch Time=16.015625 +GPU:1 | Epoch: 18 | loss=10.297679901123047 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.70026683807373 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.58664608001709 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.513202667236328 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.67111587524414 | Batch Time=10.9375 +GPU:1 | Epoch: 18 | loss=10.575787544250488 | Batch Time=12.109375 +GPU:1 | Epoch: 18 | loss=10.479814529418945 | Batch Time=17.578125 +GPU:1 | Epoch: 18 | loss=10.439346313476562 | Batch Time=17.578125 +GPU:1 | Epoch: 18 | loss=10.687965393066406 | Batch Time=19.53125 +GPU:1 | Epoch: 18 | loss=10.500732421875 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.53405475616455 | Batch Time=15.625 +GPU:1 | Epoch: 18 | loss=10.387090682983398 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.605103492736816 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.558063507080078 | Batch Time=16.796875 +GPU:1 | Epoch: 18 | loss=10.437976837158203 | Batch Time=16.40625 +GPU:1 | Epoch: 18 | loss=10.452566146850586 | Batch Time=16.40625 +GPU:1 | Epoch: 18 | loss=10.53813362121582 | Batch Time=16.015625 +GPU:1 | Epoch: 18 | loss=10.508986473083496 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.406204223632812 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.497769355773926 | Batch Time=16.40625 +GPU:1 | Epoch: 18 | loss=10.365840911865234 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.496272087097168 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.492910385131836 | Batch Time=16.796875 +GPU:1 | Epoch: 18 | loss=10.512214660644531 | Batch Time=17.1875 +GPU:1 | Epoch: 18 | loss=10.437788009643555 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.553783416748047 | Batch Time=10.546875 +GPU:1 | Epoch: 18 | loss=10.480449676513672 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.430604934692383 | Batch Time=16.015625 +GPU:1 | Epoch: 18 | loss=10.563756942749023 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.518239974975586 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.37955379486084 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.64968490600586 | Batch Time=13.671875 +GPU:1 | Epoch: 18 | loss=10.520584106445312 | Batch Time=15.625 +GPU:1 | Epoch: 18 | loss=10.490373611450195 | Batch Time=15.625 +GPU:1 | Epoch: 18 | loss=10.738995552062988 | Batch Time=12.109375 +GPU:1 | Epoch: 18 | loss=10.462024688720703 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.781240463256836 | Batch Time=10.15625 +GPU:1 | Epoch: 18 | loss=10.533395767211914 | Batch Time=13.28125 +GPU:1 | Epoch: 18 | loss=10.259590148925781 | Batch Time=16.40625 +GPU:1 | Epoch: 18 | loss=10.486963272094727 | Batch Time=16.796875 +GPU:1 | Epoch: 18 | loss=10.625335693359375 | Batch Time=11.328125 +GPU:1 | Epoch: 18 | loss=10.470497131347656 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.411705017089844 | Batch Time=14.453125 +GPU:1 | Epoch: 18 | loss=10.392865180969238 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.647640228271484 | Batch Time=11.71875 +GPU:1 | Epoch: 18 | loss=10.534236907958984 | Batch Time=15.234375 +GPU:1 | Epoch: 18 | loss=10.48630142211914 | Batch Time=14.84375 +GPU:1 | Epoch: 18 | loss=10.392088890075684 | Batch Time=16.015625 +GPU:1 | Epoch: 18 | loss=10.433156967163086 | Batch Time=12.5 +GPU:0 | Epoch: 18 | loss=10.614985466003418 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.664570808410645 | Batch Time=16.40625 +GPU:0 | Epoch: 18 | loss=10.65837287902832 | Batch Time=13.671875 +GPU:0 | Epoch: 18 | loss=10.947837829589844 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.654619216918945 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.650867462158203 | Batch Time=10.546875 +GPU:0 | Epoch: 18 | loss=10.679285049438477 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.631048202514648 | Batch Time=13.671875 +GPU:0 | Epoch: 18 | loss=10.54456901550293 | Batch Time=14.84375 +GPU:0 | Epoch: 18 | loss=10.566362380981445 | Batch Time=17.1875 +GPU:0 | Epoch: 18 | loss=10.68996810913086 | Batch Time=10.15625 +GPU:0 | Epoch: 18 | loss=10.870901107788086 | Batch Time=16.015625 +GPU:0 | Epoch: 18 | loss=10.611979484558105 | Batch Time=14.0625 +GPU:0 | Epoch: 18 | loss=10.6671142578125 | Batch Time=17.1875 +GPU:0 | Epoch: 18 | loss=10.621368408203125 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.664546966552734 | Batch Time=11.71875 +GPU:0 | Epoch: 18 | loss=10.590642929077148 | Batch Time=16.40625 +GPU:0 | Epoch: 18 | loss=10.377386093139648 | Batch Time=17.96875 +GPU:0 | Epoch: 18 | loss=10.721410751342773 | Batch Time=12.109375 +GPU:0 | Epoch: 18 | loss=10.524333953857422 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.593894958496094 | Batch Time=12.5 +GPU:0 | Epoch: 18 | loss=10.727873802185059 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.602581977844238 | Batch Time=13.671875 +GPU:0 | Epoch: 18 | loss=10.621816635131836 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.714139938354492 | Batch Time=9.375 +GPU:0 | Epoch: 18 | loss=10.724599838256836 | Batch Time=13.671875 +GPU:0 | Epoch: 18 | loss=10.665216445922852 | Batch Time=10.15625 +GPU:0 | Epoch: 18 | loss=10.2534761428833 | Batch Time=16.015625 +GPU:0 | Epoch: 18 | loss=10.772077560424805 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.638799667358398 | Batch Time=16.015625 +GPU:0 | Epoch: 18 | loss=10.460616111755371 | Batch Time=16.40625 +GPU:0 | Epoch: 18 | loss=10.652015686035156 | Batch Time=14.0625 +GPU:0 | Epoch: 18 | loss=10.697393417358398 | Batch Time=15.234375 +GPU:0 | Epoch: 18 | loss=10.643238067626953 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.703275680541992 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.930618286132812 | Batch Time=11.328125 +GPU:0 | Epoch: 18 | loss=10.630630493164062 | Batch Time=14.84375 +GPU:0 | Epoch: 18 | loss=10.65022087097168 | Batch Time=14.0625 +GPU:0 | Epoch: 18 | loss=10.691953659057617 | Batch Time=10.15625 +GPU:0 | Epoch: 18 | loss=10.644523620605469 | Batch Time=11.328125 +GPU:0 | Epoch: 18 | loss=10.818774223327637 | Batch Time=12.5 +GPU:0 | Epoch: 18 | loss=10.633052825927734 | Batch Time=16.40625 +GPU:0 | Epoch: 18 | loss=10.652471542358398 | Batch Time=14.84375 +GPU:0 | Epoch: 18 | loss=10.63174057006836 | Batch Time=13.671875 +GPU:0 | Epoch: 18 | loss=10.564403533935547 | Batch Time=17.578125 +GPU:0 | Epoch: 18 | loss=10.508888244628906 | Batch Time=11.328125 +GPU:0 | Epoch: 18 | loss=10.576634407043457 | Batch Time=17.578125 +GPU:0 | Epoch: 18 | loss=10.555622100830078 | Batch Time=19.140625 +GPU:0 | Epoch: 18 | loss=10.65798282623291 | Batch Time=15.234375 +GPU:0 | Epoch: 18 | loss=10.693890571594238 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.609840393066406 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.509622573852539 | Batch Time=14.0625 +GPU:0 | Epoch: 18 | loss=10.487909317016602 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.518125534057617 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.643543243408203 | Batch Time=10.9375 +GPU:0 | Epoch: 18 | loss=10.664510726928711 | Batch Time=15.625 +GPU:0 | Epoch: 18 | loss=10.478585243225098 | Batch Time=17.578125 +GPU:0 | Epoch: 18 | loss=10.430797576904297 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.591598510742188 | Batch Time=19.140625 +GPU:0 | Epoch: 18 | loss=10.434688568115234 | Batch Time=17.578125 +GPU:0 | Epoch: 18 | loss=10.537015914916992 | Batch Time=15.625 +GPU:0 | Epoch: 18 | loss=10.402750015258789 | Batch Time=15.625 +GPU:0 | Epoch: 18 | loss=10.724482536315918 | Batch Time=12.109375 +GPU:0 | Epoch: 18 | loss=10.611749649047852 | Batch Time=12.5 +GPU:0 | Epoch: 18 | loss=10.643710136413574 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.464241027832031 | Batch Time=14.84375 +GPU:0 | Epoch: 18 | loss=10.31410026550293 | Batch Time=18.359375 +GPU:0 | Epoch: 18 | loss=10.429788589477539 | Batch Time=14.0625 +GPU:0 | Epoch: 18 | loss=10.53969955444336 | Batch Time=14.0625 +GPU:0 | Epoch: 18 | loss=10.51924991607666 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.516374588012695 | Batch Time=13.671875 +GPU:0 | Epoch: 18 | loss=10.526397705078125 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.645840644836426 | Batch Time=12.109375 +GPU:0 | Epoch: 18 | loss=10.74847412109375 | Batch Time=12.5 +GPU:0 | Epoch: 18 | loss=10.588020324707031 | Batch Time=17.578125 +GPU:0 | Epoch: 18 | loss=10.400076866149902 | Batch Time=17.1875 +GPU:0 | Epoch: 18 | loss=10.522729873657227 | Batch Time=19.921875 +GPU:0 | Epoch: 18 | loss=10.486211776733398 | Batch Time=16.796875 +GPU:0 | Epoch: 18 | loss=10.579721450805664 | Batch Time=15.234375 +GPU:0 | Epoch: 18 | loss=10.422904968261719 | Batch Time=15.625 +GPU:0 | Epoch: 18 | loss=10.572072982788086 | Batch Time=14.84375 +GPU:0 | Epoch: 18 | loss=10.490537643432617 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.764405250549316 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.574193954467773 | Batch Time=12.109375 +GPU:0 | Epoch: 18 | loss=10.603507995605469 | Batch Time=10.15625 +GPU:0 | Epoch: 18 | loss=10.320388793945312 | Batch Time=16.40625 +GPU:0 | Epoch: 18 | loss=10.542558670043945 | Batch Time=15.625 +GPU:0 | Epoch: 18 | loss=10.606208801269531 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.420011520385742 | Batch Time=14.84375 +GPU:0 | Epoch: 18 | loss=10.63868236541748 | Batch Time=12.109375 +GPU:0 | Epoch: 18 | loss=10.660163879394531 | Batch Time=16.015625 +GPU:0 | Epoch: 18 | loss=10.606637954711914 | Batch Time=12.5 +GPU:0 | Epoch: 18 | loss=10.486141204833984 | Batch Time=15.234375 +GPU:0 | Epoch: 18 | loss=10.425264358520508 | Batch Time=11.328125 +GPU:0 | Epoch: 18 | loss=10.657469749450684 | Batch Time=11.328125 +GPU:0 | Epoch: 18 | loss=10.559114456176758 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.574267387390137 | Batch Time=11.71875 +GPU:0 | Epoch: 18 | loss=10.373800277709961 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.609148025512695 | Batch Time=16.40625 +GPU:0 | Epoch: 18 | loss=10.506232261657715 | Batch Time=14.0625 +GPU:0 | Epoch: 18 | loss=10.59981632232666 | Batch Time=10.9375 +GPU:0 | Epoch: 18 | loss=10.604728698730469 | Batch Time=12.5 +GPU:0 | Epoch: 18 | loss=10.481609344482422 | Batch Time=17.1875 +GPU:0 | Epoch: 18 | loss=10.334127426147461 | Batch Time=14.84375 +GPU:0 | Epoch: 18 | loss=10.580108642578125 | Batch Time=14.453125 +GPU:0 | Epoch: 18 | loss=10.543946266174316 | Batch Time=17.578125 +GPU:0 | Epoch: 18 | loss=10.31984806060791 | Batch Time=16.796875 +GPU:0 | Epoch: 18 | loss=10.409311294555664 | Batch Time=18.75 +GPU:0 | Epoch: 18 | loss=10.520410537719727 | Batch Time=11.328125 +GPU:0 | Epoch: 18 | loss=10.614696502685547 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.392019271850586 | Batch Time=16.796875 +GPU:0 | Epoch: 18 | loss=10.465746879577637 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.379707336425781 | Batch Time=16.40625 +GPU:0 | Epoch: 18 | loss=10.511734008789062 | Batch Time=11.71875 +GPU:0 | Epoch: 18 | loss=10.59503173828125 | Batch Time=15.625 +GPU:0 | Epoch: 18 | loss=10.29764461517334 | Batch Time=15.234375 +GPU:0 | Epoch: 18 | loss=10.645682334899902 | Batch Time=17.1875 +GPU:0 | Epoch: 18 | loss=10.42029094696045 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.482320785522461 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.454463958740234 | Batch Time=13.671875 +GPU:0 | Epoch: 18 | loss=10.485546112060547 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=10.431843757629395 | Batch Time=12.890625 +GPU:0 | Epoch: 18 | loss=10.509061813354492 | Batch Time=15.234375 +GPU:0 | Epoch: 18 | loss=10.5867919921875 | Batch Time=10.9375 +GPU:3 | Epoch: 18 | loss=10.768274307250977 | Batch Time=9.765625 +GPU:3 | Epoch: 18 | loss=10.67175579071045 | Batch Time=12.109375 +GPU:3 | Epoch: 18 | loss=10.554143905639648 | Batch Time=17.1875 +GPU:3 | Epoch: 18 | loss=10.775701522827148 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.700311660766602 | Batch Time=15.234375 +GPU:3 | Epoch: 18 | loss=10.752391815185547 | Batch Time=13.671875 +GPU:3 | Epoch: 18 | loss=10.673173904418945 | Batch Time=12.5 +GPU:3 | Epoch: 18 | loss=10.74560832977295 | Batch Time=14.84375 +GPU:3 | Epoch: 18 | loss=10.669769287109375 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.483835220336914 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.781291961669922 | Batch Time=16.015625 +GPU:3 | Epoch: 18 | loss=10.373733520507812 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.39836311340332 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.650468826293945 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.748870849609375 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.597058296203613 | Batch Time=11.328125 +GPU:3 | Epoch: 18 | loss=10.697990417480469 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.753960609436035 | Batch Time=12.5 +GPU:3 | Epoch: 18 | loss=10.679542541503906 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.44967269897461 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.537249565124512 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.705339431762695 | Batch Time=13.671875 +GPU:3 | Epoch: 18 | loss=10.75794506072998 | Batch Time=11.328125 +GPU:3 | Epoch: 18 | loss=10.783729553222656 | Batch Time=15.234375 +GPU:3 | Epoch: 18 | loss=10.625551223754883 | Batch Time=14.84375 +GPU:3 | Epoch: 18 | loss=10.926124572753906 | Batch Time=10.15625 +GPU:3 | Epoch: 18 | loss=10.683070182800293 | Batch Time=14.453125 +GPU:3 | Epoch: 18 | loss=10.499469757080078 | Batch Time=15.234375 +GPU:3 | Epoch: 18 | loss=10.647372245788574 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.769571304321289 | Batch Time=11.71875 +GPU:3 | Epoch: 18 | loss=10.470468521118164 | Batch Time=11.71875 +GPU:3 | Epoch: 18 | loss=10.55642318725586 | Batch Time=14.84375 +GPU:3 | Epoch: 18 | loss=10.401838302612305 | Batch Time=17.1875 +GPU:3 | Epoch: 18 | loss=10.567241668701172 | Batch Time=16.015625 +GPU:3 | Epoch: 18 | loss=10.500236511230469 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.545570373535156 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.67602825164795 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.595572471618652 | Batch Time=14.453125 +GPU:3 | Epoch: 18 | loss=10.45965576171875 | Batch Time=16.40625 +GPU:3 | Epoch: 18 | loss=10.751008987426758 | Batch Time=11.71875 +GPU:3 | Epoch: 18 | loss=10.692339897155762 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.600027084350586 | Batch Time=16.015625 +GPU:3 | Epoch: 18 | loss=10.474172592163086 | Batch Time=13.671875 +GPU:3 | Epoch: 18 | loss=10.778059959411621 | Batch Time=8.984375 +GPU:3 | Epoch: 18 | loss=10.644147872924805 | Batch Time=13.671875 +GPU:3 | Epoch: 18 | loss=10.596488952636719 | Batch Time=10.546875 +GPU:3 | Epoch: 18 | loss=10.726323127746582 | Batch Time=13.671875 +GPU:3 | Epoch: 18 | loss=10.509902000427246 | Batch Time=16.40625 +GPU:3 | Epoch: 18 | loss=10.752510070800781 | Batch Time=9.375 +GPU:3 | Epoch: 18 | loss=10.698963165283203 | Batch Time=12.5 +GPU:3 | Epoch: 18 | loss=10.63543701171875 | Batch Time=16.015625 +GPU:3 | Epoch: 18 | loss=10.507804870605469 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.690483093261719 | Batch Time=11.328125 +GPU:3 | Epoch: 18 | loss=10.665319442749023 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.525724411010742 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.32838249206543 | Batch Time=17.578125 +GPU:3 | Epoch: 18 | loss=10.466771125793457 | Batch Time=14.84375 +GPU:3 | Epoch: 18 | loss=10.568002700805664 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.453022003173828 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.464573860168457 | Batch Time=17.96875 +GPU:3 | Epoch: 18 | loss=10.629383087158203 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.479585647583008 | Batch Time=16.796875 +GPU:3 | Epoch: 18 | loss=10.689523696899414 | Batch Time=11.328125 +GPU:3 | Epoch: 18 | loss=10.437864303588867 | Batch Time=14.453125 +GPU:3 | Epoch: 18 | loss=10.736417770385742 | Batch Time=10.9375 +GPU:3 | Epoch: 18 | loss=10.650644302368164 | Batch Time=11.71875 +GPU:3 | Epoch: 18 | loss=10.688529968261719 | Batch Time=12.5 +GPU:3 | Epoch: 18 | loss=10.449539184570312 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.4052734375 | Batch Time=17.1875 +GPU:3 | Epoch: 18 | loss=10.667800903320312 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.548762321472168 | Batch Time=14.453125 +GPU:3 | Epoch: 18 | loss=10.428722381591797 | Batch Time=16.796875 +GPU:3 | Epoch: 18 | loss=10.59457778930664 | Batch Time=13.671875 +GPU:3 | Epoch: 18 | loss=10.588623046875 | Batch Time=10.9375 +GPU:3 | Epoch: 18 | loss=10.753409385681152 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.403776168823242 | Batch Time=17.1875 +GPU:3 | Epoch: 18 | loss=10.699920654296875 | Batch Time=11.71875 +GPU:3 | Epoch: 18 | loss=10.57480239868164 | Batch Time=11.328125 +GPU:3 | Epoch: 18 | loss=10.501274108886719 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.61989974975586 | Batch Time=17.96875 +GPU:3 | Epoch: 18 | loss=10.503072738647461 | Batch Time=16.796875 +GPU:3 | Epoch: 18 | loss=10.593036651611328 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.616899490356445 | Batch Time=11.71875 +GPU:3 | Epoch: 18 | loss=10.518230438232422 | Batch Time=14.84375 +GPU:3 | Epoch: 18 | loss=10.470394134521484 | Batch Time=18.359375 +GPU:3 | Epoch: 18 | loss=10.589427947998047 | Batch Time=12.5 +GPU:3 | Epoch: 18 | loss=10.508123397827148 | Batch Time=14.84375 +GPU:3 | Epoch: 18 | loss=10.37472915649414 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.609996795654297 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.558294296264648 | Batch Time=17.96875 +GPU:3 | Epoch: 18 | loss=10.644937515258789 | Batch Time=12.5 +GPU:3 | Epoch: 18 | loss=10.448543548583984 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.403182983398438 | Batch Time=14.84375 +GPU:3 | Epoch: 18 | loss=10.474817276000977 | Batch Time=15.234375 +GPU:3 | Epoch: 18 | loss=10.540532112121582 | Batch Time=12.109375 +GPU:3 | Epoch: 18 | loss=10.489832878112793 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.448996543884277 | Batch Time=16.015625 +GPU:3 | Epoch: 18 | loss=10.447236061096191 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.67249870300293 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.520345687866211 | Batch Time=11.71875 +GPU:3 | Epoch: 18 | loss=10.487208366394043 | Batch Time=14.84375 +GPU:3 | Epoch: 18 | loss=10.32126522064209 | Batch Time=15.234375 +GPU:3 | Epoch: 18 | loss=10.327327728271484 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.55476188659668 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.64075756072998 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.776946067810059 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.52823543548584 | Batch Time=12.109375 +GPU:3 | Epoch: 18 | loss=10.611605644226074 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.509602546691895 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.454879760742188 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.553060531616211 | Batch Time=11.71875 +GPU:3 | Epoch: 18 | loss=10.446967124938965 | Batch Time=14.0625 +GPU:3 | Epoch: 18 | loss=10.768478393554688 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.48748779296875 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.677362442016602 | Batch Time=13.28125 +GPU:3 | Epoch: 18 | loss=10.66245174407959 | Batch Time=12.109375 +GPU:3 | Epoch: 18 | loss=10.527730941772461 | Batch Time=12.890625 +GPU:3 | Epoch: 18 | loss=10.46547794342041 | Batch Time=16.015625 +GPU:3 | Epoch: 18 | loss=10.370901107788086 | Batch Time=18.359375 +GPU:3 | Epoch: 18 | loss=10.531503677368164 | Batch Time=13.671875 +GPU:3 | Epoch: 18 | loss=10.409022331237793 | Batch Time=17.96875 +GPU:3 | Epoch: 18 | loss=10.332128524780273 | Batch Time=12.109375 +GPU:3 | Epoch: 18 | loss=10.374197006225586 | Batch Time=15.625 +GPU:3 | Epoch: 18 | loss=10.413490295410156 | Batch Time=17.1875 +GPU:2 | Epoch: 18 | loss=10.562860488891602 | Batch Time=17.578125 +GPU:2 | Epoch: 18 | loss=10.687936782836914 | Batch Time=12.890625 +GPU:2 | Epoch: 18 | loss=10.678037643432617 | Batch Time=14.84375 +GPU:2 | Epoch: 18 | loss=10.752660751342773 | Batch Time=11.71875 +GPU:2 | Epoch: 18 | loss=10.625253677368164 | Batch Time=11.71875 +GPU:2 | Epoch: 18 | loss=10.661436080932617 | Batch Time=17.578125 +GPU:2 | Epoch: 18 | loss=10.607479095458984 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.507335662841797 | Batch Time=17.1875 +GPU:2 | Epoch: 18 | loss=10.583670616149902 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.807075500488281 | Batch Time=12.109375 +GPU:2 | Epoch: 18 | loss=10.748580932617188 | Batch Time=12.5 +GPU:2 | Epoch: 18 | loss=10.610299110412598 | Batch Time=15.625 +GPU:2 | Epoch: 18 | loss=10.486244201660156 | Batch Time=14.84375 +GPU:2 | Epoch: 18 | loss=10.580915451049805 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.757087707519531 | Batch Time=12.109375 +GPU:2 | Epoch: 18 | loss=10.440988540649414 | Batch Time=18.75 +GPU:2 | Epoch: 18 | loss=10.76223087310791 | Batch Time=12.5 +GPU:2 | Epoch: 18 | loss=10.469929695129395 | Batch Time=17.1875 +GPU:2 | Epoch: 18 | loss=10.778532981872559 | Batch Time=12.5 +GPU:2 | Epoch: 18 | loss=10.68580150604248 | Batch Time=10.15625 +GPU:2 | Epoch: 18 | loss=10.638824462890625 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.667993545532227 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.5236177444458 | Batch Time=15.625 +GPU:2 | Epoch: 18 | loss=10.498906135559082 | Batch Time=12.890625 +GPU:2 | Epoch: 18 | loss=10.663900375366211 | Batch Time=12.109375 +GPU:2 | Epoch: 18 | loss=10.822406768798828 | Batch Time=10.546875 +GPU:2 | Epoch: 18 | loss=10.629707336425781 | Batch Time=12.109375 +GPU:2 | Epoch: 18 | loss=10.532116889953613 | Batch Time=17.1875 +GPU:2 | Epoch: 18 | loss=10.652170181274414 | Batch Time=14.84375 +GPU:2 | Epoch: 18 | loss=10.578521728515625 | Batch Time=15.234375 +GPU:2 | Epoch: 18 | loss=10.507057189941406 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.648307800292969 | Batch Time=13.671875 +GPU:2 | Epoch: 18 | loss=10.523958206176758 | Batch Time=16.015625 +GPU:2 | Epoch: 18 | loss=10.448298454284668 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.615819931030273 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.576934814453125 | Batch Time=15.625 +GPU:2 | Epoch: 18 | loss=10.529909133911133 | Batch Time=14.84375 +GPU:2 | Epoch: 18 | loss=10.646345138549805 | Batch Time=11.71875 +GPU:2 | Epoch: 18 | loss=10.747406005859375 | Batch Time=11.71875 +GPU:2 | Epoch: 18 | loss=10.632959365844727 | Batch Time=13.671875 +GPU:2 | Epoch: 18 | loss=10.716085433959961 | Batch Time=11.328125 +GPU:2 | Epoch: 18 | loss=10.66596794128418 | Batch Time=15.625 +GPU:2 | Epoch: 18 | loss=10.551021575927734 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.659884452819824 | Batch Time=14.84375 +GPU:2 | Epoch: 18 | loss=10.477540969848633 | Batch Time=15.234375 +GPU:2 | Epoch: 18 | loss=10.735189437866211 | Batch Time=12.109375 +GPU:2 | Epoch: 18 | loss=10.653854370117188 | Batch Time=12.890625 +GPU:2 | Epoch: 18 | loss=10.646554946899414 | Batch Time=10.546875 +GPU:2 | Epoch: 18 | loss=10.624205589294434 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.415987014770508 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.6083984375 | Batch Time=15.625 +GPU:2 | Epoch: 18 | loss=10.553714752197266 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.609618186950684 | Batch Time=12.890625 +GPU:2 | Epoch: 18 | loss=10.425453186035156 | Batch Time=15.234375 +GPU:2 | Epoch: 18 | loss=10.702247619628906 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.647298812866211 | Batch Time=15.625 +GPU:2 | Epoch: 18 | loss=10.523094177246094 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.535503387451172 | Batch Time=14.84375 +GPU:2 | Epoch: 18 | loss=10.747941017150879 | Batch Time=10.15625 +GPU:2 | Epoch: 18 | loss=10.623445510864258 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.401473045349121 | Batch Time=18.359375 +GPU:2 | Epoch: 18 | loss=10.63856315612793 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.605438232421875 | Batch Time=11.71875 +GPU:2 | Epoch: 18 | loss=10.501686096191406 | Batch Time=14.84375 +GPU:2 | Epoch: 18 | loss=10.62921142578125 | Batch Time=11.328125 +GPU:2 | Epoch: 18 | loss=10.643525123596191 | Batch Time=12.109375 +GPU:2 | Epoch: 18 | loss=10.611579895019531 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.768306732177734 | Batch Time=12.5 +GPU:2 | Epoch: 18 | loss=10.47830867767334 | Batch Time=16.015625 +GPU:2 | Epoch: 18 | loss=10.560079574584961 | Batch Time=12.109375 +GPU:2 | Epoch: 18 | loss=10.456969261169434 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.67997932434082 | Batch Time=13.671875 +GPU:2 | Epoch: 18 | loss=10.603265762329102 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.656682014465332 | Batch Time=12.890625 +GPU:2 | Epoch: 18 | loss=10.56570816040039 | Batch Time=11.328125 +GPU:2 | Epoch: 18 | loss=10.687641143798828 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.468530654907227 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.642504692077637 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.480358123779297 | Batch Time=15.234375 +GPU:2 | Epoch: 18 | loss=10.394355773925781 | Batch Time=19.921875 +GPU:2 | Epoch: 18 | loss=10.747737884521484 | Batch Time=11.328125 +GPU:2 | Epoch: 18 | loss=10.517026901245117 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.563902854919434 | Batch Time=13.671875 +GPU:2 | Epoch: 18 | loss=10.38554573059082 | Batch Time=17.1875 +GPU:2 | Epoch: 18 | loss=10.426864624023438 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.75712776184082 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.647415161132812 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.322393417358398 | Batch Time=17.1875 +GPU:2 | Epoch: 18 | loss=10.5435791015625 | Batch Time=13.671875 +GPU:2 | Epoch: 18 | loss=10.637811660766602 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.391353607177734 | Batch Time=17.578125 +GPU:2 | Epoch: 18 | loss=10.503966331481934 | Batch Time=11.328125 +GPU:2 | Epoch: 18 | loss=10.465984344482422 | Batch Time=17.96875 +GPU:2 | Epoch: 18 | loss=10.383710861206055 | Batch Time=12.5 +GPU:2 | Epoch: 18 | loss=10.428277969360352 | Batch Time=16.015625 +GPU:2 | Epoch: 18 | loss=10.685264587402344 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.514963150024414 | Batch Time=14.0625 +GPU:2 | Epoch: 18 | loss=10.414299011230469 | Batch Time=17.578125 +GPU:2 | Epoch: 18 | loss=10.31456184387207 | Batch Time=16.015625 +GPU:2 | Epoch: 18 | loss=10.741313934326172 | Batch Time=11.328125 +GPU:2 | Epoch: 18 | loss=10.424018859863281 | Batch Time=16.40625 +GPU:2 | Epoch: 18 | loss=10.24824333190918 | Batch Time=19.140625 +GPU:2 | Epoch: 18 | loss=10.399528503417969 | Batch Time=16.40625 +GPU:2 | Epoch: 18 | loss=10.503314018249512 | Batch Time=12.890625 +GPU:2 | Epoch: 18 | loss=10.412801742553711 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.493738174438477 | Batch Time=16.015625 +GPU:2 | Epoch: 18 | loss=10.491830825805664 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.331756591796875 | Batch Time=15.234375 +GPU:2 | Epoch: 18 | loss=10.422418594360352 | Batch Time=19.53125 +GPU:2 | Epoch: 18 | loss=10.397711753845215 | Batch Time=15.234375 +GPU:2 | Epoch: 18 | loss=10.598816871643066 | Batch Time=15.625 +GPU:2 | Epoch: 18 | loss=10.710747718811035 | Batch Time=12.890625 +GPU:2 | Epoch: 18 | loss=10.62066650390625 | Batch Time=15.234375 +GPU:2 | Epoch: 18 | loss=10.420967102050781 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.56011962890625 | Batch Time=15.234375 +GPU:2 | Epoch: 18 | loss=10.621231079101562 | Batch Time=13.28125 +GPU:2 | Epoch: 18 | loss=10.563299179077148 | Batch Time=10.9375 +GPU:2 | Epoch: 18 | loss=10.47657585144043 | Batch Time=16.796875 +GPU:2 | Epoch: 18 | loss=10.428183555603027 | Batch Time=15.625 +GPU:2 | Epoch: 18 | loss=10.41203784942627 | Batch Time=12.109375 +GPU:2 | Epoch: 18 | loss=10.577878952026367 | Batch Time=12.5 +GPU:2 | Epoch: 18 | loss=10.669706344604492 | Batch Time=12.5 +GPU:2 | Epoch: 18 | loss=10.533849716186523 | Batch Time=14.453125 +GPU:2 | Epoch: 18 | loss=10.439321517944336 | Batch Time=13.28125 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 18 | loss=10.542379379272461 | Batch Time=12.890625 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=95299743744, percent=63.0, used=154355277824, free=17261871104, active=185582379008, inactive=49277792256, buffers=757313536, cached=85193621504, shared=5661769728, slab=3205963776) +AFTER TRAIN LOOP: Epoch 18 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +STARTING TRAINING: Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +BEFORE TRAIN LOOP: Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +GPU:0 | Epoch: 18 | loss=10.494632720947266 | Batch Time=17.96875 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=95299743744, percent=63.0, used=154355277824, free=17261871104, active=185582379008, inactive=49277792256, buffers=757313536, cached=85193621504, shared=5661769728, slab=3205963776) +AFTER TRAIN LOOP: Epoch 18 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +BEFORE VAL LOOP: GPU: 0 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +GPU:3 | Epoch: 18 | loss=10.412740707397461 | Batch Time=15.625 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=95299743744, percent=63.0, used=154355277824, free=17261871104, active=185582379008, inactive=49277792256, buffers=757313536, cached=85193621504, shared=5661769728, slab=3205963776) +AFTER TRAIN LOOP: Epoch 18 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +STARTING TRAINING: Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +BEFORE TRAIN LOOP: Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +GPU:2 | Epoch: 18 | loss=10.516891479492188 | Batch Time=13.28125 +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=95299743744, percent=63.0, used=154355277824, free=17261871104, active=185582379008, inactive=49277792256, buffers=757313536, cached=85193621504, shared=5661769728, slab=3205963776) +AFTER TRAIN LOOP: Epoch 18 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +STARTING TRAINING: Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +BEFORE TRAIN LOOP: Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=95300542464, percent=63.0, used=154354561024, free=17262669824, active=185582317568, inactive=49277759488, buffers=757313536, cached=85193539584, shared=5661687808, slab=3205939200) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 18 | loss=3.9360241889953613 | Batch Time=21.09375 +GPU:0 | Epoch: 18 | loss=5.234065055847168 | Batch Time=3.125 +GPU:0 | Epoch: 18 | loss=4.687231540679932 | Batch Time=17.578125 +GPU:0 | Epoch: 18 | loss=4.934563159942627 | Batch Time=3.90625 +GPU:0 | Epoch: 18 | loss=5.739521503448486 | Batch Time=1.953125 +GPU:0 | Epoch: 18 | loss=5.097143173217773 | Batch Time=5.46875 +GPU:0 | Epoch: 18 | loss=4.585333824157715 | Batch Time=6.640625 +GPU:0 | Epoch: 18 | loss=5.382248401641846 | Batch Time=5.859375 +GPU:0 | Epoch: 18 | loss=5.4044365882873535 | Batch Time=6.25 +GPU:0 | Epoch: 18 | loss=6.171414375305176 | Batch Time=1.171875 +GPU:0 | Epoch: 18 | loss=5.850116729736328 | Batch Time=2.734375 +GPU:0 | Epoch: 18 | loss=5.468409538269043 | Batch Time=7.421875 +GPU:0 | Epoch: 18 | loss=5.45800256729126 | Batch Time=10.15625 +GPU:0 | Epoch: 18 | loss=5.002710342407227 | Batch Time=10.9375 +GPU:0 | Epoch: 18 | loss=4.98520565032959 | Batch Time=6.640625 +GPU:0 | Epoch: 18 | loss=5.0501275062561035 | Batch Time=8.984375 +GPU:0 | Epoch: 18 | loss=5.301901817321777 | Batch Time=6.25 +GPU:0 | Epoch: 18 | loss=4.121160507202148 | Batch Time=15.625 +GPU:0 | Epoch: 18 | loss=4.735714435577393 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=3.407996892929077 | Batch Time=28.125 +Model top1 Accuracy: 10.8 +Acc before rounding: 10.8 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 18 | loss=3.9360241889953613 | Batch Time=21.09375 +GPU:0 | Epoch: 18 | loss=5.234065055847168 | Batch Time=3.125 +GPU:0 | Epoch: 18 | loss=4.687231540679932 | Batch Time=17.578125 +GPU:0 | Epoch: 18 | loss=4.934563159942627 | Batch Time=3.90625 +GPU:0 | Epoch: 18 | loss=5.739521503448486 | Batch Time=1.953125 +GPU:0 | Epoch: 18 | loss=5.097143173217773 | Batch Time=5.46875 +GPU:0 | Epoch: 18 | loss=4.585333824157715 | Batch Time=6.640625 +GPU:0 | Epoch: 18 | loss=5.382248401641846 | Batch Time=5.859375 +GPU:0 | Epoch: 18 | loss=5.4044365882873535 | Batch Time=6.25 +GPU:0 | Epoch: 18 | loss=6.171414375305176 | Batch Time=1.171875 +GPU:0 | Epoch: 18 | loss=5.850116729736328 | Batch Time=2.734375 +GPU:0 | Epoch: 18 | loss=5.468409538269043 | Batch Time=7.421875 +GPU:0 | Epoch: 18 | loss=5.45800256729126 | Batch Time=10.15625 +GPU:0 | Epoch: 18 | loss=5.002710342407227 | Batch Time=10.9375 +GPU:0 | Epoch: 18 | loss=4.98520565032959 | Batch Time=6.640625 +GPU:0 | Epoch: 18 | loss=5.0501275062561035 | Batch Time=8.984375 +GPU:0 | Epoch: 18 | loss=5.301901817321777 | Batch Time=6.25 +GPU:0 | Epoch: 18 | loss=4.121160507202148 | Batch Time=15.625 +GPU:0 | Epoch: 18 | loss=4.735714435577393 | Batch Time=13.28125 +GPU:0 | Epoch: 18 | loss=3.407996892929077 | Batch Time=28.125 +Model top1 Accuracy: 10.8 +Acc after rounding: 10.8 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 18 | loss=5.23529577255249 | Batch Time=7.421875 +GPU:0 | Epoch: 18 | loss=5.085639953613281 | Batch Time=9.765625 +GPU:0 | Epoch: 18 | loss=4.8659539222717285 | Batch Time=11.328125 +GPU:0 | Epoch: 18 | loss=5.110870838165283 | Batch Time=8.59375 +Model top1 Accuracy: 9.43 +Validation Acc after rounding: 9.43 +AFTER VAL LOOP: GPU: 0 | Epoch 18 | Memory Usage: svmem(total=257568083968, available=77490704384, percent=69.9, used=172164096000, free=5500522496, active=207695822848, inactive=38843150336, buffers=731250688, cached=79172214784, shared=5661917184, slab=3181748224) +Rounding model with scheme: naive +Model avg sparsity: 39.876258554581824 +GPU:0 | Epoch: 18 | Acc=10.8 | Epoch Time=17.97793041865031 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 18 | Memory Usage: svmem(total=257568083968, available=77490704384, percent=69.9, used=172164096000, free=5500522496, active=207695867904, inactive=38843150336, buffers=731344896, cached=79172120576, shared=5661917184, slab=3181748224) +STARTING TRAINING: Epoch 19 | Memory Usage: svmem(total=257568083968, available=77490704384, percent=69.9, used=172164096000, free=5500522496, active=207695867904, inactive=38843150336, buffers=731344896, cached=79172120576, shared=5661917184, slab=3181748224) +BEFORE TRAIN LOOP: Epoch 19 | Memory Usage: svmem(total=257568083968, available=77490704384, percent=69.9, used=172164096000, free=5500522496, active=207695867904, inactive=38843150336, buffers=731344896, cached=79172120576, shared=5661917184, slab=3181748224) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=77490704384, percent=69.9, used=172164096000, free=5500522496, active=207695867904, inactive=38843150336, buffers=731344896, cached=79172120576, shared=5661917184, slab=3181748224) +GPU:3 | Epoch: 19 | loss=10.441650390625 | Batch Time=14.84375 +GPU:3 | Epoch: 19 | loss=10.24258804321289 | Batch Time=19.921875 +GPU:3 | Epoch: 19 | loss=10.453393936157227 | Batch Time=18.359375 +GPU:3 | Epoch: 19 | loss=10.558269500732422 | Batch Time=13.671875 +GPU:3 | Epoch: 19 | loss=10.31498908996582 | Batch Time=13.671875 +GPU:3 | Epoch: 19 | loss=10.591962814331055 | Batch Time=13.28125 +GPU:3 | Epoch: 19 | loss=10.446690559387207 | Batch Time=18.359375 +GPU:3 | Epoch: 19 | loss=10.654083251953125 | Batch Time=13.28125 +GPU:3 | Epoch: 19 | loss=10.64427375793457 | Batch Time=12.109375 +GPU:3 | Epoch: 19 | loss=10.538150787353516 | Batch Time=15.625 +GPU:3 | Epoch: 19 | loss=10.568109512329102 | Batch Time=14.84375 +GPU:3 | Epoch: 19 | loss=10.441974639892578 | Batch Time=15.625 +GPU:3 | Epoch: 19 | loss=10.502496719360352 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.405461311340332 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.598051071166992 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.47461223602295 | Batch Time=16.015625 +GPU:3 | Epoch: 19 | loss=10.443572998046875 | Batch Time=17.578125 +GPU:3 | Epoch: 19 | loss=10.420243263244629 | Batch Time=13.28125 +GPU:3 | Epoch: 19 | loss=10.513187408447266 | Batch Time=13.671875 +GPU:3 | Epoch: 19 | loss=10.435649871826172 | Batch Time=13.671875 +GPU:3 | Epoch: 19 | loss=10.450271606445312 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.458592414855957 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.577790260314941 | Batch Time=14.453125 +GPU:3 | Epoch: 19 | loss=10.329155921936035 | Batch Time=16.40625 +GPU:3 | Epoch: 19 | loss=10.654300689697266 | Batch Time=11.71875 +GPU:3 | Epoch: 19 | loss=10.670389175415039 | Batch Time=14.84375 +GPU:3 | Epoch: 19 | loss=10.483478546142578 | Batch Time=12.109375 +GPU:3 | Epoch: 19 | loss=10.44283676147461 | Batch Time=16.015625 +GPU:3 | Epoch: 19 | loss=10.528768539428711 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.308351516723633 | Batch Time=18.75 +GPU:3 | Epoch: 19 | loss=10.32084846496582 | Batch Time=15.625 +GPU:3 | Epoch: 19 | loss=10.378355026245117 | Batch Time=16.40625 +GPU:3 | Epoch: 19 | loss=10.370294570922852 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.323148727416992 | Batch Time=15.625 +GPU:3 | Epoch: 19 | loss=10.5030517578125 | Batch Time=12.890625 +GPU:3 | Epoch: 19 | loss=10.720892906188965 | Batch Time=9.375 +GPU:3 | Epoch: 19 | loss=10.662300109863281 | Batch Time=11.71875 +GPU:3 | Epoch: 19 | loss=10.568033218383789 | Batch Time=17.578125 +GPU:3 | Epoch: 19 | loss=10.393299102783203 | Batch Time=15.625 +GPU:3 | Epoch: 19 | loss=10.484085083007812 | Batch Time=16.015625 +GPU:3 | Epoch: 19 | loss=10.200135231018066 | Batch Time=16.40625 +GPU:3 | Epoch: 19 | loss=10.323982238769531 | Batch Time=18.359375 +GPU:3 | Epoch: 19 | loss=10.455792427062988 | Batch Time=14.453125 +GPU:3 | Epoch: 19 | loss=10.302018165588379 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.607057571411133 | Batch Time=12.109375 +GPU:3 | Epoch: 19 | loss=10.44295883178711 | Batch Time=14.453125 +GPU:3 | Epoch: 19 | loss=10.673601150512695 | Batch Time=11.71875 +GPU:3 | Epoch: 19 | loss=10.31559944152832 | Batch Time=19.53125 +GPU:3 | Epoch: 19 | loss=10.265804290771484 | Batch Time=13.671875 +GPU:3 | Epoch: 19 | loss=10.292518615722656 | Batch Time=17.96875 +GPU:3 | Epoch: 19 | loss=10.524372100830078 | Batch Time=12.890625 +GPU:3 | Epoch: 19 | loss=10.358633995056152 | Batch Time=14.453125 +GPU:3 | Epoch: 19 | loss=10.565912246704102 | Batch Time=9.375 +GPU:3 | Epoch: 19 | loss=10.238540649414062 | Batch Time=15.625 +GPU:3 | Epoch: 19 | loss=10.466903686523438 | Batch Time=17.578125 +GPU:3 | Epoch: 19 | loss=10.296152114868164 | Batch Time=16.015625 +GPU:3 | Epoch: 19 | loss=10.421468734741211 | Batch Time=15.234375 +GPU:3 | Epoch: 19 | loss=10.468891143798828 | Batch Time=14.84375 +GPU:3 | Epoch: 19 | loss=10.363630294799805 | Batch Time=17.578125 +GPU:3 | Epoch: 19 | loss=10.568618774414062 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.58474349975586 | Batch Time=11.71875 +GPU:3 | Epoch: 19 | loss=10.458080291748047 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.317045211791992 | Batch Time=11.71875 +GPU:3 | Epoch: 19 | loss=10.468650817871094 | Batch Time=15.234375 +GPU:3 | Epoch: 19 | loss=10.340763092041016 | Batch Time=16.015625 +GPU:3 | Epoch: 19 | loss=10.425121307373047 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.249859809875488 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.309476852416992 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.406007766723633 | Batch Time=12.109375 +GPU:3 | Epoch: 19 | loss=10.503240585327148 | Batch Time=11.328125 +GPU:3 | Epoch: 19 | loss=10.197458267211914 | Batch Time=16.796875 +GPU:3 | Epoch: 19 | loss=10.508323669433594 | Batch Time=11.71875 +GPU:3 | Epoch: 19 | loss=10.398443222045898 | Batch Time=17.96875 +GPU:3 | Epoch: 19 | loss=10.423118591308594 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.354028701782227 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.359014511108398 | Batch Time=14.453125 +GPU:3 | Epoch: 19 | loss=10.36812973022461 | Batch Time=13.671875 +GPU:3 | Epoch: 19 | loss=10.310824394226074 | Batch Time=16.796875 +GPU:3 | Epoch: 19 | loss=10.348241806030273 | Batch Time=17.578125 +GPU:3 | Epoch: 19 | loss=10.30173110961914 | Batch Time=12.5 +GPU:3 | Epoch: 19 | loss=10.512752532958984 | Batch Time=18.359375 +GPU:3 | Epoch: 19 | loss=10.397001266479492 | Batch Time=14.84375 +GPU:3 | Epoch: 19 | loss=10.394974708557129 | Batch Time=12.109375 +GPU:3 | Epoch: 19 | loss=10.452081680297852 | Batch Time=17.578125 +GPU:3 | Epoch: 19 | loss=10.386739730834961 | Batch Time=13.28125 +GPU:3 | Epoch: 19 | loss=10.338044166564941 | Batch Time=13.28125 +GPU:3 | Epoch: 19 | loss=10.464759826660156 | Batch Time=11.71875 +GPU:3 | Epoch: 19 | loss=10.43281364440918 | Batch Time=12.890625 +GPU:3 | Epoch: 19 | loss=10.28464126586914 | Batch Time=12.890625 +GPU:3 | Epoch: 19 | loss=10.279190063476562 | Batch Time=15.234375 +GPU:3 | Epoch: 19 | loss=10.419827461242676 | Batch Time=11.328125 +GPU:3 | Epoch: 19 | loss=10.467841148376465 | Batch Time=14.84375 +GPU:3 | Epoch: 19 | loss=10.275796890258789 | Batch Time=18.75 +GPU:3 | Epoch: 19 | loss=10.296249389648438 | Batch Time=15.234375 +GPU:3 | Epoch: 19 | loss=10.448740005493164 | Batch Time=13.28125 +GPU:3 | Epoch: 19 | loss=10.265600204467773 | Batch Time=17.578125 +GPU:3 | Epoch: 19 | loss=10.32091236114502 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.279747009277344 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.3260498046875 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.378540992736816 | Batch Time=15.625 +GPU:3 | Epoch: 19 | loss=10.489532470703125 | Batch Time=10.9375 +GPU:3 | Epoch: 19 | loss=10.297813415527344 | Batch Time=14.84375 +GPU:3 | Epoch: 19 | loss=10.375937461853027 | Batch Time=12.109375 +GPU:3 | Epoch: 19 | loss=10.33790111541748 | Batch Time=15.625 +GPU:3 | Epoch: 19 | loss=10.1919584274292 | Batch Time=21.09375 +GPU:3 | Epoch: 19 | loss=10.51507568359375 | Batch Time=11.328125 +GPU:3 | Epoch: 19 | loss=10.479509353637695 | Batch Time=15.234375 +GPU:3 | Epoch: 19 | loss=10.294944763183594 | Batch Time=16.796875 +GPU:3 | Epoch: 19 | loss=10.287724494934082 | Batch Time=14.453125 +GPU:3 | Epoch: 19 | loss=10.221458435058594 | Batch Time=20.3125 +GPU:3 | Epoch: 19 | loss=10.294393539428711 | Batch Time=14.453125 +GPU:3 | Epoch: 19 | loss=10.286996841430664 | Batch Time=21.09375 +GPU:3 | Epoch: 19 | loss=10.342866897583008 | Batch Time=16.796875 +GPU:3 | Epoch: 19 | loss=10.289875030517578 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.269251823425293 | Batch Time=12.5 +GPU:3 | Epoch: 19 | loss=10.175115585327148 | Batch Time=17.1875 +GPU:3 | Epoch: 19 | loss=10.479969024658203 | Batch Time=16.015625 +GPU:3 | Epoch: 19 | loss=10.421180725097656 | Batch Time=14.0625 +GPU:3 | Epoch: 19 | loss=10.068023681640625 | Batch Time=20.703125 +GPU:3 | Epoch: 19 | loss=10.483798027038574 | Batch Time=12.5 +GPU:3 | Epoch: 19 | loss=10.349797248840332 | Batch Time=15.234375 +GPU:3 | Epoch: 19 | loss=10.254022598266602 | Batch Time=14.84375 +GPU:3 | Epoch: 19 | loss=10.327068328857422 | Batch Time=16.015625 +GPU:3 | Epoch: 19 | loss=10.245634078979492 | Batch Time=17.96875 +GPU:1 | Epoch: 19 | loss=10.521270751953125 | Batch Time=15.625 +GPU:1 | Epoch: 19 | loss=10.54693603515625 | Batch Time=12.890625 +GPU:1 | Epoch: 19 | loss=10.425666809082031 | Batch Time=17.1875 +GPU:1 | Epoch: 19 | loss=10.526288032531738 | Batch Time=16.796875 +GPU:1 | Epoch: 19 | loss=10.455142974853516 | Batch Time=17.578125 +GPU:1 | Epoch: 19 | loss=10.58301830291748 | Batch Time=12.5 +GPU:1 | Epoch: 19 | loss=10.419343948364258 | Batch Time=12.5 +GPU:1 | Epoch: 19 | loss=10.508346557617188 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.291831970214844 | Batch Time=19.53125 +GPU:1 | Epoch: 19 | loss=10.371015548706055 | Batch Time=17.1875 +GPU:1 | Epoch: 19 | loss=10.594111442565918 | Batch Time=15.625 +GPU:1 | Epoch: 19 | loss=10.32601261138916 | Batch Time=17.96875 +GPU:1 | Epoch: 19 | loss=10.747831344604492 | Batch Time=10.9375 +GPU:1 | Epoch: 19 | loss=10.445097923278809 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.554025650024414 | Batch Time=18.359375 +GPU:1 | Epoch: 19 | loss=10.754814147949219 | Batch Time=14.0625 +GPU:1 | Epoch: 19 | loss=10.606472969055176 | Batch Time=10.15625 +GPU:1 | Epoch: 19 | loss=10.444616317749023 | Batch Time=15.625 +GPU:1 | Epoch: 19 | loss=10.473991394042969 | Batch Time=15.625 +GPU:1 | Epoch: 19 | loss=10.480255126953125 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.386346817016602 | Batch Time=14.0625 +GPU:1 | Epoch: 19 | loss=10.502445220947266 | Batch Time=15.625 +GPU:1 | Epoch: 19 | loss=10.473287582397461 | Batch Time=13.671875 +GPU:1 | Epoch: 19 | loss=10.36945915222168 | Batch Time=18.359375 +GPU:1 | Epoch: 19 | loss=10.546460151672363 | Batch Time=17.96875 +GPU:1 | Epoch: 19 | loss=10.58696174621582 | Batch Time=13.28125 +GPU:1 | Epoch: 19 | loss=10.562812805175781 | Batch Time=12.5 +GPU:1 | Epoch: 19 | loss=10.482330322265625 | Batch Time=11.328125 +GPU:1 | Epoch: 19 | loss=10.554006576538086 | Batch Time=11.71875 +GPU:1 | Epoch: 19 | loss=10.427864074707031 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.447677612304688 | Batch Time=13.28125 +GPU:1 | Epoch: 19 | loss=10.415481567382812 | Batch Time=18.359375 +GPU:1 | Epoch: 19 | loss=10.56203842163086 | Batch Time=13.671875 +GPU:1 | Epoch: 19 | loss=10.362586975097656 | Batch Time=19.921875 +GPU:1 | Epoch: 19 | loss=10.554100036621094 | Batch Time=10.9375 +GPU:1 | Epoch: 19 | loss=10.251226425170898 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.449935913085938 | Batch Time=12.890625 +GPU:1 | Epoch: 19 | loss=10.484092712402344 | Batch Time=12.5 +GPU:1 | Epoch: 19 | loss=10.504895210266113 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.506792068481445 | Batch Time=14.0625 +GPU:1 | Epoch: 19 | loss=10.239715576171875 | Batch Time=15.234375 +GPU:1 | Epoch: 19 | loss=10.366095542907715 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.567449569702148 | Batch Time=14.0625 +GPU:1 | Epoch: 19 | loss=10.404950141906738 | Batch Time=17.1875 +GPU:1 | Epoch: 19 | loss=10.609413146972656 | Batch Time=13.28125 +GPU:1 | Epoch: 19 | loss=10.383161544799805 | Batch Time=15.234375 +GPU:1 | Epoch: 19 | loss=10.475824356079102 | Batch Time=12.109375 +GPU:1 | Epoch: 19 | loss=10.447000503540039 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.69883918762207 | Batch Time=12.109375 +GPU:1 | Epoch: 19 | loss=10.496767044067383 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.549066543579102 | Batch Time=15.234375 +GPU:1 | Epoch: 19 | loss=10.50775146484375 | Batch Time=12.890625 +GPU:1 | Epoch: 19 | loss=10.618173599243164 | Batch Time=13.671875 +GPU:1 | Epoch: 19 | loss=10.212403297424316 | Batch Time=20.3125 +GPU:1 | Epoch: 19 | loss=10.477681159973145 | Batch Time=12.5 +GPU:1 | Epoch: 19 | loss=10.163712501525879 | Batch Time=21.484375 +GPU:1 | Epoch: 19 | loss=10.51083755493164 | Batch Time=13.28125 +GPU:1 | Epoch: 19 | loss=10.49703598022461 | Batch Time=10.9375 +GPU:1 | Epoch: 19 | loss=10.299236297607422 | Batch Time=17.96875 +GPU:1 | Epoch: 19 | loss=10.429756164550781 | Batch Time=15.625 +GPU:1 | Epoch: 19 | loss=10.366554260253906 | Batch Time=18.359375 +GPU:1 | Epoch: 19 | loss=10.477190017700195 | Batch Time=13.671875 +GPU:1 | Epoch: 19 | loss=10.448535919189453 | Batch Time=13.28125 +GPU:1 | Epoch: 19 | loss=10.605430603027344 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.663335800170898 | Batch Time=11.71875 +GPU:1 | Epoch: 19 | loss=10.441973686218262 | Batch Time=12.109375 +GPU:1 | Epoch: 19 | loss=10.583887100219727 | Batch Time=14.0625 +GPU:1 | Epoch: 19 | loss=10.566386222839355 | Batch Time=13.28125 +GPU:1 | Epoch: 19 | loss=10.351667404174805 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.460428237915039 | Batch Time=18.75 +GPU:1 | Epoch: 19 | loss=10.356176376342773 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.479410171508789 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.299676895141602 | Batch Time=17.1875 +GPU:1 | Epoch: 19 | loss=10.39153003692627 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.544432640075684 | Batch Time=15.625 +GPU:1 | Epoch: 19 | loss=10.706103324890137 | Batch Time=12.890625 +GPU:1 | Epoch: 19 | loss=10.632869720458984 | Batch Time=9.375 +GPU:1 | Epoch: 19 | loss=10.449239730834961 | Batch Time=17.1875 +GPU:1 | Epoch: 19 | loss=10.316488265991211 | Batch Time=17.578125 +GPU:1 | Epoch: 19 | loss=10.22538948059082 | Batch Time=19.53125 +GPU:1 | Epoch: 19 | loss=10.336915969848633 | Batch Time=13.671875 +GPU:1 | Epoch: 19 | loss=10.400033950805664 | Batch Time=13.671875 +GPU:1 | Epoch: 19 | loss=10.331040382385254 | Batch Time=11.71875 +GPU:1 | Epoch: 19 | loss=10.47067642211914 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.472963333129883 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.283693313598633 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.34548568725586 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.50735855102539 | Batch Time=15.234375 +GPU:1 | Epoch: 19 | loss=10.209880828857422 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.435210227966309 | Batch Time=13.28125 +GPU:1 | Epoch: 19 | loss=10.344265937805176 | Batch Time=16.796875 +GPU:1 | Epoch: 19 | loss=10.23971939086914 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.292854309082031 | Batch Time=16.796875 +GPU:1 | Epoch: 19 | loss=10.517041206359863 | Batch Time=13.28125 +GPU:1 | Epoch: 19 | loss=10.382549285888672 | Batch Time=14.0625 +GPU:1 | Epoch: 19 | loss=10.253177642822266 | Batch Time=16.796875 +GPU:1 | Epoch: 19 | loss=10.418067932128906 | Batch Time=14.0625 +GPU:1 | Epoch: 19 | loss=10.273773193359375 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.485234260559082 | Batch Time=10.9375 +GPU:1 | Epoch: 19 | loss=10.330615997314453 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.271699905395508 | Batch Time=15.234375 +GPU:1 | Epoch: 19 | loss=10.361948013305664 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.142232894897461 | Batch Time=17.96875 +GPU:1 | Epoch: 19 | loss=10.424032211303711 | Batch Time=15.234375 +GPU:1 | Epoch: 19 | loss=10.351272583007812 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.435546875 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.261795043945312 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.477842330932617 | Batch Time=11.71875 +GPU:1 | Epoch: 19 | loss=10.261935234069824 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.318018913269043 | Batch Time=16.796875 +GPU:1 | Epoch: 19 | loss=10.632251739501953 | Batch Time=12.109375 +GPU:1 | Epoch: 19 | loss=10.26085090637207 | Batch Time=17.96875 +GPU:1 | Epoch: 19 | loss=10.180757522583008 | Batch Time=14.84375 +GPU:1 | Epoch: 19 | loss=10.077556610107422 | Batch Time=23.046875 +GPU:1 | Epoch: 19 | loss=10.354641914367676 | Batch Time=14.453125 +GPU:1 | Epoch: 19 | loss=10.195169448852539 | Batch Time=16.015625 +GPU:1 | Epoch: 19 | loss=10.209817886352539 | Batch Time=19.53125 +GPU:1 | Epoch: 19 | loss=10.290895462036133 | Batch Time=14.0625 +GPU:1 | Epoch: 19 | loss=10.296798706054688 | Batch Time=15.625 +GPU:1 | Epoch: 19 | loss=10.400711059570312 | Batch Time=16.796875 +GPU:1 | Epoch: 19 | loss=10.125711441040039 | Batch Time=17.578125 +GPU:1 | Epoch: 19 | loss=10.371419906616211 | Batch Time=12.5 +GPU:1 | Epoch: 19 | loss=10.360931396484375 | Batch Time=17.578125 +GPU:1 | Epoch: 19 | loss=10.32647705078125 | Batch Time=12.5 +GPU:2 | Epoch: 19 | loss=10.49679183959961 | Batch Time=12.890625 +GPU:2 | Epoch: 19 | loss=10.563426971435547 | Batch Time=15.234375 +GPU:2 | Epoch: 19 | loss=10.44917106628418 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.493587493896484 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.573031425476074 | Batch Time=12.890625 +GPU:2 | Epoch: 19 | loss=10.4232177734375 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.400680541992188 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.303709030151367 | Batch Time=17.1875 +GPU:2 | Epoch: 19 | loss=10.514404296875 | Batch Time=14.0625 +GPU:2 | Epoch: 19 | loss=10.509456634521484 | Batch Time=16.015625 +GPU:2 | Epoch: 19 | loss=10.537327766418457 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.447578430175781 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.596954345703125 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.490403175354004 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.54806900024414 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.679788589477539 | Batch Time=10.9375 +GPU:2 | Epoch: 19 | loss=10.176008224487305 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.479575157165527 | Batch Time=16.40625 +GPU:2 | Epoch: 19 | loss=10.428092956542969 | Batch Time=13.671875 +GPU:2 | Epoch: 19 | loss=10.52467155456543 | Batch Time=10.9375 +GPU:2 | Epoch: 19 | loss=10.417993545532227 | Batch Time=15.234375 +GPU:2 | Epoch: 19 | loss=10.486623764038086 | Batch Time=12.890625 +GPU:2 | Epoch: 19 | loss=10.300432205200195 | Batch Time=16.015625 +GPU:2 | Epoch: 19 | loss=10.532119750976562 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.447467803955078 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.445585250854492 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.615612030029297 | Batch Time=10.9375 +GPU:2 | Epoch: 19 | loss=10.475455284118652 | Batch Time=11.71875 +GPU:2 | Epoch: 19 | loss=10.38121223449707 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.243328094482422 | Batch Time=17.578125 +GPU:2 | Epoch: 19 | loss=10.387532234191895 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.686635971069336 | Batch Time=12.5 +GPU:2 | Epoch: 19 | loss=10.509408950805664 | Batch Time=12.890625 +GPU:2 | Epoch: 19 | loss=10.301403045654297 | Batch Time=13.671875 +GPU:2 | Epoch: 19 | loss=10.579161643981934 | Batch Time=15.234375 +GPU:2 | Epoch: 19 | loss=10.590487480163574 | Batch Time=7.8125 +GPU:2 | Epoch: 19 | loss=10.432914733886719 | Batch Time=15.234375 +GPU:2 | Epoch: 19 | loss=10.524288177490234 | Batch Time=12.109375 +GPU:2 | Epoch: 19 | loss=10.4230375289917 | Batch Time=17.1875 +GPU:2 | Epoch: 19 | loss=10.457891464233398 | Batch Time=17.1875 +GPU:2 | Epoch: 19 | loss=10.499835014343262 | Batch Time=12.109375 +GPU:2 | Epoch: 19 | loss=10.391927719116211 | Batch Time=18.359375 +GPU:2 | Epoch: 19 | loss=10.42003059387207 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.468034744262695 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.321514129638672 | Batch Time=17.1875 +GPU:2 | Epoch: 19 | loss=10.533634185791016 | Batch Time=11.71875 +GPU:2 | Epoch: 19 | loss=10.36674976348877 | Batch Time=19.921875 +GPU:2 | Epoch: 19 | loss=10.327621459960938 | Batch Time=16.015625 +GPU:2 | Epoch: 19 | loss=10.358980178833008 | Batch Time=15.234375 +GPU:2 | Epoch: 19 | loss=10.264317512512207 | Batch Time=19.53125 +GPU:2 | Epoch: 19 | loss=10.493875503540039 | Batch Time=16.015625 +GPU:2 | Epoch: 19 | loss=10.202505111694336 | Batch Time=19.53125 +GPU:2 | Epoch: 19 | loss=10.495046615600586 | Batch Time=15.234375 +GPU:2 | Epoch: 19 | loss=10.422446250915527 | Batch Time=14.0625 +GPU:2 | Epoch: 19 | loss=10.38534927368164 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.316780090332031 | Batch Time=16.40625 +GPU:2 | Epoch: 19 | loss=10.437023162841797 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.302831649780273 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.43879508972168 | Batch Time=16.015625 +GPU:2 | Epoch: 19 | loss=10.436840057373047 | Batch Time=14.0625 +GPU:2 | Epoch: 19 | loss=10.385953903198242 | Batch Time=14.0625 +GPU:2 | Epoch: 19 | loss=10.169805526733398 | Batch Time=17.96875 +GPU:2 | Epoch: 19 | loss=10.412808418273926 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.576601028442383 | Batch Time=12.5 +GPU:2 | Epoch: 19 | loss=10.307701110839844 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.597330093383789 | Batch Time=13.671875 +GPU:2 | Epoch: 19 | loss=10.626489639282227 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.30753231048584 | Batch Time=18.359375 +GPU:2 | Epoch: 19 | loss=10.623458862304688 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.420044898986816 | Batch Time=14.0625 +GPU:2 | Epoch: 19 | loss=10.238730430603027 | Batch Time=19.140625 +GPU:2 | Epoch: 19 | loss=10.24499797821045 | Batch Time=17.1875 +GPU:2 | Epoch: 19 | loss=10.552691459655762 | Batch Time=13.671875 +GPU:2 | Epoch: 19 | loss=10.514225006103516 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.518329620361328 | Batch Time=12.890625 +GPU:2 | Epoch: 19 | loss=10.36727523803711 | Batch Time=14.0625 +GPU:2 | Epoch: 19 | loss=10.531524658203125 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.428264617919922 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.44091796875 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.41346549987793 | Batch Time=13.671875 +GPU:2 | Epoch: 19 | loss=10.40292739868164 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.566608428955078 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.501510620117188 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.475188255310059 | Batch Time=12.5 +GPU:2 | Epoch: 19 | loss=10.395648956298828 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.576393127441406 | Batch Time=11.71875 +GPU:2 | Epoch: 19 | loss=10.205991744995117 | Batch Time=17.1875 +GPU:2 | Epoch: 19 | loss=10.306913375854492 | Batch Time=12.890625 +GPU:2 | Epoch: 19 | loss=10.47276496887207 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.302390098571777 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.380134582519531 | Batch Time=10.15625 +GPU:2 | Epoch: 19 | loss=10.531381607055664 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.262395858764648 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.475419998168945 | Batch Time=16.015625 +GPU:2 | Epoch: 19 | loss=10.321966171264648 | Batch Time=17.578125 +GPU:2 | Epoch: 19 | loss=10.250909805297852 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.392553329467773 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.288381576538086 | Batch Time=18.75 +GPU:2 | Epoch: 19 | loss=10.204012870788574 | Batch Time=16.015625 +GPU:2 | Epoch: 19 | loss=10.304320335388184 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.339250564575195 | Batch Time=18.359375 +GPU:2 | Epoch: 19 | loss=10.464765548706055 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.435314178466797 | Batch Time=14.0625 +GPU:2 | Epoch: 19 | loss=10.283370971679688 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.23579216003418 | Batch Time=16.015625 +GPU:2 | Epoch: 19 | loss=10.597723007202148 | Batch Time=12.109375 +GPU:2 | Epoch: 19 | loss=10.42607307434082 | Batch Time=14.453125 +GPU:2 | Epoch: 19 | loss=10.522563934326172 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.352681159973145 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.512687683105469 | Batch Time=11.328125 +GPU:2 | Epoch: 19 | loss=10.42816162109375 | Batch Time=11.328125 +GPU:2 | Epoch: 19 | loss=10.198745727539062 | Batch Time=15.234375 +GPU:2 | Epoch: 19 | loss=10.44420051574707 | Batch Time=14.0625 +GPU:2 | Epoch: 19 | loss=10.462230682373047 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.362191200256348 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.347391128540039 | Batch Time=16.796875 +GPU:2 | Epoch: 19 | loss=10.259231567382812 | Batch Time=14.84375 +GPU:2 | Epoch: 19 | loss=10.387709617614746 | Batch Time=11.71875 +GPU:2 | Epoch: 19 | loss=10.51303482055664 | Batch Time=15.625 +GPU:2 | Epoch: 19 | loss=10.51003360748291 | Batch Time=13.28125 +GPU:2 | Epoch: 19 | loss=10.208894729614258 | Batch Time=18.75 +GPU:2 | Epoch: 19 | loss=10.246053695678711 | Batch Time=17.578125 +GPU:2 | Epoch: 19 | loss=10.275911331176758 | Batch Time=19.53125 +GPU:2 | Epoch: 19 | loss=10.226980209350586 | Batch Time=17.96875 +GPU:0 | Epoch: 19 | loss=10.608134269714355 | Batch Time=12.5 +GPU:0 | Epoch: 19 | loss=10.415946960449219 | Batch Time=16.40625 +GPU:0 | Epoch: 19 | loss=10.579730033874512 | Batch Time=12.5 +GPU:0 | Epoch: 19 | loss=10.519110679626465 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.571319580078125 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.38726806640625 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=10.628140449523926 | Batch Time=11.71875 +GPU:0 | Epoch: 19 | loss=10.583492279052734 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.567501068115234 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.595273971557617 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.41454792022705 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.331565856933594 | Batch Time=15.625 +GPU:0 | Epoch: 19 | loss=10.370469093322754 | Batch Time=16.40625 +GPU:0 | Epoch: 19 | loss=10.421416282653809 | Batch Time=17.1875 +GPU:0 | Epoch: 19 | loss=10.52082633972168 | Batch Time=12.890625 +GPU:0 | Epoch: 19 | loss=10.399417877197266 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=10.434297561645508 | Batch Time=17.1875 +GPU:0 | Epoch: 19 | loss=10.510050773620605 | Batch Time=14.0625 +GPU:0 | Epoch: 19 | loss=10.51104736328125 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=10.332084655761719 | Batch Time=19.921875 +GPU:0 | Epoch: 19 | loss=10.262256622314453 | Batch Time=17.578125 +GPU:0 | Epoch: 19 | loss=10.411092758178711 | Batch Time=17.578125 +GPU:0 | Epoch: 19 | loss=10.397315979003906 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.54922103881836 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.540138244628906 | Batch Time=10.546875 +GPU:0 | Epoch: 19 | loss=10.474691390991211 | Batch Time=17.578125 +GPU:0 | Epoch: 19 | loss=10.249290466308594 | Batch Time=18.359375 +GPU:0 | Epoch: 19 | loss=10.407662391662598 | Batch Time=10.9375 +GPU:0 | Epoch: 19 | loss=10.47944450378418 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=10.364786148071289 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.427877426147461 | Batch Time=18.75 +GPU:0 | Epoch: 19 | loss=10.289926528930664 | Batch Time=15.625 +GPU:0 | Epoch: 19 | loss=10.38991928100586 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.354732513427734 | Batch Time=17.96875 +GPU:0 | Epoch: 19 | loss=10.47793960571289 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.442009925842285 | Batch Time=14.0625 +GPU:0 | Epoch: 19 | loss=10.334680557250977 | Batch Time=15.625 +GPU:0 | Epoch: 19 | loss=10.386406898498535 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.457670211791992 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.337872505187988 | Batch Time=14.0625 +GPU:0 | Epoch: 19 | loss=10.250160217285156 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=10.668525695800781 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=10.436956405639648 | Batch Time=16.796875 +GPU:0 | Epoch: 19 | loss=10.61686897277832 | Batch Time=10.9375 +GPU:0 | Epoch: 19 | loss=10.398157119750977 | Batch Time=14.453125 +GPU:0 | Epoch: 19 | loss=10.430639266967773 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=10.343345642089844 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.450826644897461 | Batch Time=17.1875 +GPU:0 | Epoch: 19 | loss=10.507925033569336 | Batch Time=11.71875 +GPU:0 | Epoch: 19 | loss=10.477588653564453 | Batch Time=14.0625 +GPU:0 | Epoch: 19 | loss=10.351390838623047 | Batch Time=12.890625 +GPU:0 | Epoch: 19 | loss=10.197364807128906 | Batch Time=18.359375 +GPU:0 | Epoch: 19 | loss=10.487065315246582 | Batch Time=12.5 +GPU:0 | Epoch: 19 | loss=10.408927917480469 | Batch Time=16.796875 +GPU:0 | Epoch: 19 | loss=10.126710891723633 | Batch Time=19.140625 +GPU:0 | Epoch: 19 | loss=10.328832626342773 | Batch Time=17.96875 +GPU:0 | Epoch: 19 | loss=10.569332122802734 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.385963439941406 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=10.31306266784668 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=10.406919479370117 | Batch Time=12.890625 +GPU:0 | Epoch: 19 | loss=10.393170356750488 | Batch Time=14.453125 +GPU:0 | Epoch: 19 | loss=10.486454010009766 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.532283782958984 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.461488723754883 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.326432228088379 | Batch Time=16.40625 +GPU:0 | Epoch: 19 | loss=10.105873107910156 | Batch Time=16.796875 +GPU:0 | Epoch: 19 | loss=10.399347305297852 | Batch Time=10.546875 +GPU:0 | Epoch: 19 | loss=10.303182601928711 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.142730712890625 | Batch Time=19.921875 +GPU:0 | Epoch: 19 | loss=10.231307983398438 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=10.335393905639648 | Batch Time=19.921875 +GPU:0 | Epoch: 19 | loss=10.492192268371582 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.296289443969727 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=10.298074722290039 | Batch Time=16.796875 +GPU:0 | Epoch: 19 | loss=10.460220336914062 | Batch Time=16.40625 +GPU:0 | Epoch: 19 | loss=10.587188720703125 | Batch Time=10.15625 +GPU:0 | Epoch: 19 | loss=10.429716110229492 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.275219917297363 | Batch Time=16.796875 +GPU:0 | Epoch: 19 | loss=10.29237174987793 | Batch Time=16.796875 +GPU:0 | Epoch: 19 | loss=10.350975036621094 | Batch Time=12.890625 +GPU:0 | Epoch: 19 | loss=10.46325397491455 | Batch Time=12.5 +GPU:0 | Epoch: 19 | loss=10.395790100097656 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=10.39577865600586 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.509166717529297 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=10.178058624267578 | Batch Time=19.140625 +GPU:0 | Epoch: 19 | loss=10.437714576721191 | Batch Time=17.578125 +GPU:0 | Epoch: 19 | loss=10.234066009521484 | Batch Time=16.40625 +GPU:0 | Epoch: 19 | loss=10.349700927734375 | Batch Time=13.671875 +GPU:0 | Epoch: 19 | loss=10.368967056274414 | Batch Time=15.625 +GPU:0 | Epoch: 19 | loss=10.396652221679688 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=10.241379737854004 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.453969955444336 | Batch Time=14.0625 +GPU:0 | Epoch: 19 | loss=10.354955673217773 | Batch Time=16.796875 +GPU:0 | Epoch: 19 | loss=10.097323417663574 | Batch Time=17.96875 +GPU:0 | Epoch: 19 | loss=10.19789981842041 | Batch Time=19.53125 +GPU:0 | Epoch: 19 | loss=10.49957275390625 | Batch Time=12.5 +GPU:0 | Epoch: 19 | loss=10.297595977783203 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.519063949584961 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.389972686767578 | Batch Time=14.453125 +GPU:0 | Epoch: 19 | loss=10.320359230041504 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.22140121459961 | Batch Time=18.359375 +GPU:0 | Epoch: 19 | loss=10.305644989013672 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.334901809692383 | Batch Time=17.578125 +GPU:0 | Epoch: 19 | loss=10.31495189666748 | Batch Time=16.40625 +GPU:0 | Epoch: 19 | loss=10.430882453918457 | Batch Time=9.375 +GPU:0 | Epoch: 19 | loss=10.397953033447266 | Batch Time=14.0625 +GPU:0 | Epoch: 19 | loss=10.332345008850098 | Batch Time=15.625 +GPU:0 | Epoch: 19 | loss=10.207963943481445 | Batch Time=15.625 +GPU:0 | Epoch: 19 | loss=10.445993423461914 | Batch Time=12.890625 +GPU:0 | Epoch: 19 | loss=10.432108879089355 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.401071548461914 | Batch Time=10.9375 +GPU:0 | Epoch: 19 | loss=10.277885437011719 | Batch Time=15.625 +GPU:0 | Epoch: 19 | loss=10.259322166442871 | Batch Time=17.96875 +GPU:0 | Epoch: 19 | loss=10.422980308532715 | Batch Time=14.0625 +GPU:0 | Epoch: 19 | loss=10.413105964660645 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=10.240687370300293 | Batch Time=17.96875 +GPU:0 | Epoch: 19 | loss=10.19426155090332 | Batch Time=16.40625 +GPU:0 | Epoch: 19 | loss=10.222061157226562 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=10.39341926574707 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.264713287353516 | Batch Time=16.40625 +GPU:0 | Epoch: 19 | loss=10.473298072814941 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.153190612792969 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=10.300098419189453 | Batch Time=14.84375 +GPU:0 | Epoch: 19 | loss=10.421956062316895 | Batch Time=16.015625 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 19 | loss=10.240667343139648 | Batch Time=16.015625 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749672960, percent=65.9, used=161905094656, free=16922005504, active=189031567360, inactive=46236332032, buffers=542220288, cached=78198763520, shared=5661884416, slab=3070816256) +AFTER TRAIN LOOP: Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +STARTING TRAINING: Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +BEFORE TRAIN LOOP: Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +GPU:2 | Epoch: 19 | loss=10.360710144042969 | Batch Time=14.84375 +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749672960, percent=65.9, used=161905094656, free=16922005504, active=189031567360, inactive=46236332032, buffers=542220288, cached=78198763520, shared=5661884416, slab=3070816256) +AFTER TRAIN LOOP: Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +STARTING TRAINING: Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +BEFORE TRAIN LOOP: Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +GPU:0 | Epoch: 19 | loss=10.366643905639648 | Batch Time=14.0625 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +AFTER TRAIN LOOP: Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749758976, percent=65.9, used=161905209344, free=16922054656, active=189031460864, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070775296) +BEFORE VAL LOOP: GPU: 0 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749758976, percent=65.9, used=161905209344, free=16922054656, active=189031460864, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070775296) +GPU:3 | Epoch: 19 | loss=10.15781021118164 | Batch Time=13.28125 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749742592, percent=65.9, used=161905225728, free=16922038272, active=189031473152, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070783488) +AFTER TRAIN LOOP: Epoch 19 | Memory Usage: svmem(total=257568083968, available=87749758976, percent=65.9, used=161905209344, free=16922054656, active=189031460864, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070775296) +STARTING TRAINING: Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749758976, percent=65.9, used=161905209344, free=16922054656, active=189031460864, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070775296) +BEFORE TRAIN LOOP: Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749758976, percent=65.9, used=161905209344, free=16922054656, active=189031460864, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070775296) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=87749758976, percent=65.9, used=161905209344, free=16922054656, active=189031460864, inactive=46236303360, buffers=542220288, cached=78198599680, shared=5661720576, slab=3070775296) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 19 | loss=3.5238194465637207 | Batch Time=29.296875 +GPU:0 | Epoch: 19 | loss=5.169803142547607 | Batch Time=10.15625 +GPU:0 | Epoch: 19 | loss=4.50384521484375 | Batch Time=19.53125 +GPU:0 | Epoch: 19 | loss=3.3711020946502686 | Batch Time=23.046875 +GPU:0 | Epoch: 19 | loss=4.926790237426758 | Batch Time=6.25 +GPU:0 | Epoch: 19 | loss=3.987050771713257 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=5.111251354217529 | Batch Time=7.03125 +GPU:0 | Epoch: 19 | loss=5.048952579498291 | Batch Time=4.6875 +GPU:0 | Epoch: 19 | loss=5.475464344024658 | Batch Time=5.078125 +GPU:0 | Epoch: 19 | loss=6.109066963195801 | Batch Time=0.78125 +GPU:0 | Epoch: 19 | loss=5.69036865234375 | Batch Time=1.5625 +GPU:0 | Epoch: 19 | loss=4.75058126449585 | Batch Time=19.140625 +GPU:0 | Epoch: 19 | loss=5.296530723571777 | Batch Time=12.890625 +GPU:0 | Epoch: 19 | loss=4.551669597625732 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=4.817022323608398 | Batch Time=8.984375 +GPU:0 | Epoch: 19 | loss=5.025746822357178 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=5.035422325134277 | Batch Time=5.078125 +GPU:0 | Epoch: 19 | loss=3.97953200340271 | Batch Time=22.265625 +GPU:0 | Epoch: 19 | loss=3.9054698944091797 | Batch Time=18.75 +GPU:0 | Epoch: 19 | loss=3.4663424491882324 | Batch Time=25.78125 +Model top1 Accuracy: 13.6 +Acc before rounding: 13.6 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 19 | loss=3.5238194465637207 | Batch Time=29.296875 +GPU:0 | Epoch: 19 | loss=5.169803142547607 | Batch Time=10.15625 +GPU:0 | Epoch: 19 | loss=4.50384521484375 | Batch Time=19.53125 +GPU:0 | Epoch: 19 | loss=3.3711020946502686 | Batch Time=23.046875 +GPU:0 | Epoch: 19 | loss=4.926790237426758 | Batch Time=6.25 +GPU:0 | Epoch: 19 | loss=3.987050771713257 | Batch Time=16.015625 +GPU:0 | Epoch: 19 | loss=5.111251354217529 | Batch Time=7.03125 +GPU:0 | Epoch: 19 | loss=5.048952579498291 | Batch Time=4.6875 +GPU:0 | Epoch: 19 | loss=5.475464344024658 | Batch Time=5.078125 +GPU:0 | Epoch: 19 | loss=6.109066963195801 | Batch Time=0.78125 +GPU:0 | Epoch: 19 | loss=5.69036865234375 | Batch Time=1.5625 +GPU:0 | Epoch: 19 | loss=4.75058126449585 | Batch Time=19.140625 +GPU:0 | Epoch: 19 | loss=5.296530723571777 | Batch Time=12.890625 +GPU:0 | Epoch: 19 | loss=4.551669597625732 | Batch Time=15.234375 +GPU:0 | Epoch: 19 | loss=4.817022323608398 | Batch Time=8.984375 +GPU:0 | Epoch: 19 | loss=5.025746822357178 | Batch Time=13.28125 +GPU:0 | Epoch: 19 | loss=5.035422325134277 | Batch Time=5.078125 +GPU:0 | Epoch: 19 | loss=3.97953200340271 | Batch Time=22.265625 +GPU:0 | Epoch: 19 | loss=3.9054698944091797 | Batch Time=18.75 +GPU:0 | Epoch: 19 | loss=3.4663424491882324 | Batch Time=25.78125 +Model top1 Accuracy: 13.6 +Acc after rounding: 13.6 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 19 | loss=4.8590989112854 | Batch Time=14.453125 +GPU:0 | Epoch: 19 | loss=4.866205215454102 | Batch Time=12.890625 +GPU:0 | Epoch: 19 | loss=4.706315994262695 | Batch Time=12.5 +GPU:0 | Epoch: 19 | loss=4.779258728027344 | Batch Time=12.890625 +Model top1 Accuracy: 11.41 +Validation Acc after rounding: 11.41 +AFTER VAL LOOP: GPU: 0 | Epoch 19 | Memory Usage: svmem(total=257568083968, available=69921292288, percent=72.9, used=179733000192, free=5577633792, active=209200287744, inactive=37307035648, buffers=551817216, cached=71705632768, shared=5662035968, slab=3060019200) +Rounding model with scheme: naive +Model avg sparsity: 39.3353686387834 +GPU:0 | Epoch: 19 | Acc=13.6 | Epoch Time=18.450809637705486 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 19 | Memory Usage: svmem(total=257568083968, available=69921292288, percent=72.9, used=179733000192, free=5577633792, active=209200287744, inactive=37307035648, buffers=551829504, cached=71705620480, shared=5662035968, slab=3060019200) +STARTING TRAINING: Epoch 20 | Memory Usage: svmem(total=257568083968, available=69921292288, percent=72.9, used=179733000192, free=5577633792, active=209200287744, inactive=37307035648, buffers=551829504, cached=71705620480, shared=5662035968, slab=3060019200) +BEFORE TRAIN LOOP: Epoch 20 | Memory Usage: svmem(total=257568083968, available=69921292288, percent=72.9, used=179733000192, free=5577633792, active=209200287744, inactive=37307035648, buffers=551829504, cached=71705620480, shared=5662035968, slab=3060019200) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=69921292288, percent=72.9, used=179733000192, free=5577633792, active=209200287744, inactive=37307035648, buffers=551829504, cached=71705620480, shared=5662035968, slab=3060019200) +GPU:2 | Epoch: 20 | loss=10.40693187713623 | Batch Time=14.0625 +GPU:2 | Epoch: 20 | loss=10.382227897644043 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.36056900024414 | Batch Time=14.453125 +GPU:2 | Epoch: 20 | loss=10.396523475646973 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.22950267791748 | Batch Time=12.890625 +GPU:2 | Epoch: 20 | loss=10.25362777709961 | Batch Time=17.578125 +GPU:2 | Epoch: 20 | loss=10.231075286865234 | Batch Time=19.53125 +GPU:2 | Epoch: 20 | loss=10.383569717407227 | Batch Time=12.890625 +GPU:2 | Epoch: 20 | loss=10.219932556152344 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.240819931030273 | Batch Time=19.921875 +GPU:2 | Epoch: 20 | loss=10.220235824584961 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.28874397277832 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.276087760925293 | Batch Time=17.1875 +GPU:2 | Epoch: 20 | loss=10.43386459350586 | Batch Time=14.453125 +GPU:2 | Epoch: 20 | loss=10.432191848754883 | Batch Time=15.234375 +GPU:2 | Epoch: 20 | loss=10.32636833190918 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.334598541259766 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.37647533416748 | Batch Time=12.5 +GPU:2 | Epoch: 20 | loss=10.348067283630371 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.210186004638672 | Batch Time=17.578125 +GPU:2 | Epoch: 20 | loss=10.176485061645508 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.418914794921875 | Batch Time=14.84375 +GPU:2 | Epoch: 20 | loss=10.231609344482422 | Batch Time=19.140625 +GPU:2 | Epoch: 20 | loss=10.186100006103516 | Batch Time=17.578125 +GPU:2 | Epoch: 20 | loss=10.366741180419922 | Batch Time=14.84375 +GPU:2 | Epoch: 20 | loss=10.126020431518555 | Batch Time=22.265625 +GPU:2 | Epoch: 20 | loss=10.02598762512207 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.358476638793945 | Batch Time=17.1875 +GPU:2 | Epoch: 20 | loss=10.545310020446777 | Batch Time=13.671875 +GPU:2 | Epoch: 20 | loss=10.034863471984863 | Batch Time=18.75 +GPU:2 | Epoch: 20 | loss=10.285232543945312 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.32358455657959 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.46351432800293 | Batch Time=12.890625 +GPU:2 | Epoch: 20 | loss=10.294113159179688 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.128019332885742 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.16907024383545 | Batch Time=16.40625 +GPU:2 | Epoch: 20 | loss=10.30709171295166 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.213431358337402 | Batch Time=17.578125 +GPU:2 | Epoch: 20 | loss=10.291070938110352 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.321277618408203 | Batch Time=16.40625 +GPU:2 | Epoch: 20 | loss=10.387899398803711 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.18035888671875 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.208090782165527 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.396814346313477 | Batch Time=12.109375 +GPU:2 | Epoch: 20 | loss=10.41010856628418 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.498640060424805 | Batch Time=11.71875 +GPU:2 | Epoch: 20 | loss=10.268087387084961 | Batch Time=13.671875 +GPU:2 | Epoch: 20 | loss=10.34941291809082 | Batch Time=19.140625 +GPU:2 | Epoch: 20 | loss=10.35136890411377 | Batch Time=13.671875 +GPU:2 | Epoch: 20 | loss=10.33210563659668 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.141507148742676 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.319229125976562 | Batch Time=17.1875 +GPU:2 | Epoch: 20 | loss=10.330936431884766 | Batch Time=14.453125 +GPU:2 | Epoch: 20 | loss=10.233675956726074 | Batch Time=14.453125 +GPU:2 | Epoch: 20 | loss=10.334044456481934 | Batch Time=14.0625 +GPU:2 | Epoch: 20 | loss=10.406230926513672 | Batch Time=13.28125 +GPU:2 | Epoch: 20 | loss=10.426323890686035 | Batch Time=12.5 +GPU:2 | Epoch: 20 | loss=10.116609573364258 | Batch Time=20.3125 +GPU:2 | Epoch: 20 | loss=10.21422004699707 | Batch Time=13.671875 +GPU:2 | Epoch: 20 | loss=10.262777328491211 | Batch Time=17.1875 +GPU:2 | Epoch: 20 | loss=10.026178359985352 | Batch Time=17.578125 +GPU:2 | Epoch: 20 | loss=10.243831634521484 | Batch Time=17.1875 +GPU:2 | Epoch: 20 | loss=10.436206817626953 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.389888763427734 | Batch Time=14.0625 +GPU:2 | Epoch: 20 | loss=10.382095336914062 | Batch Time=12.890625 +GPU:2 | Epoch: 20 | loss=10.333553314208984 | Batch Time=14.453125 +GPU:2 | Epoch: 20 | loss=10.249330520629883 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.254788398742676 | Batch Time=16.40625 +GPU:2 | Epoch: 20 | loss=10.1590576171875 | Batch Time=14.453125 +GPU:2 | Epoch: 20 | loss=10.56039047241211 | Batch Time=10.546875 +GPU:2 | Epoch: 20 | loss=10.196466445922852 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.177026748657227 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.248978614807129 | Batch Time=14.453125 +GPU:2 | Epoch: 20 | loss=10.350479125976562 | Batch Time=10.546875 +GPU:2 | Epoch: 20 | loss=10.194748878479004 | Batch Time=19.921875 +GPU:2 | Epoch: 20 | loss=10.317760467529297 | Batch Time=10.9375 +GPU:2 | Epoch: 20 | loss=10.151324272155762 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.24903392791748 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.206422805786133 | Batch Time=15.234375 +GPU:2 | Epoch: 20 | loss=10.394929885864258 | Batch Time=14.0625 +GPU:2 | Epoch: 20 | loss=10.229730606079102 | Batch Time=14.0625 +GPU:2 | Epoch: 20 | loss=10.24664306640625 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.132731437683105 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.386560440063477 | Batch Time=15.234375 +GPU:2 | Epoch: 20 | loss=10.218692779541016 | Batch Time=15.234375 +GPU:2 | Epoch: 20 | loss=10.304183959960938 | Batch Time=16.40625 +GPU:2 | Epoch: 20 | loss=10.230794906616211 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.290882110595703 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.184619903564453 | Batch Time=13.28125 +GPU:2 | Epoch: 20 | loss=10.15214729309082 | Batch Time=18.75 +GPU:2 | Epoch: 20 | loss=10.192289352416992 | Batch Time=14.84375 +GPU:2 | Epoch: 20 | loss=10.309608459472656 | Batch Time=12.890625 +GPU:2 | Epoch: 20 | loss=10.073357582092285 | Batch Time=19.921875 +GPU:2 | Epoch: 20 | loss=10.252214431762695 | Batch Time=17.1875 +GPU:2 | Epoch: 20 | loss=10.211601257324219 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.293944358825684 | Batch Time=14.0625 +GPU:2 | Epoch: 20 | loss=10.074748992919922 | Batch Time=17.1875 +GPU:2 | Epoch: 20 | loss=10.015113830566406 | Batch Time=19.140625 +GPU:2 | Epoch: 20 | loss=10.108409881591797 | Batch Time=14.453125 +GPU:2 | Epoch: 20 | loss=9.969669342041016 | Batch Time=19.140625 +GPU:2 | Epoch: 20 | loss=10.150226593017578 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.105859756469727 | Batch Time=12.890625 +GPU:2 | Epoch: 20 | loss=10.417994499206543 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.304664611816406 | Batch Time=14.0625 +GPU:2 | Epoch: 20 | loss=10.374366760253906 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.204435348510742 | Batch Time=18.359375 +GPU:2 | Epoch: 20 | loss=10.17233943939209 | Batch Time=18.359375 +GPU:2 | Epoch: 20 | loss=10.15577220916748 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.283472061157227 | Batch Time=11.328125 +GPU:2 | Epoch: 20 | loss=10.219785690307617 | Batch Time=17.578125 +GPU:2 | Epoch: 20 | loss=10.246792793273926 | Batch Time=15.625 +GPU:2 | Epoch: 20 | loss=10.214856147766113 | Batch Time=16.015625 +GPU:2 | Epoch: 20 | loss=10.396689414978027 | Batch Time=11.328125 +GPU:2 | Epoch: 20 | loss=10.034754753112793 | Batch Time=18.359375 +GPU:2 | Epoch: 20 | loss=9.982063293457031 | Batch Time=14.84375 +GPU:2 | Epoch: 20 | loss=10.365266799926758 | Batch Time=16.796875 +GPU:2 | Epoch: 20 | loss=10.29926586151123 | Batch Time=16.40625 +GPU:2 | Epoch: 20 | loss=10.247541427612305 | Batch Time=13.28125 +GPU:2 | Epoch: 20 | loss=10.429000854492188 | Batch Time=10.15625 +GPU:2 | Epoch: 20 | loss=10.362569808959961 | Batch Time=15.234375 +GPU:2 | Epoch: 20 | loss=10.10421085357666 | Batch Time=17.96875 +GPU:2 | Epoch: 20 | loss=10.304325103759766 | Batch Time=13.28125 +GPU:2 | Epoch: 20 | loss=10.026992797851562 | Batch Time=16.40625 +GPU:2 | Epoch: 20 | loss=10.350439071655273 | Batch Time=17.1875 +GPU:3 | Epoch: 20 | loss=10.347822189331055 | Batch Time=15.625 +GPU:3 | Epoch: 20 | loss=10.340237617492676 | Batch Time=15.625 +GPU:3 | Epoch: 20 | loss=10.545022964477539 | Batch Time=15.625 +GPU:3 | Epoch: 20 | loss=10.471842765808105 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.370986938476562 | Batch Time=17.1875 +GPU:3 | Epoch: 20 | loss=10.335704803466797 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.555293083190918 | Batch Time=9.765625 +GPU:3 | Epoch: 20 | loss=10.11474609375 | Batch Time=19.140625 +GPU:3 | Epoch: 20 | loss=10.163618087768555 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.226226806640625 | Batch Time=12.5 +GPU:3 | Epoch: 20 | loss=10.24362564086914 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.17423152923584 | Batch Time=15.625 +GPU:3 | Epoch: 20 | loss=10.237582206726074 | Batch Time=12.5 +GPU:3 | Epoch: 20 | loss=10.255631446838379 | Batch Time=15.234375 +GPU:3 | Epoch: 20 | loss=10.334081649780273 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.238698959350586 | Batch Time=16.015625 +GPU:3 | Epoch: 20 | loss=10.509267807006836 | Batch Time=14.453125 +GPU:3 | Epoch: 20 | loss=10.154695510864258 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.571802139282227 | Batch Time=10.9375 +GPU:3 | Epoch: 20 | loss=10.127946853637695 | Batch Time=16.015625 +GPU:3 | Epoch: 20 | loss=10.308357238769531 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.578064918518066 | Batch Time=9.375 +GPU:3 | Epoch: 20 | loss=10.402186393737793 | Batch Time=13.28125 +GPU:3 | Epoch: 20 | loss=10.199837684631348 | Batch Time=15.234375 +GPU:3 | Epoch: 20 | loss=10.301553726196289 | Batch Time=13.671875 +GPU:3 | Epoch: 20 | loss=10.50245475769043 | Batch Time=14.0625 +GPU:3 | Epoch: 20 | loss=10.378847122192383 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.125541687011719 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.279550552368164 | Batch Time=15.234375 +GPU:3 | Epoch: 20 | loss=10.315267562866211 | Batch Time=12.109375 +GPU:3 | Epoch: 20 | loss=10.108707427978516 | Batch Time=16.015625 +GPU:3 | Epoch: 20 | loss=10.41983699798584 | Batch Time=12.5 +GPU:3 | Epoch: 20 | loss=10.344164848327637 | Batch Time=14.0625 +GPU:3 | Epoch: 20 | loss=10.117827415466309 | Batch Time=13.671875 +GPU:3 | Epoch: 20 | loss=10.201658248901367 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.253291130065918 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.160359382629395 | Batch Time=13.28125 +GPU:3 | Epoch: 20 | loss=10.075294494628906 | Batch Time=18.75 +GPU:3 | Epoch: 20 | loss=10.254573822021484 | Batch Time=16.40625 +GPU:3 | Epoch: 20 | loss=10.186266899108887 | Batch Time=17.578125 +GPU:3 | Epoch: 20 | loss=10.326472282409668 | Batch Time=17.578125 +GPU:3 | Epoch: 20 | loss=10.390144348144531 | Batch Time=14.453125 +GPU:3 | Epoch: 20 | loss=10.232563972473145 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.11762809753418 | Batch Time=16.015625 +GPU:3 | Epoch: 20 | loss=10.086021423339844 | Batch Time=16.40625 +GPU:3 | Epoch: 20 | loss=10.346122741699219 | Batch Time=12.890625 +GPU:3 | Epoch: 20 | loss=10.287601470947266 | Batch Time=15.625 +GPU:3 | Epoch: 20 | loss=10.345802307128906 | Batch Time=15.234375 +GPU:3 | Epoch: 20 | loss=10.293294906616211 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.368852615356445 | Batch Time=17.578125 +GPU:3 | Epoch: 20 | loss=10.218425750732422 | Batch Time=14.0625 +GPU:3 | Epoch: 20 | loss=10.310729026794434 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.18624210357666 | Batch Time=16.40625 +GPU:3 | Epoch: 20 | loss=10.361449241638184 | Batch Time=14.453125 +GPU:3 | Epoch: 20 | loss=10.252659797668457 | Batch Time=11.71875 +GPU:3 | Epoch: 20 | loss=10.257227897644043 | Batch Time=15.625 +GPU:3 | Epoch: 20 | loss=10.283390045166016 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.346304893493652 | Batch Time=13.671875 +GPU:3 | Epoch: 20 | loss=10.171809196472168 | Batch Time=17.578125 +GPU:3 | Epoch: 20 | loss=10.293611526489258 | Batch Time=17.578125 +GPU:3 | Epoch: 20 | loss=10.1690034866333 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.365580558776855 | Batch Time=15.234375 +GPU:3 | Epoch: 20 | loss=10.39462661743164 | Batch Time=10.9375 +GPU:3 | Epoch: 20 | loss=10.340963363647461 | Batch Time=17.96875 +GPU:3 | Epoch: 20 | loss=10.425312995910645 | Batch Time=12.109375 +GPU:3 | Epoch: 20 | loss=10.095806121826172 | Batch Time=20.703125 +GPU:3 | Epoch: 20 | loss=10.37647819519043 | Batch Time=14.0625 +GPU:3 | Epoch: 20 | loss=10.196634292602539 | Batch Time=18.75 +GPU:3 | Epoch: 20 | loss=10.280425071716309 | Batch Time=13.28125 +GPU:3 | Epoch: 20 | loss=10.064058303833008 | Batch Time=19.53125 +GPU:3 | Epoch: 20 | loss=10.166888236999512 | Batch Time=12.5 +GPU:3 | Epoch: 20 | loss=10.236223220825195 | Batch Time=13.671875 +GPU:3 | Epoch: 20 | loss=10.192769050598145 | Batch Time=16.40625 +GPU:3 | Epoch: 20 | loss=10.451248168945312 | Batch Time=14.0625 +GPU:3 | Epoch: 20 | loss=10.308143615722656 | Batch Time=15.234375 +GPU:3 | Epoch: 20 | loss=10.008012771606445 | Batch Time=18.75 +GPU:3 | Epoch: 20 | loss=10.412376403808594 | Batch Time=12.5 +GPU:3 | Epoch: 20 | loss=10.436551094055176 | Batch Time=12.890625 +GPU:3 | Epoch: 20 | loss=10.152986526489258 | Batch Time=17.96875 +GPU:3 | Epoch: 20 | loss=10.19223403930664 | Batch Time=19.140625 +GPU:3 | Epoch: 20 | loss=10.435416221618652 | Batch Time=12.109375 +GPU:3 | Epoch: 20 | loss=10.236724853515625 | Batch Time=18.75 +GPU:3 | Epoch: 20 | loss=10.251258850097656 | Batch Time=17.96875 +GPU:3 | Epoch: 20 | loss=10.082847595214844 | Batch Time=17.578125 +GPU:3 | Epoch: 20 | loss=10.389823913574219 | Batch Time=15.625 +GPU:3 | Epoch: 20 | loss=10.209723472595215 | Batch Time=16.40625 +GPU:3 | Epoch: 20 | loss=10.26257610321045 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.318578720092773 | Batch Time=14.0625 +GPU:3 | Epoch: 20 | loss=10.379777908325195 | Batch Time=12.109375 +GPU:3 | Epoch: 20 | loss=10.017271041870117 | Batch Time=15.625 +GPU:3 | Epoch: 20 | loss=10.312042236328125 | Batch Time=14.0625 +GPU:3 | Epoch: 20 | loss=10.240547180175781 | Batch Time=11.71875 +GPU:3 | Epoch: 20 | loss=10.226683616638184 | Batch Time=16.015625 +GPU:3 | Epoch: 20 | loss=10.146529197692871 | Batch Time=16.40625 +GPU:3 | Epoch: 20 | loss=10.004888534545898 | Batch Time=18.359375 +GPU:3 | Epoch: 20 | loss=10.143494606018066 | Batch Time=17.578125 +GPU:3 | Epoch: 20 | loss=10.101046562194824 | Batch Time=17.1875 +GPU:3 | Epoch: 20 | loss=10.178709030151367 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.55329418182373 | Batch Time=11.328125 +GPU:3 | Epoch: 20 | loss=9.976118087768555 | Batch Time=17.96875 +GPU:3 | Epoch: 20 | loss=10.296895027160645 | Batch Time=13.671875 +GPU:3 | Epoch: 20 | loss=10.339253425598145 | Batch Time=14.453125 +GPU:3 | Epoch: 20 | loss=10.334526062011719 | Batch Time=12.109375 +GPU:3 | Epoch: 20 | loss=10.262861251831055 | Batch Time=12.890625 +GPU:3 | Epoch: 20 | loss=10.319503784179688 | Batch Time=14.0625 +GPU:3 | Epoch: 20 | loss=10.205646514892578 | Batch Time=17.96875 +GPU:3 | Epoch: 20 | loss=9.971259117126465 | Batch Time=16.40625 +GPU:3 | Epoch: 20 | loss=10.237710952758789 | Batch Time=16.015625 +GPU:3 | Epoch: 20 | loss=10.027336120605469 | Batch Time=17.1875 +GPU:3 | Epoch: 20 | loss=10.28207778930664 | Batch Time=15.234375 +GPU:3 | Epoch: 20 | loss=10.33163833618164 | Batch Time=13.28125 +GPU:3 | Epoch: 20 | loss=10.146411895751953 | Batch Time=19.53125 +GPU:3 | Epoch: 20 | loss=9.990501403808594 | Batch Time=17.96875 +GPU:3 | Epoch: 20 | loss=10.108051300048828 | Batch Time=16.796875 +GPU:3 | Epoch: 20 | loss=10.18107795715332 | Batch Time=14.453125 +GPU:3 | Epoch: 20 | loss=10.200851440429688 | Batch Time=16.015625 +GPU:3 | Epoch: 20 | loss=10.092134475708008 | Batch Time=14.84375 +GPU:3 | Epoch: 20 | loss=10.18742847442627 | Batch Time=17.1875 +GPU:3 | Epoch: 20 | loss=10.051717758178711 | Batch Time=17.1875 +GPU:3 | Epoch: 20 | loss=10.091239929199219 | Batch Time=21.875 +GPU:3 | Epoch: 20 | loss=10.106746673583984 | Batch Time=13.671875 +GPU:3 | Epoch: 20 | loss=10.171550750732422 | Batch Time=18.359375 +GPU:3 | Epoch: 20 | loss=9.936786651611328 | Batch Time=19.140625 +GPU:3 | Epoch: 20 | loss=10.325387001037598 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.317416191101074 | Batch Time=14.84375 +GPU:1 | Epoch: 20 | loss=10.405813217163086 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.400096893310547 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.126577377319336 | Batch Time=16.796875 +GPU:1 | Epoch: 20 | loss=10.395970344543457 | Batch Time=16.796875 +GPU:1 | Epoch: 20 | loss=10.365100860595703 | Batch Time=14.84375 +GPU:1 | Epoch: 20 | loss=10.461235046386719 | Batch Time=12.5 +GPU:1 | Epoch: 20 | loss=10.286389350891113 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.311079025268555 | Batch Time=15.625 +GPU:1 | Epoch: 20 | loss=10.44199275970459 | Batch Time=11.328125 +GPU:1 | Epoch: 20 | loss=10.371490478515625 | Batch Time=10.15625 +GPU:1 | Epoch: 20 | loss=10.482965469360352 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.300207138061523 | Batch Time=14.0625 +GPU:1 | Epoch: 20 | loss=10.277952194213867 | Batch Time=14.0625 +GPU:1 | Epoch: 20 | loss=10.162286758422852 | Batch Time=19.921875 +GPU:1 | Epoch: 20 | loss=10.281363487243652 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.564986228942871 | Batch Time=15.625 +GPU:1 | Epoch: 20 | loss=10.47446346282959 | Batch Time=10.9375 +GPU:1 | Epoch: 20 | loss=10.372251510620117 | Batch Time=12.109375 +GPU:1 | Epoch: 20 | loss=10.234445571899414 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.369522094726562 | Batch Time=14.84375 +GPU:1 | Epoch: 20 | loss=10.40214729309082 | Batch Time=16.40625 +GPU:1 | Epoch: 20 | loss=10.368593215942383 | Batch Time=16.796875 +GPU:1 | Epoch: 20 | loss=10.406803131103516 | Batch Time=12.109375 +GPU:1 | Epoch: 20 | loss=10.437307357788086 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.1536226272583 | Batch Time=12.890625 +GPU:1 | Epoch: 20 | loss=10.18178653717041 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.294707298278809 | Batch Time=17.1875 +GPU:1 | Epoch: 20 | loss=10.147823333740234 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.439599990844727 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.31197452545166 | Batch Time=17.1875 +GPU:1 | Epoch: 20 | loss=10.335556030273438 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.375019073486328 | Batch Time=14.0625 +GPU:1 | Epoch: 20 | loss=10.424800872802734 | Batch Time=12.109375 +GPU:1 | Epoch: 20 | loss=10.153694152832031 | Batch Time=17.1875 +GPU:1 | Epoch: 20 | loss=10.511690139770508 | Batch Time=12.5 +GPU:1 | Epoch: 20 | loss=10.297321319580078 | Batch Time=16.796875 +GPU:1 | Epoch: 20 | loss=10.196120262145996 | Batch Time=18.359375 +GPU:1 | Epoch: 20 | loss=10.20736312866211 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.267730712890625 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.177005767822266 | Batch Time=19.140625 +GPU:1 | Epoch: 20 | loss=10.252500534057617 | Batch Time=17.578125 +GPU:1 | Epoch: 20 | loss=9.996780395507812 | Batch Time=21.09375 +GPU:1 | Epoch: 20 | loss=10.304539680480957 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.316210746765137 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.232604026794434 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.401482582092285 | Batch Time=10.9375 +GPU:1 | Epoch: 20 | loss=10.20954704284668 | Batch Time=16.796875 +GPU:1 | Epoch: 20 | loss=10.206710815429688 | Batch Time=17.578125 +GPU:1 | Epoch: 20 | loss=10.34389877319336 | Batch Time=17.1875 +GPU:1 | Epoch: 20 | loss=10.320751190185547 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.156891822814941 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.203546524047852 | Batch Time=19.140625 +GPU:1 | Epoch: 20 | loss=10.31760025024414 | Batch Time=14.84375 +GPU:1 | Epoch: 20 | loss=10.252725601196289 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.293262481689453 | Batch Time=11.71875 +GPU:1 | Epoch: 20 | loss=10.351217269897461 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.108776092529297 | Batch Time=18.75 +GPU:1 | Epoch: 20 | loss=10.233378410339355 | Batch Time=16.796875 +GPU:1 | Epoch: 20 | loss=10.17890739440918 | Batch Time=18.359375 +GPU:1 | Epoch: 20 | loss=10.209773063659668 | Batch Time=18.75 +GPU:1 | Epoch: 20 | loss=10.285627365112305 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.052640914916992 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.244516372680664 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.331684112548828 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.16634750366211 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.180061340332031 | Batch Time=17.1875 +GPU:1 | Epoch: 20 | loss=10.345212936401367 | Batch Time=14.84375 +GPU:1 | Epoch: 20 | loss=10.201635360717773 | Batch Time=17.1875 +GPU:1 | Epoch: 20 | loss=10.111072540283203 | Batch Time=18.359375 +GPU:1 | Epoch: 20 | loss=10.517566680908203 | Batch Time=10.546875 +GPU:1 | Epoch: 20 | loss=10.186128616333008 | Batch Time=18.75 +GPU:1 | Epoch: 20 | loss=10.312424659729004 | Batch Time=16.40625 +GPU:1 | Epoch: 20 | loss=10.310041427612305 | Batch Time=14.0625 +GPU:1 | Epoch: 20 | loss=10.489232063293457 | Batch Time=11.71875 +GPU:1 | Epoch: 20 | loss=10.253719329833984 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.263151168823242 | Batch Time=16.40625 +GPU:1 | Epoch: 20 | loss=10.217823028564453 | Batch Time=15.625 +GPU:1 | Epoch: 20 | loss=10.11849594116211 | Batch Time=15.625 +GPU:1 | Epoch: 20 | loss=10.269817352294922 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.183066368103027 | Batch Time=14.84375 +GPU:1 | Epoch: 20 | loss=10.274002075195312 | Batch Time=19.53125 +GPU:1 | Epoch: 20 | loss=10.406497955322266 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.466808319091797 | Batch Time=12.109375 +GPU:1 | Epoch: 20 | loss=9.971739768981934 | Batch Time=21.875 +GPU:1 | Epoch: 20 | loss=10.071711540222168 | Batch Time=18.75 +GPU:1 | Epoch: 20 | loss=10.235721588134766 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.146878242492676 | Batch Time=15.625 +GPU:1 | Epoch: 20 | loss=10.289600372314453 | Batch Time=14.84375 +GPU:1 | Epoch: 20 | loss=10.084121704101562 | Batch Time=17.578125 +GPU:1 | Epoch: 20 | loss=10.33270263671875 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.382204055786133 | Batch Time=19.140625 +GPU:1 | Epoch: 20 | loss=10.335412979125977 | Batch Time=13.671875 +GPU:1 | Epoch: 20 | loss=10.209961891174316 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.335495948791504 | Batch Time=14.0625 +GPU:1 | Epoch: 20 | loss=10.186437606811523 | Batch Time=15.625 +GPU:1 | Epoch: 20 | loss=10.246536254882812 | Batch Time=16.015625 +GPU:1 | Epoch: 20 | loss=10.272197723388672 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.376474380493164 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.10774040222168 | Batch Time=20.703125 +GPU:1 | Epoch: 20 | loss=9.969385147094727 | Batch Time=16.40625 +GPU:1 | Epoch: 20 | loss=10.515798568725586 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.079788208007812 | Batch Time=16.40625 +GPU:1 | Epoch: 20 | loss=10.281314849853516 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.207172393798828 | Batch Time=17.578125 +GPU:1 | Epoch: 20 | loss=10.179288864135742 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=9.95247745513916 | Batch Time=16.40625 +GPU:1 | Epoch: 20 | loss=10.222792625427246 | Batch Time=16.40625 +GPU:1 | Epoch: 20 | loss=10.013971328735352 | Batch Time=19.140625 +GPU:1 | Epoch: 20 | loss=10.352004051208496 | Batch Time=11.71875 +GPU:1 | Epoch: 20 | loss=10.329463958740234 | Batch Time=14.84375 +GPU:1 | Epoch: 20 | loss=10.198604583740234 | Batch Time=13.671875 +GPU:1 | Epoch: 20 | loss=10.35118293762207 | Batch Time=20.703125 +GPU:1 | Epoch: 20 | loss=10.136865615844727 | Batch Time=13.28125 +GPU:1 | Epoch: 20 | loss=10.200689315795898 | Batch Time=15.234375 +GPU:1 | Epoch: 20 | loss=10.145212173461914 | Batch Time=20.3125 +GPU:1 | Epoch: 20 | loss=10.047220230102539 | Batch Time=21.875 +GPU:1 | Epoch: 20 | loss=10.295778274536133 | Batch Time=14.453125 +GPU:1 | Epoch: 20 | loss=10.074136734008789 | Batch Time=15.625 +GPU:1 | Epoch: 20 | loss=10.292057037353516 | Batch Time=18.75 +GPU:1 | Epoch: 20 | loss=10.213798522949219 | Batch Time=16.40625 +GPU:1 | Epoch: 20 | loss=10.107574462890625 | Batch Time=17.578125 +GPU:1 | Epoch: 20 | loss=10.218825340270996 | Batch Time=12.109375 +GPU:1 | Epoch: 20 | loss=10.112210273742676 | Batch Time=19.53125 +GPU:0 | Epoch: 20 | loss=10.23091983795166 | Batch Time=18.75 +GPU:0 | Epoch: 20 | loss=10.205507278442383 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.428400039672852 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.338847160339355 | Batch Time=12.890625 +GPU:0 | Epoch: 20 | loss=10.324088096618652 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.315982818603516 | Batch Time=14.84375 +GPU:0 | Epoch: 20 | loss=10.241266250610352 | Batch Time=16.015625 +GPU:0 | Epoch: 20 | loss=10.245009422302246 | Batch Time=17.96875 +GPU:0 | Epoch: 20 | loss=10.259780883789062 | Batch Time=14.0625 +GPU:0 | Epoch: 20 | loss=10.539121627807617 | Batch Time=10.9375 +GPU:0 | Epoch: 20 | loss=10.151582717895508 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=10.559106826782227 | Batch Time=13.671875 +GPU:0 | Epoch: 20 | loss=10.269087791442871 | Batch Time=14.453125 +GPU:0 | Epoch: 20 | loss=10.169073104858398 | Batch Time=16.40625 +GPU:0 | Epoch: 20 | loss=10.35551643371582 | Batch Time=12.890625 +GPU:0 | Epoch: 20 | loss=10.269460678100586 | Batch Time=16.015625 +GPU:0 | Epoch: 20 | loss=10.221121788024902 | Batch Time=17.96875 +GPU:0 | Epoch: 20 | loss=10.24652099609375 | Batch Time=16.796875 +GPU:0 | Epoch: 20 | loss=10.214800834655762 | Batch Time=19.53125 +GPU:0 | Epoch: 20 | loss=10.067659378051758 | Batch Time=18.359375 +GPU:0 | Epoch: 20 | loss=10.195501327514648 | Batch Time=18.75 +GPU:0 | Epoch: 20 | loss=10.286426544189453 | Batch Time=16.015625 +GPU:0 | Epoch: 20 | loss=10.27696418762207 | Batch Time=17.578125 +GPU:0 | Epoch: 20 | loss=10.198747634887695 | Batch Time=14.453125 +GPU:0 | Epoch: 20 | loss=10.40611457824707 | Batch Time=14.453125 +GPU:0 | Epoch: 20 | loss=10.282676696777344 | Batch Time=13.671875 +GPU:0 | Epoch: 20 | loss=10.340116500854492 | Batch Time=16.40625 +GPU:0 | Epoch: 20 | loss=10.174568176269531 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=10.454111099243164 | Batch Time=14.0625 +GPU:0 | Epoch: 20 | loss=10.209794998168945 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=10.17011833190918 | Batch Time=19.140625 +GPU:0 | Epoch: 20 | loss=10.391985893249512 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.244756698608398 | Batch Time=12.5 +GPU:0 | Epoch: 20 | loss=10.459100723266602 | Batch Time=16.796875 +GPU:0 | Epoch: 20 | loss=10.240768432617188 | Batch Time=18.75 +GPU:0 | Epoch: 20 | loss=10.400829315185547 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.32076358795166 | Batch Time=14.0625 +GPU:0 | Epoch: 20 | loss=10.242471694946289 | Batch Time=19.921875 +GPU:0 | Epoch: 20 | loss=10.344274520874023 | Batch Time=12.890625 +GPU:0 | Epoch: 20 | loss=10.212379455566406 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.344093322753906 | Batch Time=13.671875 +GPU:0 | Epoch: 20 | loss=10.346240043640137 | Batch Time=11.71875 +GPU:0 | Epoch: 20 | loss=10.1521635055542 | Batch Time=17.96875 +GPU:0 | Epoch: 20 | loss=10.181098937988281 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.250080108642578 | Batch Time=18.75 +GPU:0 | Epoch: 20 | loss=10.517400741577148 | Batch Time=11.71875 +GPU:0 | Epoch: 20 | loss=10.166123390197754 | Batch Time=19.140625 +GPU:0 | Epoch: 20 | loss=10.364781379699707 | Batch Time=11.328125 +GPU:0 | Epoch: 20 | loss=10.165350914001465 | Batch Time=12.890625 +GPU:0 | Epoch: 20 | loss=10.099563598632812 | Batch Time=16.015625 +GPU:0 | Epoch: 20 | loss=10.510698318481445 | Batch Time=14.0625 +GPU:0 | Epoch: 20 | loss=10.29205322265625 | Batch Time=13.671875 +GPU:0 | Epoch: 20 | loss=10.204320907592773 | Batch Time=14.84375 +GPU:0 | Epoch: 20 | loss=9.965200424194336 | Batch Time=19.921875 +GPU:0 | Epoch: 20 | loss=10.296178817749023 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.395486831665039 | Batch Time=13.28125 +GPU:0 | Epoch: 20 | loss=10.293472290039062 | Batch Time=14.0625 +GPU:0 | Epoch: 20 | loss=10.191631317138672 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.051453590393066 | Batch Time=17.578125 +GPU:0 | Epoch: 20 | loss=10.300142288208008 | Batch Time=12.890625 +GPU:0 | Epoch: 20 | loss=10.126188278198242 | Batch Time=19.140625 +GPU:0 | Epoch: 20 | loss=10.32090950012207 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.114572525024414 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.125276565551758 | Batch Time=22.65625 +GPU:0 | Epoch: 20 | loss=10.249755859375 | Batch Time=16.015625 +GPU:0 | Epoch: 20 | loss=10.397274017333984 | Batch Time=16.015625 +GPU:0 | Epoch: 20 | loss=10.366323471069336 | Batch Time=14.84375 +GPU:0 | Epoch: 20 | loss=10.154341697692871 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.07210922241211 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=10.096979141235352 | Batch Time=19.53125 +GPU:0 | Epoch: 20 | loss=10.252043724060059 | Batch Time=14.84375 +GPU:0 | Epoch: 20 | loss=10.432329177856445 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.25721549987793 | Batch Time=14.453125 +GPU:0 | Epoch: 20 | loss=10.2652587890625 | Batch Time=13.28125 +GPU:0 | Epoch: 20 | loss=10.4327974319458 | Batch Time=13.28125 +GPU:0 | Epoch: 20 | loss=10.125972747802734 | Batch Time=12.5 +GPU:0 | Epoch: 20 | loss=10.07841968536377 | Batch Time=18.359375 +GPU:0 | Epoch: 20 | loss=9.840176582336426 | Batch Time=19.53125 +GPU:0 | Epoch: 20 | loss=10.337127685546875 | Batch Time=17.96875 +GPU:0 | Epoch: 20 | loss=10.269184112548828 | Batch Time=14.0625 +GPU:0 | Epoch: 20 | loss=10.234546661376953 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.157485961914062 | Batch Time=19.921875 +GPU:0 | Epoch: 20 | loss=10.236799240112305 | Batch Time=12.5 +GPU:0 | Epoch: 20 | loss=10.235207557678223 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.340673446655273 | Batch Time=13.28125 +GPU:0 | Epoch: 20 | loss=10.172357559204102 | Batch Time=16.40625 +GPU:0 | Epoch: 20 | loss=10.303300857543945 | Batch Time=13.28125 +GPU:0 | Epoch: 20 | loss=10.206611633300781 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.166718482971191 | Batch Time=17.578125 +GPU:0 | Epoch: 20 | loss=10.3468017578125 | Batch Time=14.0625 +GPU:0 | Epoch: 20 | loss=10.209275245666504 | Batch Time=12.109375 +GPU:0 | Epoch: 20 | loss=10.257244110107422 | Batch Time=17.96875 +GPU:0 | Epoch: 20 | loss=10.276092529296875 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.448766708374023 | Batch Time=12.109375 +GPU:0 | Epoch: 20 | loss=9.983409881591797 | Batch Time=16.015625 +GPU:0 | Epoch: 20 | loss=10.364370346069336 | Batch Time=14.84375 +GPU:0 | Epoch: 20 | loss=10.258153915405273 | Batch Time=11.328125 +GPU:0 | Epoch: 20 | loss=10.321708679199219 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.100946426391602 | Batch Time=19.140625 +GPU:0 | Epoch: 20 | loss=10.269752502441406 | Batch Time=17.578125 +GPU:0 | Epoch: 20 | loss=10.094970703125 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=10.06096076965332 | Batch Time=17.96875 +GPU:0 | Epoch: 20 | loss=10.153459548950195 | Batch Time=17.578125 +GPU:0 | Epoch: 20 | loss=10.236029624938965 | Batch Time=14.84375 +GPU:0 | Epoch: 20 | loss=10.089681625366211 | Batch Time=16.40625 +GPU:0 | Epoch: 20 | loss=10.330987930297852 | Batch Time=13.671875 +GPU:0 | Epoch: 20 | loss=10.099180221557617 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.073022842407227 | Batch Time=13.28125 +GPU:0 | Epoch: 20 | loss=10.3194580078125 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=10.08953857421875 | Batch Time=18.359375 +GPU:0 | Epoch: 20 | loss=10.073540687561035 | Batch Time=18.75 +GPU:0 | Epoch: 20 | loss=10.052549362182617 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=10.073339462280273 | Batch Time=18.359375 +GPU:0 | Epoch: 20 | loss=10.17973518371582 | Batch Time=14.84375 +GPU:0 | Epoch: 20 | loss=10.063763618469238 | Batch Time=13.671875 +GPU:0 | Epoch: 20 | loss=10.123476028442383 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=10.300535202026367 | Batch Time=12.5 +GPU:0 | Epoch: 20 | loss=10.237720489501953 | Batch Time=12.890625 +GPU:0 | Epoch: 20 | loss=10.470226287841797 | Batch Time=12.5 +GPU:0 | Epoch: 20 | loss=10.322853088378906 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=10.338088035583496 | Batch Time=11.328125 +GPU:0 | Epoch: 20 | loss=10.129613876342773 | Batch Time=16.015625 +GPU:0 | Epoch: 20 | loss=10.050348281860352 | Batch Time=19.53125 +GPU:0 | Epoch: 20 | loss=9.992401123046875 | Batch Time=18.359375 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 20 | loss=10.276315689086914 | Batch Time=16.796875 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205418496, percent=68.9, used=169449369600, free=17078259712, active=191188910080, inactive=43961831424, buffers=444940288, cached=70595514368, shared=5661995008, slab=2959532032) +AFTER TRAIN LOOP: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205942784, percent=68.9, used=169448927232, free=17078784000, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +BEFORE VAL LOOP: GPU: 0 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205942784, percent=68.9, used=169448927232, free=17078784000, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +GPU:1 | Epoch: 20 | loss=10.151835441589355 | Batch Time=16.40625 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=80204902400, percent=68.9, used=169449885696, free=17077743616, active=191188910080, inactive=43961831424, buffers=444940288, cached=70595514368, shared=5661995008, slab=2959532032) +AFTER TRAIN LOOP: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205942784, percent=68.9, used=169448927232, free=17078784000, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +BEFORE PRUNE: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205942784, percent=68.9, used=169448927232, free=17078784000, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +Pruning Model: +AFTER PRUNE: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +STARTING TRAINING: Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +BEFORE TRAIN LOOP: Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +GPU:3 | Epoch: 20 | loss=10.123262405395508 | Batch Time=17.1875 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=80204902400, percent=68.9, used=169449885696, free=17077743616, active=191188910080, inactive=43961831424, buffers=444940288, cached=70595514368, shared=5661995008, slab=2959532032) +AFTER TRAIN LOOP: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205942784, percent=68.9, used=169448927232, free=17078784000, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +BEFORE PRUNE: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205942784, percent=68.9, used=169448927232, free=17078784000, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +Pruning Model: +AFTER PRUNE: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +STARTING TRAINING: Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +BEFORE TRAIN LOOP: Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +GPU:2 | Epoch: 20 | loss=10.22979736328125 | Batch Time=19.140625 +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205426688, percent=68.9, used=169449443328, free=17078267904, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +AFTER TRAIN LOOP: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205942784, percent=68.9, used=169448927232, free=17078784000, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +BEFORE PRUNE: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80205942784, percent=68.9, used=169448927232, free=17078784000, active=191188660224, inactive=43961798656, buffers=444940288, cached=70595432448, shared=5661913088, slab=2959523840) +Pruning Model: +AFTER PRUNE: Epoch 20 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +STARTING TRAINING: Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +BEFORE TRAIN LOOP: Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=80206987264, percent=68.9, used=169447768064, free=17077776384, active=191188238336, inactive=43963588608, buffers=444940288, cached=70597599232, shared=5661749248, slab=2959499264) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 20 | loss=3.7919671535491943 | Batch Time=23.828125 +GPU:0 | Epoch: 20 | loss=5.663386344909668 | Batch Time=6.25 +GPU:0 | Epoch: 20 | loss=4.276147365570068 | Batch Time=26.953125 +GPU:0 | Epoch: 20 | loss=3.542724609375 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=4.815751552581787 | Batch Time=9.375 +GPU:0 | Epoch: 20 | loss=3.7635841369628906 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=5.928250312805176 | Batch Time=3.125 +GPU:0 | Epoch: 20 | loss=4.676860809326172 | Batch Time=11.328125 +GPU:0 | Epoch: 20 | loss=4.945694923400879 | Batch Time=8.984375 +GPU:0 | Epoch: 20 | loss=5.753722667694092 | Batch Time=1.953125 +GPU:0 | Epoch: 20 | loss=5.51354455947876 | Batch Time=0.78125 +GPU:0 | Epoch: 20 | loss=4.709539413452148 | Batch Time=12.5 +GPU:0 | Epoch: 20 | loss=5.482210159301758 | Batch Time=8.59375 +GPU:0 | Epoch: 20 | loss=4.469051361083984 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=4.614912509918213 | Batch Time=7.8125 +GPU:0 | Epoch: 20 | loss=5.0106964111328125 | Batch Time=8.984375 +GPU:0 | Epoch: 20 | loss=4.655536651611328 | Batch Time=5.859375 +GPU:0 | Epoch: 20 | loss=3.587695837020874 | Batch Time=26.5625 +GPU:0 | Epoch: 20 | loss=4.682285308837891 | Batch Time=11.328125 +GPU:0 | Epoch: 20 | loss=3.477062702178955 | Batch Time=25.78125 +Model top1 Accuracy: 13.73 +Acc before rounding: 13.73 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 20 | loss=3.7919671535491943 | Batch Time=23.828125 +GPU:0 | Epoch: 20 | loss=5.663386344909668 | Batch Time=6.25 +GPU:0 | Epoch: 20 | loss=4.276147365570068 | Batch Time=26.953125 +GPU:0 | Epoch: 20 | loss=3.542724609375 | Batch Time=17.1875 +GPU:0 | Epoch: 20 | loss=4.815751552581787 | Batch Time=9.375 +GPU:0 | Epoch: 20 | loss=3.7635841369628906 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=5.928250312805176 | Batch Time=3.125 +GPU:0 | Epoch: 20 | loss=4.676860809326172 | Batch Time=11.328125 +GPU:0 | Epoch: 20 | loss=4.945694923400879 | Batch Time=8.984375 +GPU:0 | Epoch: 20 | loss=5.753722667694092 | Batch Time=1.953125 +GPU:0 | Epoch: 20 | loss=5.51354455947876 | Batch Time=0.78125 +GPU:0 | Epoch: 20 | loss=4.709539413452148 | Batch Time=12.5 +GPU:0 | Epoch: 20 | loss=5.482210159301758 | Batch Time=8.59375 +GPU:0 | Epoch: 20 | loss=4.469051361083984 | Batch Time=15.234375 +GPU:0 | Epoch: 20 | loss=4.614912509918213 | Batch Time=7.8125 +GPU:0 | Epoch: 20 | loss=5.0106964111328125 | Batch Time=8.984375 +GPU:0 | Epoch: 20 | loss=4.655536651611328 | Batch Time=5.859375 +GPU:0 | Epoch: 20 | loss=3.587695837020874 | Batch Time=26.5625 +GPU:0 | Epoch: 20 | loss=4.682285308837891 | Batch Time=11.328125 +GPU:0 | Epoch: 20 | loss=3.477062702178955 | Batch Time=25.78125 +Model top1 Accuracy: 13.73 +Acc after rounding: 13.73 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 20 | loss=4.79033899307251 | Batch Time=12.890625 +GPU:0 | Epoch: 20 | loss=4.826981067657471 | Batch Time=14.0625 +GPU:0 | Epoch: 20 | loss=4.643728733062744 | Batch Time=15.625 +GPU:0 | Epoch: 20 | loss=4.831057548522949 | Batch Time=17.578125 +Model top1 Accuracy: 11.85 +Validation Acc after rounding: 11.85 +AFTER VAL LOOP: GPU: 0 | Epoch 20 | Memory Usage: svmem(total=257568083968, available=62420627456, percent=75.8, used=187233918976, free=5454823424, active=212814614528, inactive=33852960768, buffers=454070272, cached=64425271296, shared=5662138368, slab=2954977280) +BEFORE PRUNE: Epoch 20 | Memory Usage: svmem(total=257568083968, available=62420627456, percent=75.8, used=187233918976, free=5454823424, active=212814614528, inactive=33852960768, buffers=454070272, cached=64425271296, shared=5662138368, slab=2954977280) +Pruning Model: +AFTER PRUNE: Epoch 20 | Memory Usage: svmem(total=257568083968, available=62420627456, percent=75.8, used=187233918976, free=5454823424, active=212814614528, inactive=33852960768, buffers=454070272, cached=64425271296, shared=5662138368, slab=2954977280) +Rounding model with scheme: naive +Model avg sparsity: 38.796290985279505 +GPU:0 | Epoch: 20 | Acc=13.73 | Epoch Time=19.115270467599235 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 20 | Memory Usage: svmem(total=257568083968, available=62420664320, percent=75.8, used=187233955840, free=5454823424, active=212814589952, inactive=33852944384, buffers=454086656, cached=64425218048, shared=5662060544, slab=2954977280) +STARTING TRAINING: Epoch 21 | Memory Usage: svmem(total=257568083968, available=62420664320, percent=75.8, used=187233955840, free=5454823424, active=212814589952, inactive=33852944384, buffers=454086656, cached=64425218048, shared=5662060544, slab=2954977280) +BEFORE TRAIN LOOP: Epoch 21 | Memory Usage: svmem(total=257568083968, available=62420664320, percent=75.8, used=187233955840, free=5454823424, active=212814589952, inactive=33852944384, buffers=454086656, cached=64425218048, shared=5662060544, slab=2954977280) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=62420664320, percent=75.8, used=187233955840, free=5454823424, active=212814589952, inactive=33852944384, buffers=454086656, cached=64425218048, shared=5662060544, slab=2954977280) +GPU:2 | Epoch: 21 | loss=10.132835388183594 | Batch Time=17.1875 +GPU:2 | Epoch: 21 | loss=9.72581672668457 | Batch Time=21.09375 +GPU:2 | Epoch: 21 | loss=10.253379821777344 | Batch Time=15.625 +GPU:2 | Epoch: 21 | loss=10.153266906738281 | Batch Time=15.625 +GPU:2 | Epoch: 21 | loss=10.15970230102539 | Batch Time=17.578125 +GPU:2 | Epoch: 21 | loss=10.008691787719727 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=9.980031967163086 | Batch Time=18.359375 +GPU:2 | Epoch: 21 | loss=10.309489250183105 | Batch Time=16.015625 +GPU:2 | Epoch: 21 | loss=9.97081184387207 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=10.330845832824707 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=10.36336898803711 | Batch Time=13.28125 +GPU:2 | Epoch: 21 | loss=10.188447952270508 | Batch Time=14.84375 +GPU:2 | Epoch: 21 | loss=9.888208389282227 | Batch Time=21.875 +GPU:2 | Epoch: 21 | loss=10.033981323242188 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.196584701538086 | Batch Time=14.84375 +GPU:2 | Epoch: 21 | loss=10.020772933959961 | Batch Time=17.578125 +GPU:2 | Epoch: 21 | loss=10.168521881103516 | Batch Time=15.234375 +GPU:2 | Epoch: 21 | loss=10.127416610717773 | Batch Time=16.015625 +GPU:2 | Epoch: 21 | loss=10.415373802185059 | Batch Time=12.5 +GPU:2 | Epoch: 21 | loss=9.946931838989258 | Batch Time=20.3125 +GPU:2 | Epoch: 21 | loss=10.29934310913086 | Batch Time=13.671875 +GPU:2 | Epoch: 21 | loss=10.17664909362793 | Batch Time=12.109375 +GPU:2 | Epoch: 21 | loss=10.107597351074219 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=10.291009902954102 | Batch Time=13.28125 +GPU:2 | Epoch: 21 | loss=10.068964004516602 | Batch Time=14.84375 +GPU:2 | Epoch: 21 | loss=10.2259521484375 | Batch Time=12.890625 +GPU:2 | Epoch: 21 | loss=10.35511589050293 | Batch Time=13.28125 +GPU:2 | Epoch: 21 | loss=10.232778549194336 | Batch Time=11.71875 +GPU:2 | Epoch: 21 | loss=10.258943557739258 | Batch Time=14.0625 +GPU:2 | Epoch: 21 | loss=10.164970397949219 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=9.9830322265625 | Batch Time=17.578125 +GPU:2 | Epoch: 21 | loss=10.260747909545898 | Batch Time=12.890625 +GPU:2 | Epoch: 21 | loss=10.093069076538086 | Batch Time=14.84375 +GPU:2 | Epoch: 21 | loss=10.030675888061523 | Batch Time=14.84375 +GPU:2 | Epoch: 21 | loss=10.168354988098145 | Batch Time=19.140625 +GPU:2 | Epoch: 21 | loss=10.012960433959961 | Batch Time=19.140625 +GPU:2 | Epoch: 21 | loss=10.186946868896484 | Batch Time=15.234375 +GPU:2 | Epoch: 21 | loss=10.222185134887695 | Batch Time=13.28125 +GPU:2 | Epoch: 21 | loss=9.967938423156738 | Batch Time=19.53125 +GPU:2 | Epoch: 21 | loss=10.232696533203125 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=10.301984786987305 | Batch Time=14.0625 +GPU:2 | Epoch: 21 | loss=10.23206615447998 | Batch Time=12.890625 +GPU:2 | Epoch: 21 | loss=9.88584041595459 | Batch Time=19.140625 +GPU:2 | Epoch: 21 | loss=10.132213592529297 | Batch Time=16.015625 +GPU:2 | Epoch: 21 | loss=10.061151504516602 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=10.017505645751953 | Batch Time=14.453125 +GPU:2 | Epoch: 21 | loss=10.066024780273438 | Batch Time=17.96875 +GPU:2 | Epoch: 21 | loss=10.129862785339355 | Batch Time=16.015625 +GPU:2 | Epoch: 21 | loss=10.237337112426758 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.06219482421875 | Batch Time=15.234375 +GPU:2 | Epoch: 21 | loss=10.258727073669434 | Batch Time=15.625 +GPU:2 | Epoch: 21 | loss=10.131217956542969 | Batch Time=19.140625 +GPU:2 | Epoch: 21 | loss=10.166604995727539 | Batch Time=14.453125 +GPU:2 | Epoch: 21 | loss=9.885757446289062 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=10.033056259155273 | Batch Time=17.1875 +GPU:2 | Epoch: 21 | loss=10.041751861572266 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=9.899992942810059 | Batch Time=19.53125 +GPU:2 | Epoch: 21 | loss=10.013148307800293 | Batch Time=19.53125 +GPU:2 | Epoch: 21 | loss=10.252089500427246 | Batch Time=17.1875 +GPU:2 | Epoch: 21 | loss=10.039634704589844 | Batch Time=18.359375 +GPU:2 | Epoch: 21 | loss=10.018619537353516 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.062292098999023 | Batch Time=20.703125 +GPU:2 | Epoch: 21 | loss=10.344566345214844 | Batch Time=13.28125 +GPU:2 | Epoch: 21 | loss=10.054740905761719 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=10.047870635986328 | Batch Time=14.84375 +GPU:2 | Epoch: 21 | loss=10.032600402832031 | Batch Time=19.140625 +GPU:2 | Epoch: 21 | loss=10.053930282592773 | Batch Time=19.921875 +GPU:2 | Epoch: 21 | loss=10.04307746887207 | Batch Time=16.015625 +GPU:2 | Epoch: 21 | loss=9.972513198852539 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=10.064949035644531 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.508628845214844 | Batch Time=12.5 +GPU:2 | Epoch: 21 | loss=10.08736801147461 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=9.987726211547852 | Batch Time=17.578125 +GPU:2 | Epoch: 21 | loss=9.889842987060547 | Batch Time=20.3125 +GPU:2 | Epoch: 21 | loss=10.105693817138672 | Batch Time=12.109375 +GPU:2 | Epoch: 21 | loss=10.012433052062988 | Batch Time=17.1875 +GPU:2 | Epoch: 21 | loss=10.101144790649414 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=10.156954765319824 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=10.353930473327637 | Batch Time=15.234375 +GPU:2 | Epoch: 21 | loss=10.333179473876953 | Batch Time=10.9375 +GPU:2 | Epoch: 21 | loss=9.950660705566406 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=10.087268829345703 | Batch Time=21.09375 +GPU:2 | Epoch: 21 | loss=9.984275817871094 | Batch Time=17.96875 +GPU:2 | Epoch: 21 | loss=10.029294967651367 | Batch Time=20.3125 +GPU:2 | Epoch: 21 | loss=10.00004768371582 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=10.153167724609375 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=9.942987442016602 | Batch Time=16.015625 +GPU:2 | Epoch: 21 | loss=9.97325325012207 | Batch Time=15.234375 +GPU:2 | Epoch: 21 | loss=10.337618827819824 | Batch Time=12.5 +GPU:2 | Epoch: 21 | loss=9.871170043945312 | Batch Time=19.921875 +GPU:2 | Epoch: 21 | loss=10.072944641113281 | Batch Time=13.28125 +GPU:2 | Epoch: 21 | loss=9.800019264221191 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=9.997496604919434 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=10.193136215209961 | Batch Time=15.234375 +GPU:2 | Epoch: 21 | loss=10.046497344970703 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.051376342773438 | Batch Time=15.625 +GPU:2 | Epoch: 21 | loss=9.921049118041992 | Batch Time=16.015625 +GPU:2 | Epoch: 21 | loss=10.139202117919922 | Batch Time=12.109375 +GPU:2 | Epoch: 21 | loss=10.346726417541504 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=9.971811294555664 | Batch Time=17.96875 +GPU:2 | Epoch: 21 | loss=10.147040367126465 | Batch Time=15.234375 +GPU:2 | Epoch: 21 | loss=10.183006286621094 | Batch Time=16.40625 +GPU:2 | Epoch: 21 | loss=10.08132553100586 | Batch Time=14.453125 +GPU:2 | Epoch: 21 | loss=10.065814018249512 | Batch Time=14.0625 +GPU:2 | Epoch: 21 | loss=10.14089584350586 | Batch Time=15.234375 +GPU:2 | Epoch: 21 | loss=9.984563827514648 | Batch Time=17.578125 +GPU:2 | Epoch: 21 | loss=9.980244636535645 | Batch Time=17.578125 +GPU:2 | Epoch: 21 | loss=9.905279159545898 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.032678604125977 | Batch Time=16.015625 +GPU:2 | Epoch: 21 | loss=9.909111022949219 | Batch Time=17.96875 +GPU:2 | Epoch: 21 | loss=10.018649101257324 | Batch Time=19.921875 +GPU:2 | Epoch: 21 | loss=10.175538063049316 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.005440711975098 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.086146354675293 | Batch Time=12.890625 +GPU:2 | Epoch: 21 | loss=9.959892272949219 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=10.045215606689453 | Batch Time=19.53125 +GPU:2 | Epoch: 21 | loss=9.761197090148926 | Batch Time=17.96875 +GPU:2 | Epoch: 21 | loss=10.163156509399414 | Batch Time=17.1875 +GPU:2 | Epoch: 21 | loss=10.086579322814941 | Batch Time=16.796875 +GPU:2 | Epoch: 21 | loss=10.016486167907715 | Batch Time=14.84375 +GPU:2 | Epoch: 21 | loss=9.97983455657959 | Batch Time=13.28125 +GPU:2 | Epoch: 21 | loss=9.973148345947266 | Batch Time=18.75 +GPU:2 | Epoch: 21 | loss=10.155479431152344 | Batch Time=14.84375 +GPU:2 | Epoch: 21 | loss=10.050398826599121 | Batch Time=19.140625 +GPU:2 | Epoch: 21 | loss=9.91610336303711 | Batch Time=17.96875 +GPU:0 | Epoch: 21 | loss=10.203624725341797 | Batch Time=14.453125 +GPU:0 | Epoch: 21 | loss=10.173823356628418 | Batch Time=16.40625 +GPU:0 | Epoch: 21 | loss=10.006782531738281 | Batch Time=17.1875 +GPU:0 | Epoch: 21 | loss=10.080028533935547 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=10.173661231994629 | Batch Time=16.015625 +GPU:0 | Epoch: 21 | loss=10.043598175048828 | Batch Time=13.671875 +GPU:0 | Epoch: 21 | loss=10.232065200805664 | Batch Time=12.109375 +GPU:0 | Epoch: 21 | loss=10.38364315032959 | Batch Time=12.109375 +GPU:0 | Epoch: 21 | loss=10.191770553588867 | Batch Time=13.28125 +GPU:0 | Epoch: 21 | loss=10.09776782989502 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=10.316629409790039 | Batch Time=12.890625 +GPU:0 | Epoch: 21 | loss=10.218221664428711 | Batch Time=12.890625 +GPU:0 | Epoch: 21 | loss=10.147735595703125 | Batch Time=15.234375 +GPU:0 | Epoch: 21 | loss=10.130949020385742 | Batch Time=14.453125 +GPU:0 | Epoch: 21 | loss=10.027985572814941 | Batch Time=14.84375 +GPU:0 | Epoch: 21 | loss=10.105819702148438 | Batch Time=18.75 +GPU:0 | Epoch: 21 | loss=10.100778579711914 | Batch Time=18.359375 +GPU:0 | Epoch: 21 | loss=10.131956100463867 | Batch Time=16.015625 +GPU:0 | Epoch: 21 | loss=10.15228271484375 | Batch Time=16.796875 +GPU:0 | Epoch: 21 | loss=10.067445755004883 | Batch Time=17.578125 +GPU:0 | Epoch: 21 | loss=10.153558731079102 | Batch Time=17.1875 +GPU:0 | Epoch: 21 | loss=9.928312301635742 | Batch Time=18.75 +GPU:0 | Epoch: 21 | loss=10.18695068359375 | Batch Time=20.3125 +GPU:0 | Epoch: 21 | loss=10.235587120056152 | Batch Time=12.109375 +GPU:0 | Epoch: 21 | loss=10.143310546875 | Batch Time=14.84375 +GPU:0 | Epoch: 21 | loss=10.32781982421875 | Batch Time=14.84375 +GPU:0 | Epoch: 21 | loss=9.983095169067383 | Batch Time=21.09375 +GPU:0 | Epoch: 21 | loss=9.966962814331055 | Batch Time=20.703125 +GPU:0 | Epoch: 21 | loss=10.079463958740234 | Batch Time=19.921875 +GPU:0 | Epoch: 21 | loss=10.258139610290527 | Batch Time=12.5 +GPU:0 | Epoch: 21 | loss=10.013368606567383 | Batch Time=19.921875 +GPU:0 | Epoch: 21 | loss=10.21432876586914 | Batch Time=14.453125 +GPU:0 | Epoch: 21 | loss=10.355138778686523 | Batch Time=13.28125 +GPU:0 | Epoch: 21 | loss=10.034610748291016 | Batch Time=16.796875 +GPU:0 | Epoch: 21 | loss=10.091634750366211 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=10.14385986328125 | Batch Time=16.40625 +GPU:0 | Epoch: 21 | loss=10.136590957641602 | Batch Time=16.015625 +GPU:0 | Epoch: 21 | loss=10.090517044067383 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=10.199884414672852 | Batch Time=14.84375 +GPU:0 | Epoch: 21 | loss=10.332136154174805 | Batch Time=12.109375 +GPU:0 | Epoch: 21 | loss=9.947122573852539 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=10.271724700927734 | Batch Time=16.40625 +GPU:0 | Epoch: 21 | loss=10.206748008728027 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=9.965327262878418 | Batch Time=20.703125 +GPU:0 | Epoch: 21 | loss=10.086783409118652 | Batch Time=16.796875 +GPU:0 | Epoch: 21 | loss=9.964794158935547 | Batch Time=16.40625 +GPU:0 | Epoch: 21 | loss=10.043746948242188 | Batch Time=17.96875 +GPU:0 | Epoch: 21 | loss=10.022014617919922 | Batch Time=17.578125 +GPU:0 | Epoch: 21 | loss=10.120296478271484 | Batch Time=14.84375 +GPU:0 | Epoch: 21 | loss=9.949447631835938 | Batch Time=19.53125 +GPU:0 | Epoch: 21 | loss=10.097232818603516 | Batch Time=16.796875 +GPU:0 | Epoch: 21 | loss=10.083645820617676 | Batch Time=14.453125 +GPU:0 | Epoch: 21 | loss=10.110553741455078 | Batch Time=12.890625 +GPU:0 | Epoch: 21 | loss=9.934707641601562 | Batch Time=21.484375 +GPU:0 | Epoch: 21 | loss=10.111268997192383 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=9.938254356384277 | Batch Time=19.53125 +GPU:0 | Epoch: 21 | loss=10.085609436035156 | Batch Time=17.578125 +GPU:0 | Epoch: 21 | loss=9.744264602661133 | Batch Time=23.4375 +GPU:0 | Epoch: 21 | loss=10.14738655090332 | Batch Time=13.28125 +GPU:0 | Epoch: 21 | loss=10.010660171508789 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=10.151008605957031 | Batch Time=16.40625 +GPU:0 | Epoch: 21 | loss=10.134956359863281 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=10.113283157348633 | Batch Time=15.234375 +GPU:0 | Epoch: 21 | loss=9.969552993774414 | Batch Time=18.75 +GPU:0 | Epoch: 21 | loss=10.112333297729492 | Batch Time=17.1875 +GPU:0 | Epoch: 21 | loss=9.987070083618164 | Batch Time=14.453125 +GPU:0 | Epoch: 21 | loss=10.333372116088867 | Batch Time=13.671875 +GPU:0 | Epoch: 21 | loss=10.332867622375488 | Batch Time=14.0625 +GPU:0 | Epoch: 21 | loss=9.898445129394531 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=9.95792293548584 | Batch Time=21.09375 +GPU:0 | Epoch: 21 | loss=10.233016967773438 | Batch Time=16.015625 +GPU:0 | Epoch: 21 | loss=10.260919570922852 | Batch Time=12.890625 +GPU:0 | Epoch: 21 | loss=9.886811256408691 | Batch Time=18.359375 +GPU:0 | Epoch: 21 | loss=10.273683547973633 | Batch Time=15.234375 +GPU:0 | Epoch: 21 | loss=9.986837387084961 | Batch Time=18.359375 +GPU:0 | Epoch: 21 | loss=10.134711265563965 | Batch Time=17.96875 +GPU:0 | Epoch: 21 | loss=10.02199935913086 | Batch Time=14.453125 +GPU:0 | Epoch: 21 | loss=10.415637016296387 | Batch Time=13.28125 +GPU:0 | Epoch: 21 | loss=9.925800323486328 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=9.92428207397461 | Batch Time=17.1875 +GPU:0 | Epoch: 21 | loss=10.081026077270508 | Batch Time=12.890625 +GPU:0 | Epoch: 21 | loss=10.006837844848633 | Batch Time=17.1875 +GPU:0 | Epoch: 21 | loss=10.110471725463867 | Batch Time=17.96875 +GPU:0 | Epoch: 21 | loss=10.032858848571777 | Batch Time=17.1875 +GPU:0 | Epoch: 21 | loss=10.124496459960938 | Batch Time=13.671875 +GPU:0 | Epoch: 21 | loss=9.843749046325684 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=9.964439392089844 | Batch Time=15.234375 +GPU:0 | Epoch: 21 | loss=10.054607391357422 | Batch Time=18.75 +GPU:0 | Epoch: 21 | loss=10.063589096069336 | Batch Time=16.015625 +GPU:0 | Epoch: 21 | loss=10.027711868286133 | Batch Time=16.015625 +GPU:0 | Epoch: 21 | loss=9.870378494262695 | Batch Time=19.921875 +GPU:0 | Epoch: 21 | loss=10.28952407836914 | Batch Time=14.0625 +GPU:0 | Epoch: 21 | loss=10.212651252746582 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=9.822662353515625 | Batch Time=19.53125 +GPU:0 | Epoch: 21 | loss=9.983318328857422 | Batch Time=16.40625 +GPU:0 | Epoch: 21 | loss=10.152887344360352 | Batch Time=14.453125 +GPU:0 | Epoch: 21 | loss=9.986417770385742 | Batch Time=17.96875 +GPU:0 | Epoch: 21 | loss=9.980067253112793 | Batch Time=14.0625 +GPU:0 | Epoch: 21 | loss=10.047727584838867 | Batch Time=16.796875 +GPU:0 | Epoch: 21 | loss=10.220199584960938 | Batch Time=15.234375 +GPU:0 | Epoch: 21 | loss=10.12734603881836 | Batch Time=17.1875 +GPU:0 | Epoch: 21 | loss=10.055152893066406 | Batch Time=21.875 +GPU:0 | Epoch: 21 | loss=10.05748462677002 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=10.231742858886719 | Batch Time=14.453125 +GPU:0 | Epoch: 21 | loss=10.017282485961914 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=9.989017486572266 | Batch Time=18.359375 +GPU:0 | Epoch: 21 | loss=9.965686798095703 | Batch Time=17.96875 +GPU:0 | Epoch: 21 | loss=10.095436096191406 | Batch Time=16.796875 +GPU:0 | Epoch: 21 | loss=9.898992538452148 | Batch Time=18.359375 +GPU:0 | Epoch: 21 | loss=10.016551971435547 | Batch Time=14.84375 +GPU:0 | Epoch: 21 | loss=10.113450050354004 | Batch Time=13.28125 +GPU:0 | Epoch: 21 | loss=10.153559684753418 | Batch Time=16.40625 +GPU:0 | Epoch: 21 | loss=10.08055305480957 | Batch Time=13.28125 +GPU:0 | Epoch: 21 | loss=10.05290699005127 | Batch Time=17.578125 +GPU:0 | Epoch: 21 | loss=10.2410888671875 | Batch Time=14.84375 +GPU:0 | Epoch: 21 | loss=10.06772232055664 | Batch Time=17.96875 +GPU:0 | Epoch: 21 | loss=10.058560371398926 | Batch Time=14.84375 +GPU:0 | Epoch: 21 | loss=9.672479629516602 | Batch Time=20.3125 +GPU:0 | Epoch: 21 | loss=10.110147476196289 | Batch Time=12.109375 +GPU:0 | Epoch: 21 | loss=10.10595989227295 | Batch Time=16.015625 +GPU:0 | Epoch: 21 | loss=9.977039337158203 | Batch Time=16.015625 +GPU:0 | Epoch: 21 | loss=10.118885040283203 | Batch Time=10.546875 +GPU:0 | Epoch: 21 | loss=9.765989303588867 | Batch Time=21.09375 +GPU:0 | Epoch: 21 | loss=10.085442543029785 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.16163444519043 | Batch Time=16.40625 +GPU:3 | Epoch: 21 | loss=10.275959014892578 | Batch Time=14.453125 +GPU:3 | Epoch: 21 | loss=10.361074447631836 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.132498741149902 | Batch Time=16.40625 +GPU:3 | Epoch: 21 | loss=10.134831428527832 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.443695068359375 | Batch Time=12.890625 +GPU:3 | Epoch: 21 | loss=10.22701644897461 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.257301330566406 | Batch Time=14.453125 +GPU:3 | Epoch: 21 | loss=10.033248901367188 | Batch Time=19.921875 +GPU:3 | Epoch: 21 | loss=10.043682098388672 | Batch Time=17.96875 +GPU:3 | Epoch: 21 | loss=10.213785171508789 | Batch Time=17.578125 +GPU:3 | Epoch: 21 | loss=10.046735763549805 | Batch Time=18.75 +GPU:3 | Epoch: 21 | loss=10.375760078430176 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.19267463684082 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.390382766723633 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.033676147460938 | Batch Time=18.75 +GPU:3 | Epoch: 21 | loss=10.299238204956055 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.258918762207031 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.269925117492676 | Batch Time=17.96875 +GPU:3 | Epoch: 21 | loss=10.198631286621094 | Batch Time=16.015625 +GPU:3 | Epoch: 21 | loss=10.083606719970703 | Batch Time=19.140625 +GPU:3 | Epoch: 21 | loss=10.166114807128906 | Batch Time=12.890625 +GPU:3 | Epoch: 21 | loss=10.494338035583496 | Batch Time=12.890625 +GPU:3 | Epoch: 21 | loss=10.047318458557129 | Batch Time=16.40625 +GPU:3 | Epoch: 21 | loss=10.272037506103516 | Batch Time=12.109375 +GPU:3 | Epoch: 21 | loss=10.204341888427734 | Batch Time=12.890625 +GPU:3 | Epoch: 21 | loss=10.089590072631836 | Batch Time=17.1875 +GPU:3 | Epoch: 21 | loss=9.999038696289062 | Batch Time=18.359375 +GPU:3 | Epoch: 21 | loss=10.191671371459961 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.250358581542969 | Batch Time=14.84375 +GPU:3 | Epoch: 21 | loss=10.219619750976562 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.062862396240234 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.183847427368164 | Batch Time=12.5 +GPU:3 | Epoch: 21 | loss=10.087059020996094 | Batch Time=14.453125 +GPU:3 | Epoch: 21 | loss=10.291555404663086 | Batch Time=12.109375 +GPU:3 | Epoch: 21 | loss=10.019386291503906 | Batch Time=21.09375 +GPU:3 | Epoch: 21 | loss=10.081897735595703 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.218070983886719 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.433735847473145 | Batch Time=12.5 +GPU:3 | Epoch: 21 | loss=10.338153839111328 | Batch Time=12.890625 +GPU:3 | Epoch: 21 | loss=10.151920318603516 | Batch Time=12.109375 +GPU:3 | Epoch: 21 | loss=9.95556354522705 | Batch Time=17.578125 +GPU:3 | Epoch: 21 | loss=10.169293403625488 | Batch Time=16.40625 +GPU:3 | Epoch: 21 | loss=10.079032897949219 | Batch Time=17.578125 +GPU:3 | Epoch: 21 | loss=10.221824645996094 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.032389640808105 | Batch Time=14.84375 +GPU:3 | Epoch: 21 | loss=10.246208190917969 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.333126068115234 | Batch Time=14.84375 +GPU:3 | Epoch: 21 | loss=10.101776123046875 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.08920669555664 | Batch Time=12.890625 +GPU:3 | Epoch: 21 | loss=10.107429504394531 | Batch Time=16.796875 +GPU:3 | Epoch: 21 | loss=10.16750431060791 | Batch Time=16.015625 +GPU:3 | Epoch: 21 | loss=10.288372039794922 | Batch Time=13.28125 +GPU:3 | Epoch: 21 | loss=9.984535217285156 | Batch Time=18.359375 +GPU:3 | Epoch: 21 | loss=10.056131362915039 | Batch Time=18.75 +GPU:3 | Epoch: 21 | loss=9.934192657470703 | Batch Time=21.09375 +GPU:3 | Epoch: 21 | loss=10.161043167114258 | Batch Time=13.671875 +GPU:3 | Epoch: 21 | loss=10.051900863647461 | Batch Time=14.84375 +GPU:3 | Epoch: 21 | loss=10.169174194335938 | Batch Time=12.890625 +GPU:3 | Epoch: 21 | loss=10.119760513305664 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.005243301391602 | Batch Time=18.75 +GPU:3 | Epoch: 21 | loss=10.043901443481445 | Batch Time=20.3125 +GPU:3 | Epoch: 21 | loss=10.131523132324219 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.180112838745117 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.154169082641602 | Batch Time=16.015625 +GPU:3 | Epoch: 21 | loss=9.904735565185547 | Batch Time=19.140625 +GPU:3 | Epoch: 21 | loss=10.13995361328125 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.124494552612305 | Batch Time=14.453125 +GPU:3 | Epoch: 21 | loss=10.331360816955566 | Batch Time=14.453125 +GPU:3 | Epoch: 21 | loss=10.073003768920898 | Batch Time=17.1875 +GPU:3 | Epoch: 21 | loss=10.266786575317383 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.343640327453613 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.157428741455078 | Batch Time=18.359375 +GPU:3 | Epoch: 21 | loss=9.946798324584961 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.197061538696289 | Batch Time=11.71875 +GPU:3 | Epoch: 21 | loss=10.172835350036621 | Batch Time=11.71875 +GPU:3 | Epoch: 21 | loss=10.248615264892578 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.028488159179688 | Batch Time=20.3125 +GPU:3 | Epoch: 21 | loss=10.106544494628906 | Batch Time=16.40625 +GPU:3 | Epoch: 21 | loss=10.041955947875977 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.092626571655273 | Batch Time=17.578125 +GPU:3 | Epoch: 21 | loss=10.149295806884766 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=9.875478744506836 | Batch Time=18.75 +GPU:3 | Epoch: 21 | loss=9.99882698059082 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=9.918970108032227 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=9.76899242401123 | Batch Time=23.4375 +GPU:3 | Epoch: 21 | loss=10.08035945892334 | Batch Time=16.015625 +GPU:3 | Epoch: 21 | loss=10.100711822509766 | Batch Time=19.140625 +GPU:3 | Epoch: 21 | loss=10.167238235473633 | Batch Time=13.671875 +GPU:3 | Epoch: 21 | loss=10.02940845489502 | Batch Time=18.359375 +GPU:3 | Epoch: 21 | loss=10.052392959594727 | Batch Time=17.1875 +GPU:3 | Epoch: 21 | loss=10.190906524658203 | Batch Time=13.671875 +GPU:3 | Epoch: 21 | loss=10.16746711730957 | Batch Time=13.28125 +GPU:3 | Epoch: 21 | loss=10.086774826049805 | Batch Time=16.796875 +GPU:3 | Epoch: 21 | loss=10.162105560302734 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=10.108491897583008 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.022388458251953 | Batch Time=21.484375 +GPU:3 | Epoch: 21 | loss=10.117087364196777 | Batch Time=16.796875 +GPU:3 | Epoch: 21 | loss=9.933582305908203 | Batch Time=18.75 +GPU:3 | Epoch: 21 | loss=10.174600601196289 | Batch Time=14.84375 +GPU:3 | Epoch: 21 | loss=10.088825225830078 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.044008255004883 | Batch Time=17.96875 +GPU:3 | Epoch: 21 | loss=10.201759338378906 | Batch Time=14.0625 +GPU:3 | Epoch: 21 | loss=10.075549125671387 | Batch Time=17.96875 +GPU:3 | Epoch: 21 | loss=9.931846618652344 | Batch Time=17.96875 +GPU:3 | Epoch: 21 | loss=9.991859436035156 | Batch Time=13.671875 +GPU:3 | Epoch: 21 | loss=10.089889526367188 | Batch Time=19.140625 +GPU:3 | Epoch: 21 | loss=10.196846008300781 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.201708793640137 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.059752464294434 | Batch Time=14.84375 +GPU:3 | Epoch: 21 | loss=10.143485069274902 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=9.755071640014648 | Batch Time=23.046875 +GPU:3 | Epoch: 21 | loss=10.272249221801758 | Batch Time=14.453125 +GPU:3 | Epoch: 21 | loss=10.317176818847656 | Batch Time=12.109375 +GPU:3 | Epoch: 21 | loss=10.011695861816406 | Batch Time=19.140625 +GPU:3 | Epoch: 21 | loss=10.411184310913086 | Batch Time=12.890625 +GPU:3 | Epoch: 21 | loss=10.107283592224121 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.22016429901123 | Batch Time=15.234375 +GPU:3 | Epoch: 21 | loss=9.967187881469727 | Batch Time=15.625 +GPU:3 | Epoch: 21 | loss=10.060284614562988 | Batch Time=19.140625 +GPU:3 | Epoch: 21 | loss=10.214639663696289 | Batch Time=16.015625 +GPU:3 | Epoch: 21 | loss=9.77349853515625 | Batch Time=18.75 +GPU:3 | Epoch: 21 | loss=9.998556137084961 | Batch Time=18.75 +GPU:3 | Epoch: 21 | loss=9.623285293579102 | Batch Time=20.703125 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 21 | loss=10.471115112304688 | Batch Time=13.671875 +GPU:1 | Epoch: 21 | loss=9.942169189453125 | Batch Time=21.875 +GPU:1 | Epoch: 21 | loss=10.071146965026855 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=9.963445663452148 | Batch Time=18.359375 +GPU:1 | Epoch: 21 | loss=10.250563621520996 | Batch Time=14.84375 +GPU:1 | Epoch: 21 | loss=10.130324363708496 | Batch Time=14.453125 +GPU:1 | Epoch: 21 | loss=10.257354736328125 | Batch Time=14.453125 +GPU:1 | Epoch: 21 | loss=10.009344100952148 | Batch Time=20.3125 +GPU:1 | Epoch: 21 | loss=10.100994110107422 | Batch Time=15.234375 +GPU:1 | Epoch: 21 | loss=10.132343292236328 | Batch Time=16.40625 +GPU:1 | Epoch: 21 | loss=10.094773292541504 | Batch Time=13.671875 +GPU:1 | Epoch: 21 | loss=10.320453643798828 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.04489517211914 | Batch Time=20.703125 +GPU:1 | Epoch: 21 | loss=10.477775573730469 | Batch Time=10.9375 +GPU:1 | Epoch: 21 | loss=10.20669937133789 | Batch Time=15.625 +GPU:1 | Epoch: 21 | loss=10.289566040039062 | Batch Time=12.5 +GPU:1 | Epoch: 21 | loss=10.143558502197266 | Batch Time=17.1875 +GPU:1 | Epoch: 21 | loss=10.248720169067383 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=10.086221694946289 | Batch Time=18.75 +GPU:1 | Epoch: 21 | loss=10.159990310668945 | Batch Time=16.40625 +GPU:1 | Epoch: 21 | loss=10.0809326171875 | Batch Time=19.53125 +GPU:1 | Epoch: 21 | loss=10.048248291015625 | Batch Time=19.140625 +GPU:1 | Epoch: 21 | loss=9.960481643676758 | Batch Time=16.796875 +GPU:1 | Epoch: 21 | loss=10.068184852600098 | Batch Time=15.625 +GPU:1 | Epoch: 21 | loss=9.951211929321289 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=10.182595252990723 | Batch Time=12.5 +GPU:1 | Epoch: 21 | loss=10.181174278259277 | Batch Time=17.578125 +GPU:1 | Epoch: 21 | loss=10.041011810302734 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=10.093764305114746 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=10.195207595825195 | Batch Time=12.890625 +GPU:1 | Epoch: 21 | loss=10.19518756866455 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.266773223876953 | Batch Time=12.5 +GPU:1 | Epoch: 21 | loss=10.110130310058594 | Batch Time=14.84375 +GPU:1 | Epoch: 21 | loss=10.220951080322266 | Batch Time=16.796875 +GPU:1 | Epoch: 21 | loss=10.389232635498047 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.250038146972656 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=10.132064819335938 | Batch Time=17.1875 +GPU:1 | Epoch: 21 | loss=10.093791961669922 | Batch Time=18.75 +GPU:1 | Epoch: 21 | loss=10.061403274536133 | Batch Time=14.0625 +GPU:1 | Epoch: 21 | loss=10.05412483215332 | Batch Time=16.40625 +GPU:1 | Epoch: 21 | loss=10.117733001708984 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=10.159523010253906 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=9.932613372802734 | Batch Time=19.53125 +GPU:1 | Epoch: 21 | loss=9.906734466552734 | Batch Time=19.921875 +GPU:1 | Epoch: 21 | loss=10.005237579345703 | Batch Time=15.234375 +GPU:1 | Epoch: 21 | loss=10.06307601928711 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=10.064502716064453 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=9.989481925964355 | Batch Time=16.796875 +GPU:1 | Epoch: 21 | loss=10.231091499328613 | Batch Time=14.84375 +GPU:1 | Epoch: 21 | loss=10.196654319763184 | Batch Time=15.625 +GPU:1 | Epoch: 21 | loss=10.174448013305664 | Batch Time=14.0625 +GPU:1 | Epoch: 21 | loss=10.008989334106445 | Batch Time=15.234375 +GPU:1 | Epoch: 21 | loss=10.00009536743164 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=10.056835174560547 | Batch Time=18.75 +GPU:1 | Epoch: 21 | loss=9.98365592956543 | Batch Time=18.75 +GPU:1 | Epoch: 21 | loss=10.381888389587402 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.028541564941406 | Batch Time=16.40625 +GPU:1 | Epoch: 21 | loss=10.165416717529297 | Batch Time=16.796875 +GPU:1 | Epoch: 21 | loss=10.137884140014648 | Batch Time=14.84375 +GPU:1 | Epoch: 21 | loss=10.18814468383789 | Batch Time=12.890625 +GPU:1 | Epoch: 21 | loss=10.124313354492188 | Batch Time=13.671875 +GPU:1 | Epoch: 21 | loss=9.993146896362305 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=10.090397834777832 | Batch Time=15.234375 +GPU:1 | Epoch: 21 | loss=10.128697395324707 | Batch Time=13.671875 +GPU:1 | Epoch: 21 | loss=9.997570037841797 | Batch Time=18.359375 +GPU:1 | Epoch: 21 | loss=10.227201461791992 | Batch Time=16.796875 +GPU:1 | Epoch: 21 | loss=10.081262588500977 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.037246704101562 | Batch Time=20.703125 +GPU:1 | Epoch: 21 | loss=9.934459686279297 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=10.03874397277832 | Batch Time=16.796875 +GPU:1 | Epoch: 21 | loss=10.155069351196289 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.306950569152832 | Batch Time=13.671875 +GPU:1 | Epoch: 21 | loss=10.14877700805664 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.275030136108398 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=9.994808197021484 | Batch Time=16.796875 +GPU:1 | Epoch: 21 | loss=10.206676483154297 | Batch Time=14.0625 +GPU:1 | Epoch: 21 | loss=10.139833450317383 | Batch Time=14.84375 +GPU:1 | Epoch: 21 | loss=10.051070213317871 | Batch Time=18.75 +GPU:1 | Epoch: 21 | loss=10.220611572265625 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.318780899047852 | Batch Time=17.578125 +GPU:1 | Epoch: 21 | loss=10.175947189331055 | Batch Time=12.109375 +GPU:1 | Epoch: 21 | loss=10.00214958190918 | Batch Time=15.234375 +GPU:1 | Epoch: 21 | loss=9.964433670043945 | Batch Time=13.671875 +GPU:1 | Epoch: 21 | loss=9.959989547729492 | Batch Time=16.40625 +GPU:1 | Epoch: 21 | loss=10.030061721801758 | Batch Time=19.921875 +GPU:1 | Epoch: 21 | loss=10.16737174987793 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=9.977399826049805 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=10.17325210571289 | Batch Time=19.921875 +GPU:1 | Epoch: 21 | loss=10.076244354248047 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=10.25123405456543 | Batch Time=13.28125 +GPU:1 | Epoch: 21 | loss=10.146280288696289 | Batch Time=15.625 +GPU:1 | Epoch: 21 | loss=9.920914649963379 | Batch Time=17.1875 +GPU:1 | Epoch: 21 | loss=10.014996528625488 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=9.981708526611328 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=10.254817962646484 | Batch Time=14.84375 +GPU:1 | Epoch: 21 | loss=10.08103084564209 | Batch Time=17.1875 +GPU:1 | Epoch: 21 | loss=10.14472770690918 | Batch Time=14.84375 +GPU:1 | Epoch: 21 | loss=9.844346046447754 | Batch Time=23.046875 +GPU:1 | Epoch: 21 | loss=10.034903526306152 | Batch Time=16.40625 +GPU:1 | Epoch: 21 | loss=9.962718963623047 | Batch Time=17.578125 +GPU:1 | Epoch: 21 | loss=10.175307273864746 | Batch Time=16.40625 +GPU:1 | Epoch: 21 | loss=10.171005249023438 | Batch Time=15.625 +GPU:1 | Epoch: 21 | loss=9.990187644958496 | Batch Time=15.625 +GPU:1 | Epoch: 21 | loss=10.00875473022461 | Batch Time=14.84375 +GPU:1 | Epoch: 21 | loss=9.972726821899414 | Batch Time=16.40625 +GPU:1 | Epoch: 21 | loss=9.966812133789062 | Batch Time=14.453125 +GPU:1 | Epoch: 21 | loss=9.935298919677734 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=9.939061164855957 | Batch Time=21.484375 +GPU:1 | Epoch: 21 | loss=10.023151397705078 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=10.065169334411621 | Batch Time=17.578125 +GPU:1 | Epoch: 21 | loss=9.973237991333008 | Batch Time=18.359375 +GPU:1 | Epoch: 21 | loss=9.846929550170898 | Batch Time=21.875 +GPU:1 | Epoch: 21 | loss=9.944740295410156 | Batch Time=19.140625 +GPU:1 | Epoch: 21 | loss=10.317713737487793 | Batch Time=10.15625 +GPU:1 | Epoch: 21 | loss=10.156342506408691 | Batch Time=14.0625 +GPU:1 | Epoch: 21 | loss=9.942989349365234 | Batch Time=17.578125 +GPU:1 | Epoch: 21 | loss=9.828524589538574 | Batch Time=23.828125 +GPU:1 | Epoch: 21 | loss=10.32330322265625 | Batch Time=15.234375 +GPU:1 | Epoch: 21 | loss=10.00402545928955 | Batch Time=18.75 +GPU:1 | Epoch: 21 | loss=9.897554397583008 | Batch Time=18.75 +GPU:1 | Epoch: 21 | loss=10.230300903320312 | Batch Time=12.5 +GPU:1 | Epoch: 21 | loss=9.945417404174805 | Batch Time=17.96875 +GPU:1 | Epoch: 21 | loss=10.15699577331543 | Batch Time=16.015625 +GPU:1 | Epoch: 21 | loss=9.925329208374023 | Batch Time=19.921875 +GPU:1 | Epoch: 21 | loss=9.838775634765625 | Batch Time=18.359375 +GPU:0 | Epoch: 21 | loss=9.835793495178223 | Batch Time=23.4375 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661008384, percent=71.8, used=176993542144, free=16894754816, active=193668988928, inactive=41809383424, buffers=408702976, cached=63271084032, shared=5662109696, slab=2744836096) +AFTER TRAIN LOOP: Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +BEFORE VAL LOOP: GPU: 0 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +GPU:3 | Epoch: 21 | loss=10.011502265930176 | Batch Time=16.40625 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661008384, percent=71.8, used=176993542144, free=16894754816, active=193668988928, inactive=41809383424, buffers=408702976, cached=63271084032, shared=5662109696, slab=2744836096) +AFTER TRAIN LOOP: Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +STARTING TRAINING: Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +BEFORE TRAIN LOOP: Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661008384, percent=71.8, used=176993542144, free=16894754816, active=193668988928, inactive=41809383424, buffers=408702976, cached=63271084032, shared=5662109696, slab=2744836096) +AFTER TRAIN LOOP: Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +STARTING TRAINING: Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +BEFORE TRAIN LOOP: Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +AFTER TRAIN LOOP: Epoch 21 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +STARTING TRAINING: Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +BEFORE TRAIN LOOP: Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=72661311488, percent=71.8, used=176993406976, free=16895053824, active=193668702208, inactive=41809321984, buffers=408702976, cached=63270920192, shared=5661945856, slab=2744803328) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 21 | loss=3.4895455837249756 | Batch Time=25.390625 +GPU:0 | Epoch: 21 | loss=4.61836051940918 | Batch Time=12.109375 +GPU:0 | Epoch: 21 | loss=4.243342399597168 | Batch Time=23.4375 +GPU:0 | Epoch: 21 | loss=3.6337172985076904 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=4.576021194458008 | Batch Time=6.25 +GPU:0 | Epoch: 21 | loss=3.9322524070739746 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=4.463804721832275 | Batch Time=10.9375 +GPU:0 | Epoch: 21 | loss=5.024008750915527 | Batch Time=5.859375 +GPU:0 | Epoch: 21 | loss=4.9777727127075195 | Batch Time=6.640625 +GPU:0 | Epoch: 21 | loss=5.6436381340026855 | Batch Time=2.34375 +GPU:0 | Epoch: 21 | loss=5.312105655670166 | Batch Time=1.953125 +GPU:0 | Epoch: 21 | loss=4.447490692138672 | Batch Time=20.703125 +GPU:0 | Epoch: 21 | loss=5.450658321380615 | Batch Time=4.296875 +GPU:0 | Epoch: 21 | loss=4.398509502410889 | Batch Time=17.578125 +GPU:0 | Epoch: 21 | loss=4.599035263061523 | Batch Time=10.15625 +GPU:0 | Epoch: 21 | loss=4.9021382331848145 | Batch Time=11.71875 +GPU:0 | Epoch: 21 | loss=4.436906814575195 | Batch Time=10.9375 +GPU:0 | Epoch: 21 | loss=3.55283784866333 | Batch Time=24.609375 +GPU:0 | Epoch: 21 | loss=3.8511040210723877 | Batch Time=18.359375 +GPU:0 | Epoch: 21 | loss=3.376436710357666 | Batch Time=27.734375 +Model top1 Accuracy: 16.504 +Acc before rounding: 16.504 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 21 | loss=3.4895455837249756 | Batch Time=25.390625 +GPU:0 | Epoch: 21 | loss=4.61836051940918 | Batch Time=12.109375 +GPU:0 | Epoch: 21 | loss=4.243342399597168 | Batch Time=23.4375 +GPU:0 | Epoch: 21 | loss=3.6337172985076904 | Batch Time=19.140625 +GPU:0 | Epoch: 21 | loss=4.576021194458008 | Batch Time=6.25 +GPU:0 | Epoch: 21 | loss=3.9322524070739746 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=4.463804721832275 | Batch Time=10.9375 +GPU:0 | Epoch: 21 | loss=5.024008750915527 | Batch Time=5.859375 +GPU:0 | Epoch: 21 | loss=4.9777727127075195 | Batch Time=6.640625 +GPU:0 | Epoch: 21 | loss=5.6436381340026855 | Batch Time=2.34375 +GPU:0 | Epoch: 21 | loss=5.312105655670166 | Batch Time=1.953125 +GPU:0 | Epoch: 21 | loss=4.447490692138672 | Batch Time=20.703125 +GPU:0 | Epoch: 21 | loss=5.450658321380615 | Batch Time=4.296875 +GPU:0 | Epoch: 21 | loss=4.398509502410889 | Batch Time=17.578125 +GPU:0 | Epoch: 21 | loss=4.599035263061523 | Batch Time=10.15625 +GPU:0 | Epoch: 21 | loss=4.9021382331848145 | Batch Time=11.71875 +GPU:0 | Epoch: 21 | loss=4.436906814575195 | Batch Time=10.9375 +GPU:0 | Epoch: 21 | loss=3.55283784866333 | Batch Time=24.609375 +GPU:0 | Epoch: 21 | loss=3.8511040210723877 | Batch Time=18.359375 +GPU:0 | Epoch: 21 | loss=3.376436710357666 | Batch Time=27.734375 +Model top1 Accuracy: 16.504 +Acc after rounding: 16.504 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 21 | loss=4.534111976623535 | Batch Time=12.109375 +GPU:0 | Epoch: 21 | loss=4.536261081695557 | Batch Time=15.625 +GPU:0 | Epoch: 21 | loss=4.456872463226318 | Batch Time=18.75 +GPU:0 | Epoch: 21 | loss=4.6105241775512695 | Batch Time=10.15625 +Model top1 Accuracy: 14.36 +Validation Acc after rounding: 14.36 +AFTER VAL LOOP: GPU: 0 | Epoch 21 | Memory Usage: svmem(total=257568083968, available=54832304128, percent=78.7, used=194822205440, free=5664604160, active=215705616384, inactive=30924922880, buffers=437936128, cached=56643338240, shared=5662257152, slab=2707718144) +Rounding model with scheme: naive +Model avg sparsity: 38.26055194119684 +GPU:0 | Epoch: 21 | Acc=16.504 | Epoch Time=19.697561248143515 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 21 | Memory Usage: svmem(total=257568083968, available=54832209920, percent=78.7, used=194822377472, free=5663064064, active=215705743360, inactive=30926147584, buffers=437952512, cached=56644689920, shared=5662175232, slab=2707709952) +STARTING TRAINING: Epoch 22 | Memory Usage: svmem(total=257568083968, available=54832209920, percent=78.7, used=194822377472, free=5663064064, active=215705743360, inactive=30926147584, buffers=437952512, cached=56644689920, shared=5662175232, slab=2707709952) +BEFORE TRAIN LOOP: Epoch 22 | Memory Usage: svmem(total=257568083968, available=54832209920, percent=78.7, used=194822377472, free=5663064064, active=215705743360, inactive=30926147584, buffers=437952512, cached=56644689920, shared=5662175232, slab=2707709952) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=54832209920, percent=78.7, used=194822377472, free=5663064064, active=215705743360, inactive=30926147584, buffers=437952512, cached=56644689920, shared=5662175232, slab=2707709952) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:3 | Epoch: 22 | loss=9.977370262145996 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=9.841527938842773 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=9.987419128417969 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=9.8976411819458 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=10.015830039978027 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.903190612792969 | Batch Time=20.3125 +GPU:3 | Epoch: 22 | loss=9.997961044311523 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=10.109495162963867 | Batch Time=13.671875 +GPU:3 | Epoch: 22 | loss=10.099294662475586 | Batch Time=12.5 +GPU:3 | Epoch: 22 | loss=9.7845458984375 | Batch Time=17.578125 +GPU:3 | Epoch: 22 | loss=9.96327018737793 | Batch Time=12.890625 +GPU:3 | Epoch: 22 | loss=9.966262817382812 | Batch Time=17.96875 +GPU:3 | Epoch: 22 | loss=10.007884979248047 | Batch Time=14.453125 +GPU:3 | Epoch: 22 | loss=9.943231582641602 | Batch Time=16.015625 +GPU:3 | Epoch: 22 | loss=9.915189743041992 | Batch Time=16.015625 +GPU:3 | Epoch: 22 | loss=9.952616691589355 | Batch Time=15.234375 +GPU:3 | Epoch: 22 | loss=10.004693984985352 | Batch Time=16.40625 +GPU:3 | Epoch: 22 | loss=9.916054725646973 | Batch Time=19.921875 +GPU:3 | Epoch: 22 | loss=9.960662841796875 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=10.36915111541748 | Batch Time=11.328125 +GPU:3 | Epoch: 22 | loss=9.83500862121582 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=9.80424976348877 | Batch Time=21.09375 +GPU:3 | Epoch: 22 | loss=10.006911277770996 | Batch Time=18.75 +GPU:3 | Epoch: 22 | loss=10.139293670654297 | Batch Time=13.671875 +GPU:3 | Epoch: 22 | loss=9.904376983642578 | Batch Time=22.265625 +GPU:3 | Epoch: 22 | loss=10.031275749206543 | Batch Time=15.234375 +GPU:3 | Epoch: 22 | loss=9.946778297424316 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=9.992510795593262 | Batch Time=15.234375 +GPU:3 | Epoch: 22 | loss=9.969944953918457 | Batch Time=19.140625 +GPU:3 | Epoch: 22 | loss=9.994426727294922 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=9.936910629272461 | Batch Time=16.40625 +GPU:3 | Epoch: 22 | loss=10.106112480163574 | Batch Time=11.71875 +GPU:3 | Epoch: 22 | loss=10.107400894165039 | Batch Time=16.015625 +GPU:3 | Epoch: 22 | loss=9.966304779052734 | Batch Time=19.140625 +GPU:3 | Epoch: 22 | loss=9.925682067871094 | Batch Time=14.453125 +GPU:3 | Epoch: 22 | loss=9.900121688842773 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.996817588806152 | Batch Time=15.234375 +GPU:3 | Epoch: 22 | loss=10.04623031616211 | Batch Time=13.28125 +GPU:3 | Epoch: 22 | loss=10.170343399047852 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=9.834901809692383 | Batch Time=14.84375 +GPU:3 | Epoch: 22 | loss=10.003646850585938 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=9.808561325073242 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=9.895818710327148 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=9.790936470031738 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=10.093856811523438 | Batch Time=11.71875 +GPU:3 | Epoch: 22 | loss=9.809328079223633 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=10.154094696044922 | Batch Time=14.453125 +GPU:3 | Epoch: 22 | loss=10.26395034790039 | Batch Time=12.109375 +GPU:3 | Epoch: 22 | loss=9.814480781555176 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=9.899303436279297 | Batch Time=19.140625 +GPU:3 | Epoch: 22 | loss=9.966973304748535 | Batch Time=15.234375 +GPU:3 | Epoch: 22 | loss=9.927501678466797 | Batch Time=11.71875 +GPU:3 | Epoch: 22 | loss=10.091276168823242 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.808361053466797 | Batch Time=17.96875 +GPU:3 | Epoch: 22 | loss=10.193248748779297 | Batch Time=12.890625 +GPU:3 | Epoch: 22 | loss=9.959753036499023 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=9.915678024291992 | Batch Time=21.09375 +GPU:3 | Epoch: 22 | loss=10.076353073120117 | Batch Time=16.40625 +GPU:3 | Epoch: 22 | loss=9.906146049499512 | Batch Time=19.140625 +GPU:3 | Epoch: 22 | loss=9.866446495056152 | Batch Time=17.96875 +GPU:3 | Epoch: 22 | loss=9.90453052520752 | Batch Time=18.75 +GPU:3 | Epoch: 22 | loss=10.023139953613281 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=9.829940795898438 | Batch Time=18.75 +GPU:3 | Epoch: 22 | loss=9.814218521118164 | Batch Time=17.578125 +GPU:3 | Epoch: 22 | loss=10.12210750579834 | Batch Time=14.453125 +GPU:3 | Epoch: 22 | loss=10.02603530883789 | Batch Time=17.96875 +GPU:3 | Epoch: 22 | loss=10.021286964416504 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=9.82076644897461 | Batch Time=19.140625 +GPU:3 | Epoch: 22 | loss=10.062554359436035 | Batch Time=15.234375 +GPU:3 | Epoch: 22 | loss=9.929712295532227 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=9.874432563781738 | Batch Time=22.265625 +GPU:3 | Epoch: 22 | loss=9.998491287231445 | Batch Time=14.84375 +GPU:3 | Epoch: 22 | loss=9.922542572021484 | Batch Time=13.671875 +GPU:3 | Epoch: 22 | loss=9.901638984680176 | Batch Time=16.015625 +GPU:3 | Epoch: 22 | loss=10.147882461547852 | Batch Time=14.0625 +GPU:3 | Epoch: 22 | loss=10.055790901184082 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=10.055644035339355 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.992327690124512 | Batch Time=14.84375 +GPU:3 | Epoch: 22 | loss=9.950658798217773 | Batch Time=20.3125 +GPU:3 | Epoch: 22 | loss=9.981939315795898 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=10.091444969177246 | Batch Time=17.578125 +GPU:3 | Epoch: 22 | loss=9.903175354003906 | Batch Time=17.578125 +GPU:3 | Epoch: 22 | loss=9.984968185424805 | Batch Time=12.890625 +GPU:3 | Epoch: 22 | loss=9.997457504272461 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.805546760559082 | Batch Time=21.484375 +GPU:3 | Epoch: 22 | loss=9.899531364440918 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.649887084960938 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=9.838724136352539 | Batch Time=20.703125 +GPU:3 | Epoch: 22 | loss=9.90414810180664 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=9.755277633666992 | Batch Time=14.0625 +GPU:3 | Epoch: 22 | loss=9.751060485839844 | Batch Time=21.09375 +GPU:3 | Epoch: 22 | loss=9.759271621704102 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.939493179321289 | Batch Time=15.234375 +GPU:3 | Epoch: 22 | loss=10.153112411499023 | Batch Time=12.890625 +GPU:3 | Epoch: 22 | loss=9.855051040649414 | Batch Time=20.3125 +GPU:3 | Epoch: 22 | loss=9.672255516052246 | Batch Time=18.75 +GPU:3 | Epoch: 22 | loss=10.031715393066406 | Batch Time=14.453125 +GPU:3 | Epoch: 22 | loss=9.674789428710938 | Batch Time=19.53125 +GPU:3 | Epoch: 22 | loss=9.964256286621094 | Batch Time=16.015625 +GPU:3 | Epoch: 22 | loss=9.83289909362793 | Batch Time=19.53125 +GPU:3 | Epoch: 22 | loss=9.990732192993164 | Batch Time=18.75 +GPU:3 | Epoch: 22 | loss=9.865236282348633 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=10.011346817016602 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=9.989267349243164 | Batch Time=17.578125 +GPU:3 | Epoch: 22 | loss=10.027851104736328 | Batch Time=17.1875 +GPU:3 | Epoch: 22 | loss=10.006694793701172 | Batch Time=17.578125 +GPU:3 | Epoch: 22 | loss=10.14084243774414 | Batch Time=12.890625 +GPU:3 | Epoch: 22 | loss=9.904935836791992 | Batch Time=18.75 +GPU:3 | Epoch: 22 | loss=9.815971374511719 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.554421424865723 | Batch Time=19.53125 +GPU:3 | Epoch: 22 | loss=9.796501159667969 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=9.719990730285645 | Batch Time=19.53125 +GPU:3 | Epoch: 22 | loss=10.067667961120605 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.926042556762695 | Batch Time=15.625 +GPU:3 | Epoch: 22 | loss=9.86306095123291 | Batch Time=19.53125 +GPU:3 | Epoch: 22 | loss=10.018404960632324 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=10.003583908081055 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=9.996408462524414 | Batch Time=14.84375 +GPU:3 | Epoch: 22 | loss=9.815590858459473 | Batch Time=19.53125 +GPU:3 | Epoch: 22 | loss=9.717829704284668 | Batch Time=18.359375 +GPU:3 | Epoch: 22 | loss=9.877250671386719 | Batch Time=17.96875 +GPU:3 | Epoch: 22 | loss=9.666979789733887 | Batch Time=16.796875 +GPU:3 | Epoch: 22 | loss=9.78799057006836 | Batch Time=16.015625 +GPU:3 | Epoch: 22 | loss=10.021939277648926 | Batch Time=19.140625 +GPU:3 | Epoch: 22 | loss=10.02063274383545 | Batch Time=14.84375 +GPU:0 | Epoch: 22 | loss=10.00859546661377 | Batch Time=18.359375 +GPU:0 | Epoch: 22 | loss=9.998859405517578 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=9.966411590576172 | Batch Time=12.890625 +GPU:0 | Epoch: 22 | loss=10.15764331817627 | Batch Time=14.0625 +GPU:0 | Epoch: 22 | loss=9.96278190612793 | Batch Time=16.796875 +GPU:0 | Epoch: 22 | loss=9.660504341125488 | Batch Time=23.046875 +GPU:0 | Epoch: 22 | loss=10.12736701965332 | Batch Time=14.84375 +GPU:0 | Epoch: 22 | loss=10.27618408203125 | Batch Time=12.890625 +GPU:0 | Epoch: 22 | loss=10.023962020874023 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=10.119548797607422 | Batch Time=14.0625 +GPU:0 | Epoch: 22 | loss=9.993295669555664 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=9.982841491699219 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=9.931931495666504 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=9.749834060668945 | Batch Time=18.359375 +GPU:0 | Epoch: 22 | loss=10.278017044067383 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=10.179132461547852 | Batch Time=13.671875 +GPU:0 | Epoch: 22 | loss=9.990409851074219 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.9987211227417 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=9.87452507019043 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.992696762084961 | Batch Time=12.109375 +GPU:0 | Epoch: 22 | loss=9.716768264770508 | Batch Time=18.75 +GPU:0 | Epoch: 22 | loss=10.023187637329102 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=9.932334899902344 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=9.998075485229492 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=9.927312850952148 | Batch Time=19.53125 +GPU:0 | Epoch: 22 | loss=10.070173263549805 | Batch Time=12.5 +GPU:0 | Epoch: 22 | loss=9.827795028686523 | Batch Time=17.578125 +GPU:0 | Epoch: 22 | loss=9.987631797790527 | Batch Time=14.84375 +GPU:0 | Epoch: 22 | loss=10.018336296081543 | Batch Time=14.84375 +GPU:0 | Epoch: 22 | loss=10.01264762878418 | Batch Time=19.140625 +GPU:0 | Epoch: 22 | loss=9.970020294189453 | Batch Time=18.359375 +GPU:0 | Epoch: 22 | loss=10.032941818237305 | Batch Time=12.5 +GPU:0 | Epoch: 22 | loss=10.310272216796875 | Batch Time=11.328125 +GPU:0 | Epoch: 22 | loss=10.053411483764648 | Batch Time=12.5 +GPU:0 | Epoch: 22 | loss=9.834572792053223 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=10.183481216430664 | Batch Time=14.453125 +GPU:0 | Epoch: 22 | loss=9.862667083740234 | Batch Time=16.796875 +GPU:0 | Epoch: 22 | loss=10.11697769165039 | Batch Time=18.75 +GPU:0 | Epoch: 22 | loss=10.02194595336914 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=10.009296417236328 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=10.002412796020508 | Batch Time=19.53125 +GPU:0 | Epoch: 22 | loss=10.082664489746094 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=10.156323432922363 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=10.071878433227539 | Batch Time=12.109375 +GPU:0 | Epoch: 22 | loss=9.76791763305664 | Batch Time=21.875 +GPU:0 | Epoch: 22 | loss=10.113149642944336 | Batch Time=13.28125 +GPU:0 | Epoch: 22 | loss=9.972030639648438 | Batch Time=14.453125 +GPU:0 | Epoch: 22 | loss=10.039875030517578 | Batch Time=19.140625 +GPU:0 | Epoch: 22 | loss=9.953006744384766 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=10.049102783203125 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.903606414794922 | Batch Time=18.359375 +GPU:0 | Epoch: 22 | loss=9.843252182006836 | Batch Time=19.53125 +GPU:0 | Epoch: 22 | loss=9.957386016845703 | Batch Time=13.28125 +GPU:0 | Epoch: 22 | loss=9.973365783691406 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=10.03821849822998 | Batch Time=14.84375 +GPU:0 | Epoch: 22 | loss=9.873522758483887 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=9.850707054138184 | Batch Time=20.3125 +GPU:0 | Epoch: 22 | loss=9.93287467956543 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=9.784527778625488 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=9.900176048278809 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=9.800317764282227 | Batch Time=18.75 +GPU:0 | Epoch: 22 | loss=9.892175674438477 | Batch Time=20.3125 +GPU:0 | Epoch: 22 | loss=9.942119598388672 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=9.789085388183594 | Batch Time=18.75 +GPU:0 | Epoch: 22 | loss=9.804553985595703 | Batch Time=17.578125 +GPU:0 | Epoch: 22 | loss=9.85214614868164 | Batch Time=18.359375 +GPU:0 | Epoch: 22 | loss=9.9383544921875 | Batch Time=16.796875 +GPU:0 | Epoch: 22 | loss=9.994285583496094 | Batch Time=20.3125 +GPU:0 | Epoch: 22 | loss=10.149209976196289 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=9.923508644104004 | Batch Time=19.140625 +GPU:0 | Epoch: 22 | loss=9.928421020507812 | Batch Time=21.484375 +GPU:0 | Epoch: 22 | loss=10.13976001739502 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=9.954119682312012 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.832988739013672 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=10.19694709777832 | Batch Time=14.0625 +GPU:0 | Epoch: 22 | loss=10.023284912109375 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=9.989377975463867 | Batch Time=17.578125 +GPU:0 | Epoch: 22 | loss=10.08873176574707 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=10.099283218383789 | Batch Time=14.0625 +GPU:0 | Epoch: 22 | loss=9.678499221801758 | Batch Time=21.09375 +GPU:0 | Epoch: 22 | loss=10.028301239013672 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=9.795320510864258 | Batch Time=21.484375 +GPU:0 | Epoch: 22 | loss=9.91048812866211 | Batch Time=17.578125 +GPU:0 | Epoch: 22 | loss=9.997081756591797 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=9.74893569946289 | Batch Time=18.359375 +GPU:0 | Epoch: 22 | loss=9.896339416503906 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=9.728179931640625 | Batch Time=21.09375 +GPU:0 | Epoch: 22 | loss=10.035697937011719 | Batch Time=10.15625 +GPU:0 | Epoch: 22 | loss=9.985567092895508 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.86570930480957 | Batch Time=18.359375 +GPU:0 | Epoch: 22 | loss=9.951262474060059 | Batch Time=14.453125 +GPU:0 | Epoch: 22 | loss=9.706594467163086 | Batch Time=21.09375 +GPU:0 | Epoch: 22 | loss=9.76075553894043 | Batch Time=20.3125 +GPU:0 | Epoch: 22 | loss=9.973579406738281 | Batch Time=16.796875 +GPU:0 | Epoch: 22 | loss=9.967456817626953 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=9.776529312133789 | Batch Time=14.0625 +GPU:0 | Epoch: 22 | loss=9.847709655761719 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=9.901309967041016 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=9.91007137298584 | Batch Time=16.40625 +GPU:0 | Epoch: 22 | loss=9.881444931030273 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.628427505493164 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=9.964594841003418 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.978921890258789 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=9.909807205200195 | Batch Time=14.453125 +GPU:0 | Epoch: 22 | loss=9.706291198730469 | Batch Time=19.53125 +GPU:0 | Epoch: 22 | loss=9.840229034423828 | Batch Time=18.75 +GPU:0 | Epoch: 22 | loss=9.805566787719727 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=9.883974075317383 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=9.712718963623047 | Batch Time=20.703125 +GPU:0 | Epoch: 22 | loss=9.865220069885254 | Batch Time=17.1875 +GPU:0 | Epoch: 22 | loss=10.040441513061523 | Batch Time=14.84375 +GPU:0 | Epoch: 22 | loss=9.89564037322998 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.69835090637207 | Batch Time=19.53125 +GPU:0 | Epoch: 22 | loss=9.917539596557617 | Batch Time=14.84375 +GPU:0 | Epoch: 22 | loss=9.86201286315918 | Batch Time=18.75 +GPU:0 | Epoch: 22 | loss=9.906861305236816 | Batch Time=14.453125 +GPU:0 | Epoch: 22 | loss=9.938068389892578 | Batch Time=16.796875 +GPU:0 | Epoch: 22 | loss=9.919729232788086 | Batch Time=18.75 +GPU:0 | Epoch: 22 | loss=9.924246788024902 | Batch Time=13.671875 +GPU:0 | Epoch: 22 | loss=9.923609733581543 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=9.975303649902344 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=9.893217086791992 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.967538833618164 | Batch Time=14.453125 +GPU:0 | Epoch: 22 | loss=10.04135513305664 | Batch Time=15.625 +GPU:0 | Epoch: 22 | loss=9.7662935256958 | Batch Time=18.75 +GPU:2 | Epoch: 22 | loss=9.981135368347168 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=10.037351608276367 | Batch Time=13.671875 +GPU:2 | Epoch: 22 | loss=9.898984909057617 | Batch Time=21.09375 +GPU:2 | Epoch: 22 | loss=10.022355079650879 | Batch Time=19.53125 +GPU:2 | Epoch: 22 | loss=10.027213096618652 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=10.056137084960938 | Batch Time=19.140625 +GPU:2 | Epoch: 22 | loss=9.900646209716797 | Batch Time=19.921875 +GPU:2 | Epoch: 22 | loss=10.046936988830566 | Batch Time=19.53125 +GPU:2 | Epoch: 22 | loss=10.3541259765625 | Batch Time=13.28125 +GPU:2 | Epoch: 22 | loss=9.760505676269531 | Batch Time=20.3125 +GPU:2 | Epoch: 22 | loss=9.958137512207031 | Batch Time=17.96875 +GPU:2 | Epoch: 22 | loss=10.08586311340332 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=10.054058074951172 | Batch Time=13.28125 +GPU:2 | Epoch: 22 | loss=9.937685012817383 | Batch Time=19.921875 +GPU:2 | Epoch: 22 | loss=10.168296813964844 | Batch Time=15.625 +GPU:2 | Epoch: 22 | loss=9.940705299377441 | Batch Time=15.625 +GPU:2 | Epoch: 22 | loss=9.95114517211914 | Batch Time=19.53125 +GPU:2 | Epoch: 22 | loss=9.921051025390625 | Batch Time=17.96875 +GPU:2 | Epoch: 22 | loss=10.06146240234375 | Batch Time=13.28125 +GPU:2 | Epoch: 22 | loss=9.941122055053711 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=10.333086013793945 | Batch Time=12.890625 +GPU:2 | Epoch: 22 | loss=9.902299880981445 | Batch Time=21.09375 +GPU:2 | Epoch: 22 | loss=9.994976043701172 | Batch Time=17.96875 +GPU:2 | Epoch: 22 | loss=9.975150108337402 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=9.809228897094727 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=9.859658241271973 | Batch Time=19.140625 +GPU:2 | Epoch: 22 | loss=9.947671890258789 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=9.890995025634766 | Batch Time=19.140625 +GPU:2 | Epoch: 22 | loss=9.994364738464355 | Batch Time=14.453125 +GPU:2 | Epoch: 22 | loss=9.798168182373047 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=10.106212615966797 | Batch Time=12.890625 +GPU:2 | Epoch: 22 | loss=10.115466117858887 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=9.983257293701172 | Batch Time=19.140625 +GPU:2 | Epoch: 22 | loss=9.882125854492188 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=10.139842987060547 | Batch Time=15.625 +GPU:2 | Epoch: 22 | loss=10.044473648071289 | Batch Time=15.625 +GPU:2 | Epoch: 22 | loss=9.846979141235352 | Batch Time=21.09375 +GPU:2 | Epoch: 22 | loss=10.195903778076172 | Batch Time=12.890625 +GPU:2 | Epoch: 22 | loss=10.064126968383789 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=10.146621704101562 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=9.939226150512695 | Batch Time=17.578125 +GPU:2 | Epoch: 22 | loss=9.954845428466797 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=10.026044845581055 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=9.97854232788086 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=10.041854858398438 | Batch Time=17.96875 +GPU:2 | Epoch: 22 | loss=10.024486541748047 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=10.144909858703613 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=10.09100341796875 | Batch Time=13.671875 +GPU:2 | Epoch: 22 | loss=10.073013305664062 | Batch Time=14.84375 +GPU:2 | Epoch: 22 | loss=9.931453704833984 | Batch Time=15.625 +GPU:2 | Epoch: 22 | loss=9.7232027053833 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=10.111953735351562 | Batch Time=14.0625 +GPU:2 | Epoch: 22 | loss=9.855937957763672 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=10.007242202758789 | Batch Time=14.453125 +GPU:2 | Epoch: 22 | loss=9.824865341186523 | Batch Time=16.796875 +GPU:2 | Epoch: 22 | loss=10.005027770996094 | Batch Time=14.453125 +GPU:2 | Epoch: 22 | loss=9.979826927185059 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=9.968505859375 | Batch Time=16.796875 +GPU:2 | Epoch: 22 | loss=10.016525268554688 | Batch Time=14.84375 +GPU:2 | Epoch: 22 | loss=9.805461883544922 | Batch Time=19.53125 +GPU:2 | Epoch: 22 | loss=9.78941822052002 | Batch Time=19.140625 +GPU:2 | Epoch: 22 | loss=10.027040481567383 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=10.17822551727295 | Batch Time=14.453125 +GPU:2 | Epoch: 22 | loss=9.893171310424805 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=9.898953437805176 | Batch Time=14.84375 +GPU:2 | Epoch: 22 | loss=9.836711883544922 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=10.186893463134766 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=9.769062042236328 | Batch Time=21.09375 +GPU:2 | Epoch: 22 | loss=9.955988883972168 | Batch Time=19.53125 +GPU:2 | Epoch: 22 | loss=9.980913162231445 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=10.03702163696289 | Batch Time=19.921875 +GPU:2 | Epoch: 22 | loss=10.005094528198242 | Batch Time=14.84375 +GPU:2 | Epoch: 22 | loss=9.688698768615723 | Batch Time=21.09375 +GPU:2 | Epoch: 22 | loss=10.08390998840332 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=10.020931243896484 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=9.820511817932129 | Batch Time=16.796875 +GPU:2 | Epoch: 22 | loss=9.949445724487305 | Batch Time=14.84375 +GPU:2 | Epoch: 22 | loss=9.828893661499023 | Batch Time=19.140625 +GPU:2 | Epoch: 22 | loss=9.781065940856934 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=9.81950569152832 | Batch Time=21.09375 +GPU:2 | Epoch: 22 | loss=9.995504379272461 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=9.906071662902832 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=10.015920639038086 | Batch Time=14.453125 +GPU:2 | Epoch: 22 | loss=10.009521484375 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=9.729387283325195 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=9.78288459777832 | Batch Time=13.28125 +GPU:2 | Epoch: 22 | loss=10.156110763549805 | Batch Time=14.453125 +GPU:2 | Epoch: 22 | loss=9.821475982666016 | Batch Time=22.265625 +GPU:2 | Epoch: 22 | loss=9.683822631835938 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=9.900914192199707 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=9.974178314208984 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=9.817309379577637 | Batch Time=21.09375 +GPU:2 | Epoch: 22 | loss=10.056266784667969 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=9.859959602355957 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=9.862751007080078 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=9.810569763183594 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=9.85938835144043 | Batch Time=16.796875 +GPU:2 | Epoch: 22 | loss=10.029394149780273 | Batch Time=18.75 +GPU:2 | Epoch: 22 | loss=9.890344619750977 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=9.851680755615234 | Batch Time=14.0625 +GPU:2 | Epoch: 22 | loss=9.962728500366211 | Batch Time=19.140625 +GPU:2 | Epoch: 22 | loss=9.996635437011719 | Batch Time=15.234375 +GPU:2 | Epoch: 22 | loss=10.146045684814453 | Batch Time=16.796875 +GPU:2 | Epoch: 22 | loss=10.02074146270752 | Batch Time=13.28125 +GPU:2 | Epoch: 22 | loss=9.945663452148438 | Batch Time=15.625 +GPU:2 | Epoch: 22 | loss=9.815245628356934 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=9.931995391845703 | Batch Time=19.921875 +GPU:2 | Epoch: 22 | loss=9.770730972290039 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=9.963743209838867 | Batch Time=17.96875 +GPU:2 | Epoch: 22 | loss=9.843117713928223 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=9.861248970031738 | Batch Time=14.84375 +GPU:2 | Epoch: 22 | loss=9.968550682067871 | Batch Time=16.796875 +GPU:2 | Epoch: 22 | loss=10.047651290893555 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=10.016963958740234 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=9.953184127807617 | Batch Time=18.75 +GPU:2 | Epoch: 22 | loss=9.909883499145508 | Batch Time=17.1875 +GPU:2 | Epoch: 22 | loss=10.24001407623291 | Batch Time=11.71875 +GPU:2 | Epoch: 22 | loss=9.822606086730957 | Batch Time=19.140625 +GPU:2 | Epoch: 22 | loss=10.175186157226562 | Batch Time=16.40625 +GPU:2 | Epoch: 22 | loss=9.928016662597656 | Batch Time=18.359375 +GPU:2 | Epoch: 22 | loss=9.922429084777832 | Batch Time=17.578125 +GPU:2 | Epoch: 22 | loss=9.691076278686523 | Batch Time=22.265625 +GPU:2 | Epoch: 22 | loss=10.099262237548828 | Batch Time=16.015625 +GPU:2 | Epoch: 22 | loss=10.174283981323242 | Batch Time=13.28125 +GPU:2 | Epoch: 22 | loss=9.702130317687988 | Batch Time=22.65625 +GPU:1 | Epoch: 22 | loss=9.921855926513672 | Batch Time=17.578125 +GPU:1 | Epoch: 22 | loss=10.06987190246582 | Batch Time=13.28125 +GPU:1 | Epoch: 22 | loss=10.283767700195312 | Batch Time=13.28125 +GPU:1 | Epoch: 22 | loss=9.818613052368164 | Batch Time=18.359375 +GPU:1 | Epoch: 22 | loss=9.957955360412598 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=10.28110408782959 | Batch Time=13.671875 +GPU:1 | Epoch: 22 | loss=9.98250961303711 | Batch Time=19.921875 +GPU:1 | Epoch: 22 | loss=10.066810607910156 | Batch Time=16.40625 +GPU:1 | Epoch: 22 | loss=10.029769897460938 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.920249938964844 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.816886901855469 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=10.009419441223145 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=10.218172073364258 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=10.105224609375 | Batch Time=14.0625 +GPU:1 | Epoch: 22 | loss=10.089423179626465 | Batch Time=14.453125 +GPU:1 | Epoch: 22 | loss=10.059782981872559 | Batch Time=14.84375 +GPU:1 | Epoch: 22 | loss=9.967631340026855 | Batch Time=15.625 +GPU:1 | Epoch: 22 | loss=10.17248821258545 | Batch Time=10.546875 +GPU:1 | Epoch: 22 | loss=9.899640083312988 | Batch Time=19.140625 +GPU:1 | Epoch: 22 | loss=10.001815795898438 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.98790168762207 | Batch Time=17.578125 +GPU:1 | Epoch: 22 | loss=10.004863739013672 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=10.01736068725586 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=10.041250228881836 | Batch Time=19.140625 +GPU:1 | Epoch: 22 | loss=10.169466018676758 | Batch Time=14.0625 +GPU:1 | Epoch: 22 | loss=10.07346248626709 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=10.206937789916992 | Batch Time=11.328125 +GPU:1 | Epoch: 22 | loss=9.977084159851074 | Batch Time=10.9375 +GPU:1 | Epoch: 22 | loss=10.293258666992188 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=10.216686248779297 | Batch Time=14.453125 +GPU:1 | Epoch: 22 | loss=10.152239799499512 | Batch Time=14.84375 +GPU:1 | Epoch: 22 | loss=9.944225311279297 | Batch Time=19.921875 +GPU:1 | Epoch: 22 | loss=9.840309143066406 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=10.019437789916992 | Batch Time=18.75 +GPU:1 | Epoch: 22 | loss=10.024497032165527 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=9.967241287231445 | Batch Time=15.625 +GPU:1 | Epoch: 22 | loss=9.910917282104492 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=10.051656723022461 | Batch Time=14.84375 +GPU:1 | Epoch: 22 | loss=10.035379409790039 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=10.080324172973633 | Batch Time=14.453125 +GPU:1 | Epoch: 22 | loss=9.965362548828125 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=9.704021453857422 | Batch Time=19.140625 +GPU:1 | Epoch: 22 | loss=9.890442848205566 | Batch Time=17.578125 +GPU:1 | Epoch: 22 | loss=9.964130401611328 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=10.21055793762207 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=10.119726181030273 | Batch Time=12.890625 +GPU:1 | Epoch: 22 | loss=10.090749740600586 | Batch Time=16.40625 +GPU:1 | Epoch: 22 | loss=9.916833877563477 | Batch Time=17.578125 +GPU:1 | Epoch: 22 | loss=9.726242065429688 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=10.0348482131958 | Batch Time=18.75 +GPU:1 | Epoch: 22 | loss=10.125057220458984 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=10.056024551391602 | Batch Time=18.75 +GPU:1 | Epoch: 22 | loss=9.989349365234375 | Batch Time=18.359375 +GPU:1 | Epoch: 22 | loss=10.097173690795898 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.674514770507812 | Batch Time=21.09375 +GPU:1 | Epoch: 22 | loss=10.002521514892578 | Batch Time=18.359375 +GPU:1 | Epoch: 22 | loss=10.119953155517578 | Batch Time=14.453125 +GPU:1 | Epoch: 22 | loss=9.997790336608887 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=9.915505409240723 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=10.208417892456055 | Batch Time=14.84375 +GPU:1 | Epoch: 22 | loss=9.88573932647705 | Batch Time=18.75 +GPU:1 | Epoch: 22 | loss=9.935901641845703 | Batch Time=14.0625 +GPU:1 | Epoch: 22 | loss=9.772941589355469 | Batch Time=21.484375 +GPU:1 | Epoch: 22 | loss=10.034278869628906 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=9.820703506469727 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=9.761180877685547 | Batch Time=22.265625 +GPU:1 | Epoch: 22 | loss=10.008134841918945 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.871801376342773 | Batch Time=17.96875 +GPU:1 | Epoch: 22 | loss=9.863847732543945 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=10.020544052124023 | Batch Time=16.40625 +GPU:1 | Epoch: 22 | loss=9.88587760925293 | Batch Time=17.578125 +GPU:1 | Epoch: 22 | loss=9.966551780700684 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=9.996603012084961 | Batch Time=16.40625 +GPU:1 | Epoch: 22 | loss=10.046181678771973 | Batch Time=17.578125 +GPU:1 | Epoch: 22 | loss=9.651308059692383 | Batch Time=20.3125 +GPU:1 | Epoch: 22 | loss=9.891182899475098 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.833014488220215 | Batch Time=18.359375 +GPU:1 | Epoch: 22 | loss=10.279069900512695 | Batch Time=12.890625 +GPU:1 | Epoch: 22 | loss=9.938614845275879 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=9.810426712036133 | Batch Time=19.140625 +GPU:1 | Epoch: 22 | loss=10.015251159667969 | Batch Time=15.625 +GPU:1 | Epoch: 22 | loss=9.975443840026855 | Batch Time=18.75 +GPU:1 | Epoch: 22 | loss=10.076547622680664 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=10.046836853027344 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=9.867961883544922 | Batch Time=14.453125 +GPU:1 | Epoch: 22 | loss=9.93553352355957 | Batch Time=18.359375 +GPU:1 | Epoch: 22 | loss=10.059861183166504 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=9.863273620605469 | Batch Time=16.015625 +GPU:1 | Epoch: 22 | loss=9.777961730957031 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=9.792617797851562 | Batch Time=20.703125 +GPU:1 | Epoch: 22 | loss=10.121978759765625 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.877900123596191 | Batch Time=17.96875 +GPU:1 | Epoch: 22 | loss=9.874040603637695 | Batch Time=19.53125 +GPU:1 | Epoch: 22 | loss=9.844756126403809 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.843828201293945 | Batch Time=16.40625 +GPU:1 | Epoch: 22 | loss=9.705950736999512 | Batch Time=19.53125 +GPU:1 | Epoch: 22 | loss=9.692858695983887 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=9.743501663208008 | Batch Time=16.40625 +GPU:1 | Epoch: 22 | loss=9.833686828613281 | Batch Time=20.703125 +GPU:1 | Epoch: 22 | loss=9.71981143951416 | Batch Time=20.703125 +GPU:1 | Epoch: 22 | loss=9.795686721801758 | Batch Time=18.359375 +GPU:1 | Epoch: 22 | loss=9.772144317626953 | Batch Time=17.96875 +GPU:1 | Epoch: 22 | loss=9.98425006866455 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=9.95429801940918 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=9.731189727783203 | Batch Time=19.53125 +GPU:1 | Epoch: 22 | loss=9.880990982055664 | Batch Time=19.53125 +GPU:1 | Epoch: 22 | loss=9.77348804473877 | Batch Time=15.625 +GPU:1 | Epoch: 22 | loss=9.724710464477539 | Batch Time=21.484375 +GPU:1 | Epoch: 22 | loss=9.719707489013672 | Batch Time=16.40625 +GPU:1 | Epoch: 22 | loss=9.84544563293457 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=9.908465385437012 | Batch Time=20.703125 +GPU:1 | Epoch: 22 | loss=9.697891235351562 | Batch Time=17.96875 +GPU:1 | Epoch: 22 | loss=9.554596900939941 | Batch Time=17.578125 +GPU:1 | Epoch: 22 | loss=9.90659236907959 | Batch Time=16.40625 +GPU:1 | Epoch: 22 | loss=9.831497192382812 | Batch Time=21.484375 +GPU:1 | Epoch: 22 | loss=9.766281127929688 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=9.789815902709961 | Batch Time=19.53125 +GPU:1 | Epoch: 22 | loss=9.80146598815918 | Batch Time=18.359375 +GPU:1 | Epoch: 22 | loss=9.791580200195312 | Batch Time=19.53125 +GPU:1 | Epoch: 22 | loss=10.007094383239746 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=9.853355407714844 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=9.819055557250977 | Batch Time=15.234375 +GPU:1 | Epoch: 22 | loss=9.900853157043457 | Batch Time=16.796875 +GPU:1 | Epoch: 22 | loss=9.869392395019531 | Batch Time=17.1875 +GPU:1 | Epoch: 22 | loss=9.672762870788574 | Batch Time=19.140625 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=65114685440, percent=74.7, used=184539799552, free=16968269824, active=196521181184, inactive=39161958400, buffers=405217280, cached=55654797312, shared=5662224384, slab=2395017216) +AFTER TRAIN LOOP: Epoch 22 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +STARTING TRAINING: Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +BEFORE TRAIN LOOP: Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=65114685440, percent=74.7, used=184539799552, free=16968269824, active=196521181184, inactive=39161958400, buffers=405217280, cached=55654797312, shared=5662224384, slab=2395017216) +AFTER TRAIN LOOP: Epoch 22 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +STARTING TRAINING: Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +BEFORE TRAIN LOOP: Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=65114685440, percent=74.7, used=184539799552, free=16968269824, active=196521181184, inactive=39161958400, buffers=405217280, cached=55654797312, shared=5662224384, slab=2395017216) +AFTER TRAIN LOOP: Epoch 22 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +BEFORE VAL LOOP: GPU: 0 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=65115201536, percent=74.7, used=184539283456, free=16968785920, active=196521181184, inactive=39161958400, buffers=405217280, cached=55654797312, shared=5662224384, slab=2395017216) +AFTER TRAIN LOOP: Epoch 22 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +STARTING TRAINING: Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +BEFORE TRAIN LOOP: Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=65116024832, percent=74.7, used=184538767360, free=16969601024, active=196520153088, inactive=39161819136, buffers=405217280, cached=55654498304, shared=5661896704, slab=2394980352) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 22 | loss=2.8704898357391357 | Batch Time=36.71875 +GPU:0 | Epoch: 22 | loss=4.590761661529541 | Batch Time=14.453125 +GPU:0 | Epoch: 22 | loss=3.7809829711914062 | Batch Time=26.953125 +GPU:0 | Epoch: 22 | loss=3.2683346271514893 | Batch Time=20.703125 +GPU:0 | Epoch: 22 | loss=3.963688611984253 | Batch Time=11.328125 +GPU:0 | Epoch: 22 | loss=3.450650215148926 | Batch Time=13.671875 +GPU:0 | Epoch: 22 | loss=4.737422943115234 | Batch Time=11.71875 +GPU:0 | Epoch: 22 | loss=4.434199810028076 | Batch Time=10.546875 +GPU:0 | Epoch: 22 | loss=4.783041000366211 | Batch Time=9.765625 +GPU:0 | Epoch: 22 | loss=5.504485130310059 | Batch Time=3.515625 +GPU:0 | Epoch: 22 | loss=5.300195693969727 | Batch Time=2.734375 +GPU:0 | Epoch: 22 | loss=4.534019470214844 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=5.21653413772583 | Batch Time=5.078125 +GPU:0 | Epoch: 22 | loss=4.446648597717285 | Batch Time=17.578125 +GPU:0 | Epoch: 22 | loss=4.494438171386719 | Batch Time=12.109375 +GPU:0 | Epoch: 22 | loss=4.823638916015625 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=4.338694095611572 | Batch Time=14.0625 +GPU:0 | Epoch: 22 | loss=3.5301523208618164 | Batch Time=28.125 +GPU:0 | Epoch: 22 | loss=3.998166084289551 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=2.929673194885254 | Batch Time=34.765625 +Model top1 Accuracy: 17.482 +Acc before rounding: 17.482 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 22 | loss=2.8704898357391357 | Batch Time=36.71875 +GPU:0 | Epoch: 22 | loss=4.590761661529541 | Batch Time=14.453125 +GPU:0 | Epoch: 22 | loss=3.7809829711914062 | Batch Time=26.953125 +GPU:0 | Epoch: 22 | loss=3.2683346271514893 | Batch Time=20.703125 +GPU:0 | Epoch: 22 | loss=3.963688611984253 | Batch Time=11.328125 +GPU:0 | Epoch: 22 | loss=3.450650215148926 | Batch Time=13.671875 +GPU:0 | Epoch: 22 | loss=4.737422943115234 | Batch Time=11.71875 +GPU:0 | Epoch: 22 | loss=4.434199810028076 | Batch Time=10.546875 +GPU:0 | Epoch: 22 | loss=4.783041000366211 | Batch Time=9.765625 +GPU:0 | Epoch: 22 | loss=5.504485130310059 | Batch Time=3.515625 +GPU:0 | Epoch: 22 | loss=5.300195693969727 | Batch Time=2.734375 +GPU:0 | Epoch: 22 | loss=4.534019470214844 | Batch Time=17.96875 +GPU:0 | Epoch: 22 | loss=5.21653413772583 | Batch Time=5.078125 +GPU:0 | Epoch: 22 | loss=4.446648597717285 | Batch Time=17.578125 +GPU:0 | Epoch: 22 | loss=4.494438171386719 | Batch Time=12.109375 +GPU:0 | Epoch: 22 | loss=4.823638916015625 | Batch Time=15.234375 +GPU:0 | Epoch: 22 | loss=4.338694095611572 | Batch Time=14.0625 +GPU:0 | Epoch: 22 | loss=3.5301523208618164 | Batch Time=28.125 +GPU:0 | Epoch: 22 | loss=3.998166084289551 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=2.929673194885254 | Batch Time=34.765625 +Model top1 Accuracy: 17.482 +Acc after rounding: 17.482 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 22 | loss=4.407504558563232 | Batch Time=17.578125 +GPU:0 | Epoch: 22 | loss=4.54856538772583 | Batch Time=16.015625 +GPU:0 | Epoch: 22 | loss=4.278237342834473 | Batch Time=20.3125 +GPU:0 | Epoch: 22 | loss=4.61229133605957 | Batch Time=10.15625 +Model top1 Accuracy: 15.67 +Validation Acc after rounding: 15.67 +AFTER VAL LOOP: GPU: 0 | Epoch 22 | Memory Usage: svmem(total=257568083968, available=47318593536, percent=81.6, used=202335662080, free=5614059520, active=218145259520, inactive=28823232512, buffers=443363328, cached=49174999040, shared=5662371840, slab=2361679872) +Rounding model with scheme: naive +Model avg sparsity: 37.72453577933613 +GPU:0 | Epoch: 22 | Acc=17.482 | Epoch Time=20.405269245306652 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 22 | Memory Usage: svmem(total=257568083968, available=47318593536, percent=81.6, used=202335662080, free=5614059520, active=218145243136, inactive=28823232512, buffers=443371520, cached=49174990848, shared=5662371840, slab=2361679872) +STARTING TRAINING: Epoch 23 | Memory Usage: svmem(total=257568083968, available=47318593536, percent=81.6, used=202335662080, free=5614059520, active=218145243136, inactive=28823232512, buffers=443371520, cached=49174990848, shared=5662371840, slab=2361679872) +BEFORE TRAIN LOOP: Epoch 23 | Memory Usage: svmem(total=257568083968, available=47318593536, percent=81.6, used=202335662080, free=5614059520, active=218145243136, inactive=28823232512, buffers=443371520, cached=49174990848, shared=5662371840, slab=2361679872) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=47318593536, percent=81.6, used=202335662080, free=5614059520, active=218145243136, inactive=28823232512, buffers=443371520, cached=49174990848, shared=5662371840, slab=2361679872) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 23 | loss=9.993707656860352 | Batch Time=15.234375 +GPU:1 | Epoch: 23 | loss=9.747102737426758 | Batch Time=20.703125 +GPU:1 | Epoch: 23 | loss=9.921216011047363 | Batch Time=16.40625 +GPU:1 | Epoch: 23 | loss=9.786828994750977 | Batch Time=19.140625 +GPU:1 | Epoch: 23 | loss=9.9622802734375 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.645095825195312 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.72745418548584 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.856435775756836 | Batch Time=19.921875 +GPU:1 | Epoch: 23 | loss=10.039400100708008 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.992070198059082 | Batch Time=18.359375 +GPU:1 | Epoch: 23 | loss=10.11717700958252 | Batch Time=14.453125 +GPU:1 | Epoch: 23 | loss=9.65380859375 | Batch Time=20.703125 +GPU:1 | Epoch: 23 | loss=10.130158424377441 | Batch Time=13.28125 +GPU:1 | Epoch: 23 | loss=9.672548294067383 | Batch Time=22.65625 +GPU:1 | Epoch: 23 | loss=9.693821907043457 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.784040451049805 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.894709587097168 | Batch Time=16.40625 +GPU:1 | Epoch: 23 | loss=9.880989074707031 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.816600799560547 | Batch Time=18.359375 +GPU:1 | Epoch: 23 | loss=9.824867248535156 | Batch Time=18.359375 +GPU:1 | Epoch: 23 | loss=9.790470123291016 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.854867935180664 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.814834594726562 | Batch Time=19.53125 +GPU:1 | Epoch: 23 | loss=9.918291091918945 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=10.007290840148926 | Batch Time=12.890625 +GPU:1 | Epoch: 23 | loss=9.894556045532227 | Batch Time=14.84375 +GPU:1 | Epoch: 23 | loss=9.914735794067383 | Batch Time=16.40625 +GPU:1 | Epoch: 23 | loss=9.803064346313477 | Batch Time=14.0625 +GPU:1 | Epoch: 23 | loss=9.904264450073242 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.905823707580566 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.741168975830078 | Batch Time=21.09375 +GPU:1 | Epoch: 23 | loss=9.597713470458984 | Batch Time=21.09375 +GPU:1 | Epoch: 23 | loss=9.978363037109375 | Batch Time=16.40625 +GPU:1 | Epoch: 23 | loss=9.833728790283203 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.936393737792969 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.738462448120117 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=10.032587051391602 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.866714477539062 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.788156509399414 | Batch Time=19.921875 +GPU:1 | Epoch: 23 | loss=9.685836791992188 | Batch Time=21.875 +GPU:1 | Epoch: 23 | loss=9.841123580932617 | Batch Time=16.40625 +GPU:1 | Epoch: 23 | loss=9.748908996582031 | Batch Time=19.140625 +GPU:1 | Epoch: 23 | loss=9.862325668334961 | Batch Time=21.875 +GPU:1 | Epoch: 23 | loss=10.066194534301758 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=10.008615493774414 | Batch Time=13.28125 +GPU:1 | Epoch: 23 | loss=9.728109359741211 | Batch Time=19.53125 +GPU:1 | Epoch: 23 | loss=9.616692543029785 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.57025146484375 | Batch Time=19.53125 +GPU:1 | Epoch: 23 | loss=10.04166030883789 | Batch Time=14.84375 +GPU:1 | Epoch: 23 | loss=9.694486618041992 | Batch Time=13.671875 +GPU:1 | Epoch: 23 | loss=9.818964004516602 | Batch Time=19.140625 +GPU:1 | Epoch: 23 | loss=9.86237907409668 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.716983795166016 | Batch Time=19.53125 +GPU:1 | Epoch: 23 | loss=9.93802261352539 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.929876327514648 | Batch Time=15.234375 +GPU:1 | Epoch: 23 | loss=9.759950637817383 | Batch Time=19.53125 +GPU:1 | Epoch: 23 | loss=9.63590145111084 | Batch Time=19.140625 +GPU:1 | Epoch: 23 | loss=9.610243797302246 | Batch Time=19.921875 +GPU:1 | Epoch: 23 | loss=9.6353120803833 | Batch Time=22.65625 +GPU:1 | Epoch: 23 | loss=9.739106178283691 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.547521591186523 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.85768985748291 | Batch Time=14.84375 +GPU:1 | Epoch: 23 | loss=10.010615348815918 | Batch Time=14.0625 +GPU:1 | Epoch: 23 | loss=9.891616821289062 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.68508529663086 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.82559585571289 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.640264511108398 | Batch Time=21.09375 +GPU:1 | Epoch: 23 | loss=9.784984588623047 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.704793930053711 | Batch Time=19.140625 +GPU:1 | Epoch: 23 | loss=9.894071578979492 | Batch Time=18.359375 +GPU:1 | Epoch: 23 | loss=9.83840274810791 | Batch Time=17.578125 +GPU:1 | Epoch: 23 | loss=9.677257537841797 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.866436004638672 | Batch Time=21.484375 +GPU:1 | Epoch: 23 | loss=9.921070098876953 | Batch Time=15.234375 +GPU:1 | Epoch: 23 | loss=9.570676803588867 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.719264030456543 | Batch Time=19.921875 +GPU:1 | Epoch: 23 | loss=9.6167573928833 | Batch Time=23.046875 +GPU:1 | Epoch: 23 | loss=9.775907516479492 | Batch Time=17.578125 +GPU:1 | Epoch: 23 | loss=9.94232177734375 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.792562484741211 | Batch Time=16.40625 +GPU:1 | Epoch: 23 | loss=9.887308120727539 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.710247039794922 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.873260498046875 | Batch Time=16.015625 +GPU:1 | Epoch: 23 | loss=9.783519744873047 | Batch Time=20.703125 +GPU:1 | Epoch: 23 | loss=9.770265579223633 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.756427764892578 | Batch Time=20.3125 +GPU:1 | Epoch: 23 | loss=9.779165267944336 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.760603904724121 | Batch Time=19.921875 +GPU:1 | Epoch: 23 | loss=9.790899276733398 | Batch Time=19.140625 +GPU:1 | Epoch: 23 | loss=9.791349411010742 | Batch Time=16.015625 +GPU:1 | Epoch: 23 | loss=9.70193862915039 | Batch Time=18.359375 +GPU:1 | Epoch: 23 | loss=9.779924392700195 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=10.0592041015625 | Batch Time=13.671875 +GPU:1 | Epoch: 23 | loss=9.639715194702148 | Batch Time=21.875 +GPU:1 | Epoch: 23 | loss=9.886353492736816 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.59417724609375 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.865564346313477 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.669267654418945 | Batch Time=21.09375 +GPU:1 | Epoch: 23 | loss=9.8885498046875 | Batch Time=20.3125 +GPU:1 | Epoch: 23 | loss=9.881126403808594 | Batch Time=13.28125 +GPU:1 | Epoch: 23 | loss=9.805854797363281 | Batch Time=19.53125 +GPU:1 | Epoch: 23 | loss=9.842460632324219 | Batch Time=13.671875 +GPU:1 | Epoch: 23 | loss=9.73526382446289 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.685941696166992 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.727621078491211 | Batch Time=17.578125 +GPU:1 | Epoch: 23 | loss=9.976941108703613 | Batch Time=12.5 +GPU:1 | Epoch: 23 | loss=9.664735794067383 | Batch Time=19.921875 +GPU:1 | Epoch: 23 | loss=9.538698196411133 | Batch Time=22.65625 +GPU:1 | Epoch: 23 | loss=9.830902099609375 | Batch Time=14.0625 +GPU:1 | Epoch: 23 | loss=9.815237998962402 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.848104476928711 | Batch Time=17.578125 +GPU:1 | Epoch: 23 | loss=9.519774436950684 | Batch Time=21.484375 +GPU:1 | Epoch: 23 | loss=9.758285522460938 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.741775512695312 | Batch Time=18.75 +GPU:1 | Epoch: 23 | loss=9.77306842803955 | Batch Time=17.96875 +GPU:1 | Epoch: 23 | loss=9.768489837646484 | Batch Time=15.234375 +GPU:1 | Epoch: 23 | loss=9.8575439453125 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.748282432556152 | Batch Time=20.703125 +GPU:1 | Epoch: 23 | loss=9.80134391784668 | Batch Time=15.625 +GPU:1 | Epoch: 23 | loss=9.691787719726562 | Batch Time=18.359375 +GPU:1 | Epoch: 23 | loss=9.584104537963867 | Batch Time=16.015625 +GPU:1 | Epoch: 23 | loss=9.758054733276367 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.874526977539062 | Batch Time=17.1875 +GPU:1 | Epoch: 23 | loss=9.873823165893555 | Batch Time=16.796875 +GPU:1 | Epoch: 23 | loss=9.572685241699219 | Batch Time=19.921875 +GPU:2 | Epoch: 23 | loss=9.819625854492188 | Batch Time=19.921875 +GPU:2 | Epoch: 23 | loss=9.946502685546875 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=10.038145065307617 | Batch Time=13.671875 +GPU:2 | Epoch: 23 | loss=9.843997955322266 | Batch Time=18.75 +GPU:2 | Epoch: 23 | loss=9.822504043579102 | Batch Time=18.359375 +GPU:2 | Epoch: 23 | loss=9.899782180786133 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.693049430847168 | Batch Time=19.53125 +GPU:2 | Epoch: 23 | loss=9.86081314086914 | Batch Time=19.53125 +GPU:2 | Epoch: 23 | loss=10.039958953857422 | Batch Time=18.359375 +GPU:2 | Epoch: 23 | loss=9.964286804199219 | Batch Time=14.453125 +GPU:2 | Epoch: 23 | loss=9.75093936920166 | Batch Time=18.75 +GPU:2 | Epoch: 23 | loss=9.910165786743164 | Batch Time=18.359375 +GPU:2 | Epoch: 23 | loss=9.587041854858398 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=10.229497909545898 | Batch Time=12.890625 +GPU:2 | Epoch: 23 | loss=9.894075393676758 | Batch Time=15.625 +GPU:2 | Epoch: 23 | loss=9.88553237915039 | Batch Time=19.53125 +GPU:2 | Epoch: 23 | loss=10.013404846191406 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.89390754699707 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=10.054214477539062 | Batch Time=16.40625 +GPU:2 | Epoch: 23 | loss=9.760974884033203 | Batch Time=20.3125 +GPU:2 | Epoch: 23 | loss=10.005842208862305 | Batch Time=14.0625 +GPU:2 | Epoch: 23 | loss=9.850048065185547 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.808713912963867 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=9.924368858337402 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.870162963867188 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.908259391784668 | Batch Time=17.96875 +GPU:2 | Epoch: 23 | loss=9.787515640258789 | Batch Time=15.234375 +GPU:2 | Epoch: 23 | loss=9.695276260375977 | Batch Time=21.484375 +GPU:2 | Epoch: 23 | loss=9.96013069152832 | Batch Time=15.234375 +GPU:2 | Epoch: 23 | loss=10.046014785766602 | Batch Time=14.453125 +GPU:2 | Epoch: 23 | loss=9.797840118408203 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.593432426452637 | Batch Time=17.96875 +GPU:2 | Epoch: 23 | loss=9.705327033996582 | Batch Time=19.921875 +GPU:2 | Epoch: 23 | loss=9.835018157958984 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.83064079284668 | Batch Time=17.96875 +GPU:2 | Epoch: 23 | loss=9.763309478759766 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.92518138885498 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.79726791381836 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=9.933025360107422 | Batch Time=16.796875 +GPU:2 | Epoch: 23 | loss=9.723270416259766 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=9.783062934875488 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.71391487121582 | Batch Time=20.703125 +GPU:2 | Epoch: 23 | loss=9.667156219482422 | Batch Time=20.703125 +GPU:2 | Epoch: 23 | loss=9.907279968261719 | Batch Time=14.84375 +GPU:2 | Epoch: 23 | loss=9.732455253601074 | Batch Time=14.0625 +GPU:2 | Epoch: 23 | loss=10.054885864257812 | Batch Time=13.28125 +GPU:2 | Epoch: 23 | loss=10.050948143005371 | Batch Time=14.84375 +GPU:2 | Epoch: 23 | loss=9.873922348022461 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.700132369995117 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.637862205505371 | Batch Time=20.703125 +GPU:2 | Epoch: 23 | loss=10.004011154174805 | Batch Time=13.28125 +GPU:2 | Epoch: 23 | loss=9.887054443359375 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.788307189941406 | Batch Time=19.53125 +GPU:2 | Epoch: 23 | loss=9.643777847290039 | Batch Time=16.40625 +GPU:2 | Epoch: 23 | loss=9.772634506225586 | Batch Time=17.96875 +GPU:2 | Epoch: 23 | loss=9.782621383666992 | Batch Time=14.84375 +GPU:2 | Epoch: 23 | loss=9.735076904296875 | Batch Time=20.3125 +GPU:2 | Epoch: 23 | loss=9.693044662475586 | Batch Time=18.359375 +GPU:2 | Epoch: 23 | loss=9.961099624633789 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.750753402709961 | Batch Time=21.484375 +GPU:2 | Epoch: 23 | loss=9.70926570892334 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.740094184875488 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.70429801940918 | Batch Time=17.96875 +GPU:2 | Epoch: 23 | loss=9.683004379272461 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.570225715637207 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=9.895904541015625 | Batch Time=14.453125 +GPU:2 | Epoch: 23 | loss=9.88762378692627 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.66364574432373 | Batch Time=22.65625 +GPU:2 | Epoch: 23 | loss=10.144657135009766 | Batch Time=16.40625 +GPU:2 | Epoch: 23 | loss=9.681325912475586 | Batch Time=14.453125 +GPU:2 | Epoch: 23 | loss=9.892410278320312 | Batch Time=12.5 +GPU:2 | Epoch: 23 | loss=10.059760093688965 | Batch Time=12.5 +GPU:2 | Epoch: 23 | loss=9.735339164733887 | Batch Time=20.3125 +GPU:2 | Epoch: 23 | loss=9.76219367980957 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.459901809692383 | Batch Time=25.390625 +GPU:2 | Epoch: 23 | loss=9.483619689941406 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=9.93598747253418 | Batch Time=16.796875 +GPU:2 | Epoch: 23 | loss=9.893022537231445 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.677704811096191 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.693836212158203 | Batch Time=16.40625 +GPU:2 | Epoch: 23 | loss=9.955263137817383 | Batch Time=15.625 +GPU:2 | Epoch: 23 | loss=9.660463333129883 | Batch Time=16.796875 +GPU:2 | Epoch: 23 | loss=9.726503372192383 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.952055931091309 | Batch Time=17.96875 +GPU:2 | Epoch: 23 | loss=9.919357299804688 | Batch Time=14.84375 +GPU:2 | Epoch: 23 | loss=9.824179649353027 | Batch Time=13.671875 +GPU:2 | Epoch: 23 | loss=9.991275787353516 | Batch Time=14.0625 +GPU:2 | Epoch: 23 | loss=9.750284194946289 | Batch Time=16.40625 +GPU:2 | Epoch: 23 | loss=9.619543075561523 | Batch Time=18.75 +GPU:2 | Epoch: 23 | loss=10.001056671142578 | Batch Time=14.0625 +GPU:2 | Epoch: 23 | loss=9.889716148376465 | Batch Time=18.359375 +GPU:2 | Epoch: 23 | loss=9.738826751708984 | Batch Time=19.921875 +GPU:2 | Epoch: 23 | loss=9.913631439208984 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.98971176147461 | Batch Time=14.0625 +GPU:2 | Epoch: 23 | loss=10.024043083190918 | Batch Time=15.234375 +GPU:2 | Epoch: 23 | loss=9.564058303833008 | Batch Time=16.796875 +GPU:2 | Epoch: 23 | loss=9.77212905883789 | Batch Time=18.359375 +GPU:2 | Epoch: 23 | loss=9.82352066040039 | Batch Time=17.96875 +GPU:2 | Epoch: 23 | loss=9.899471282958984 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.54353141784668 | Batch Time=20.3125 +GPU:2 | Epoch: 23 | loss=9.586163520812988 | Batch Time=22.265625 +GPU:2 | Epoch: 23 | loss=9.899985313415527 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.981849670410156 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.696624755859375 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.915042877197266 | Batch Time=16.015625 +GPU:2 | Epoch: 23 | loss=9.84211254119873 | Batch Time=15.625 +GPU:2 | Epoch: 23 | loss=9.595947265625 | Batch Time=17.96875 +GPU:2 | Epoch: 23 | loss=9.62421703338623 | Batch Time=21.484375 +GPU:2 | Epoch: 23 | loss=9.557012557983398 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.744882583618164 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=9.688826560974121 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=9.830924034118652 | Batch Time=20.703125 +GPU:2 | Epoch: 23 | loss=9.660028457641602 | Batch Time=17.1875 +GPU:2 | Epoch: 23 | loss=9.736804962158203 | Batch Time=19.140625 +GPU:2 | Epoch: 23 | loss=9.751508712768555 | Batch Time=20.703125 +GPU:2 | Epoch: 23 | loss=9.773372650146484 | Batch Time=19.921875 +GPU:2 | Epoch: 23 | loss=9.755231857299805 | Batch Time=19.53125 +GPU:2 | Epoch: 23 | loss=9.615335464477539 | Batch Time=20.3125 +GPU:2 | Epoch: 23 | loss=9.685070037841797 | Batch Time=18.75 +GPU:2 | Epoch: 23 | loss=9.7555570602417 | Batch Time=19.53125 +GPU:2 | Epoch: 23 | loss=9.723739624023438 | Batch Time=17.578125 +GPU:2 | Epoch: 23 | loss=9.66726303100586 | Batch Time=20.3125 +GPU:2 | Epoch: 23 | loss=9.658989906311035 | Batch Time=20.3125 +GPU:2 | Epoch: 23 | loss=9.65947151184082 | Batch Time=19.921875 +GPU:2 | Epoch: 23 | loss=9.592857360839844 | Batch Time=20.3125 +GPU:3 | Epoch: 23 | loss=9.95576286315918 | Batch Time=19.140625 +GPU:3 | Epoch: 23 | loss=9.769811630249023 | Batch Time=20.3125 +GPU:3 | Epoch: 23 | loss=9.946992874145508 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.824487686157227 | Batch Time=21.09375 +GPU:3 | Epoch: 23 | loss=9.863594055175781 | Batch Time=16.796875 +GPU:3 | Epoch: 23 | loss=9.888872146606445 | Batch Time=21.09375 +GPU:3 | Epoch: 23 | loss=9.799592971801758 | Batch Time=18.75 +GPU:3 | Epoch: 23 | loss=9.82118034362793 | Batch Time=18.75 +GPU:3 | Epoch: 23 | loss=9.735068321228027 | Batch Time=18.75 +GPU:3 | Epoch: 23 | loss=9.85438346862793 | Batch Time=16.796875 +GPU:3 | Epoch: 23 | loss=9.945219039916992 | Batch Time=15.234375 +GPU:3 | Epoch: 23 | loss=9.919733047485352 | Batch Time=18.75 +GPU:3 | Epoch: 23 | loss=9.930110931396484 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.776582717895508 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.765958786010742 | Batch Time=19.140625 +GPU:3 | Epoch: 23 | loss=10.058433532714844 | Batch Time=12.5 +GPU:3 | Epoch: 23 | loss=9.788098335266113 | Batch Time=15.625 +GPU:3 | Epoch: 23 | loss=10.077789306640625 | Batch Time=14.0625 +GPU:3 | Epoch: 23 | loss=9.951254844665527 | Batch Time=14.84375 +GPU:3 | Epoch: 23 | loss=10.011585235595703 | Batch Time=13.28125 +GPU:3 | Epoch: 23 | loss=10.009761810302734 | Batch Time=16.015625 +GPU:3 | Epoch: 23 | loss=9.982213973999023 | Batch Time=12.890625 +GPU:3 | Epoch: 23 | loss=9.962322235107422 | Batch Time=12.5 +GPU:3 | Epoch: 23 | loss=9.785576820373535 | Batch Time=16.015625 +GPU:3 | Epoch: 23 | loss=9.857763290405273 | Batch Time=20.3125 +GPU:3 | Epoch: 23 | loss=9.735620498657227 | Batch Time=22.65625 +GPU:3 | Epoch: 23 | loss=9.935544967651367 | Batch Time=14.453125 +GPU:3 | Epoch: 23 | loss=9.851886749267578 | Batch Time=15.625 +GPU:3 | Epoch: 23 | loss=9.846540451049805 | Batch Time=15.234375 +GPU:3 | Epoch: 23 | loss=9.86306381225586 | Batch Time=17.578125 +GPU:3 | Epoch: 23 | loss=9.86758804321289 | Batch Time=16.015625 +GPU:3 | Epoch: 23 | loss=9.907772064208984 | Batch Time=14.84375 +GPU:3 | Epoch: 23 | loss=9.889665603637695 | Batch Time=16.796875 +GPU:3 | Epoch: 23 | loss=9.823427200317383 | Batch Time=13.28125 +GPU:3 | Epoch: 23 | loss=9.816370010375977 | Batch Time=17.578125 +GPU:3 | Epoch: 23 | loss=9.760883331298828 | Batch Time=19.921875 +GPU:3 | Epoch: 23 | loss=9.720645904541016 | Batch Time=19.140625 +GPU:3 | Epoch: 23 | loss=9.864495277404785 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.88936996459961 | Batch Time=12.890625 +GPU:3 | Epoch: 23 | loss=10.08674144744873 | Batch Time=14.453125 +GPU:3 | Epoch: 23 | loss=9.608030319213867 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.569223403930664 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.765905380249023 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=10.021831512451172 | Batch Time=10.546875 +GPU:3 | Epoch: 23 | loss=9.943206787109375 | Batch Time=18.75 +GPU:3 | Epoch: 23 | loss=9.79127025604248 | Batch Time=14.0625 +GPU:3 | Epoch: 23 | loss=9.67776870727539 | Batch Time=20.3125 +GPU:3 | Epoch: 23 | loss=9.92739486694336 | Batch Time=15.234375 +GPU:3 | Epoch: 23 | loss=9.850860595703125 | Batch Time=16.015625 +GPU:3 | Epoch: 23 | loss=9.803731918334961 | Batch Time=17.578125 +GPU:3 | Epoch: 23 | loss=9.707359313964844 | Batch Time=20.703125 +GPU:3 | Epoch: 23 | loss=9.698965072631836 | Batch Time=17.578125 +GPU:3 | Epoch: 23 | loss=9.704362869262695 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.702978134155273 | Batch Time=17.578125 +GPU:3 | Epoch: 23 | loss=9.94690227508545 | Batch Time=15.234375 +GPU:3 | Epoch: 23 | loss=9.725564956665039 | Batch Time=21.875 +GPU:3 | Epoch: 23 | loss=9.962238311767578 | Batch Time=15.625 +GPU:3 | Epoch: 23 | loss=9.68886947631836 | Batch Time=18.75 +GPU:3 | Epoch: 23 | loss=9.789766311645508 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.847700119018555 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.57453441619873 | Batch Time=20.703125 +GPU:3 | Epoch: 23 | loss=9.60776138305664 | Batch Time=17.578125 +GPU:3 | Epoch: 23 | loss=9.905044555664062 | Batch Time=16.796875 +GPU:3 | Epoch: 23 | loss=9.80047607421875 | Batch Time=16.796875 +GPU:3 | Epoch: 23 | loss=9.752264022827148 | Batch Time=21.09375 +GPU:3 | Epoch: 23 | loss=9.651639938354492 | Batch Time=21.09375 +GPU:3 | Epoch: 23 | loss=10.005178451538086 | Batch Time=21.484375 +GPU:3 | Epoch: 23 | loss=9.776524543762207 | Batch Time=14.453125 +GPU:3 | Epoch: 23 | loss=9.9444580078125 | Batch Time=13.28125 +GPU:3 | Epoch: 23 | loss=9.84018325805664 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=10.017095565795898 | Batch Time=14.0625 +GPU:3 | Epoch: 23 | loss=9.889471054077148 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.877474784851074 | Batch Time=17.1875 +GPU:3 | Epoch: 23 | loss=9.89760684967041 | Batch Time=18.75 +GPU:3 | Epoch: 23 | loss=9.710604667663574 | Batch Time=20.3125 +GPU:3 | Epoch: 23 | loss=9.605514526367188 | Batch Time=22.65625 +GPU:3 | Epoch: 23 | loss=9.80150318145752 | Batch Time=17.578125 +GPU:3 | Epoch: 23 | loss=9.766576766967773 | Batch Time=13.28125 +GPU:3 | Epoch: 23 | loss=9.87813949584961 | Batch Time=13.28125 +GPU:3 | Epoch: 23 | loss=9.906665802001953 | Batch Time=13.28125 +GPU:3 | Epoch: 23 | loss=9.80710220336914 | Batch Time=12.890625 +GPU:3 | Epoch: 23 | loss=9.730289459228516 | Batch Time=18.75 +GPU:3 | Epoch: 23 | loss=9.77248764038086 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.746828079223633 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.813552856445312 | Batch Time=21.09375 +GPU:3 | Epoch: 23 | loss=9.710441589355469 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.73644733428955 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.8187837600708 | Batch Time=17.1875 +GPU:3 | Epoch: 23 | loss=9.81060791015625 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.990182876586914 | Batch Time=14.453125 +GPU:3 | Epoch: 23 | loss=9.56801700592041 | Batch Time=19.53125 +GPU:3 | Epoch: 23 | loss=9.53049087524414 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.774445533752441 | Batch Time=19.140625 +GPU:3 | Epoch: 23 | loss=9.61298942565918 | Batch Time=19.53125 +GPU:3 | Epoch: 23 | loss=9.919650077819824 | Batch Time=13.28125 +GPU:3 | Epoch: 23 | loss=9.567893981933594 | Batch Time=20.703125 +GPU:3 | Epoch: 23 | loss=9.611824035644531 | Batch Time=16.40625 +GPU:3 | Epoch: 23 | loss=9.907474517822266 | Batch Time=15.234375 +GPU:3 | Epoch: 23 | loss=9.959943771362305 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.794130325317383 | Batch Time=15.625 +GPU:3 | Epoch: 23 | loss=9.805883407592773 | Batch Time=16.015625 +GPU:3 | Epoch: 23 | loss=9.72482967376709 | Batch Time=17.578125 +GPU:3 | Epoch: 23 | loss=9.679010391235352 | Batch Time=19.53125 +GPU:3 | Epoch: 23 | loss=9.656865119934082 | Batch Time=20.703125 +GPU:3 | Epoch: 23 | loss=9.605892181396484 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.877138137817383 | Batch Time=15.625 +GPU:3 | Epoch: 23 | loss=9.831380844116211 | Batch Time=19.921875 +GPU:3 | Epoch: 23 | loss=9.86474323272705 | Batch Time=15.625 +GPU:3 | Epoch: 23 | loss=9.853694915771484 | Batch Time=20.3125 +GPU:3 | Epoch: 23 | loss=9.777107238769531 | Batch Time=12.5 +GPU:3 | Epoch: 23 | loss=9.802188873291016 | Batch Time=15.625 +GPU:3 | Epoch: 23 | loss=9.698418617248535 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.895196914672852 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.693700790405273 | Batch Time=19.53125 +GPU:3 | Epoch: 23 | loss=9.74090576171875 | Batch Time=21.09375 +GPU:3 | Epoch: 23 | loss=9.725564956665039 | Batch Time=20.703125 +GPU:3 | Epoch: 23 | loss=9.644556999206543 | Batch Time=20.3125 +GPU:3 | Epoch: 23 | loss=9.707231521606445 | Batch Time=17.96875 +GPU:3 | Epoch: 23 | loss=9.714717864990234 | Batch Time=21.484375 +GPU:3 | Epoch: 23 | loss=9.45014762878418 | Batch Time=18.359375 +GPU:3 | Epoch: 23 | loss=9.8831148147583 | Batch Time=16.796875 +GPU:3 | Epoch: 23 | loss=9.641487121582031 | Batch Time=19.140625 +GPU:3 | Epoch: 23 | loss=9.606658935546875 | Batch Time=16.796875 +GPU:3 | Epoch: 23 | loss=9.701732635498047 | Batch Time=16.796875 +GPU:3 | Epoch: 23 | loss=9.649048805236816 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.863428115844727 | Batch Time=19.53125 +GPU:0 | Epoch: 23 | loss=9.908760070800781 | Batch Time=14.453125 +GPU:0 | Epoch: 23 | loss=9.980151176452637 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=9.85818862915039 | Batch Time=16.015625 +GPU:0 | Epoch: 23 | loss=10.005428314208984 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.716862678527832 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=9.805028915405273 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.739542007446289 | Batch Time=21.484375 +GPU:0 | Epoch: 23 | loss=9.78269100189209 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=9.982211112976074 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.771900177001953 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=9.828060150146484 | Batch Time=14.84375 +GPU:0 | Epoch: 23 | loss=9.794862747192383 | Batch Time=19.140625 +GPU:0 | Epoch: 23 | loss=9.902013778686523 | Batch Time=16.40625 +GPU:0 | Epoch: 23 | loss=9.917533874511719 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.876266479492188 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=10.008012771606445 | Batch Time=14.0625 +GPU:0 | Epoch: 23 | loss=9.758813858032227 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=9.961735725402832 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=9.648056030273438 | Batch Time=15.234375 +GPU:0 | Epoch: 23 | loss=9.954269409179688 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.877313613891602 | Batch Time=16.015625 +GPU:0 | Epoch: 23 | loss=9.974199295043945 | Batch Time=14.84375 +GPU:0 | Epoch: 23 | loss=9.99165153503418 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.852092742919922 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.688436508178711 | Batch Time=20.703125 +GPU:0 | Epoch: 23 | loss=10.03217887878418 | Batch Time=13.671875 +GPU:0 | Epoch: 23 | loss=9.934931755065918 | Batch Time=14.0625 +GPU:0 | Epoch: 23 | loss=9.866815567016602 | Batch Time=21.09375 +GPU:0 | Epoch: 23 | loss=9.809074401855469 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=9.636640548706055 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.896431922912598 | Batch Time=14.0625 +GPU:0 | Epoch: 23 | loss=9.907011985778809 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.833135604858398 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.893555641174316 | Batch Time=16.796875 +GPU:0 | Epoch: 23 | loss=9.937942504882812 | Batch Time=16.40625 +GPU:0 | Epoch: 23 | loss=9.833017349243164 | Batch Time=16.796875 +GPU:0 | Epoch: 23 | loss=9.540301322937012 | Batch Time=21.875 +GPU:0 | Epoch: 23 | loss=10.032499313354492 | Batch Time=12.109375 +GPU:0 | Epoch: 23 | loss=9.697610855102539 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.794524192810059 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.820508003234863 | Batch Time=20.3125 +GPU:0 | Epoch: 23 | loss=9.989505767822266 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.792779922485352 | Batch Time=16.015625 +GPU:0 | Epoch: 23 | loss=9.759775161743164 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.801737785339355 | Batch Time=13.28125 +GPU:0 | Epoch: 23 | loss=10.035974502563477 | Batch Time=11.71875 +GPU:0 | Epoch: 23 | loss=9.75638198852539 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.744512557983398 | Batch Time=19.921875 +GPU:0 | Epoch: 23 | loss=10.015983581542969 | Batch Time=16.40625 +GPU:0 | Epoch: 23 | loss=9.74258041381836 | Batch Time=20.703125 +GPU:0 | Epoch: 23 | loss=9.925008773803711 | Batch Time=13.28125 +GPU:0 | Epoch: 23 | loss=9.587370872497559 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.901548385620117 | Batch Time=15.234375 +GPU:0 | Epoch: 23 | loss=9.711690902709961 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.746042251586914 | Batch Time=14.453125 +GPU:0 | Epoch: 23 | loss=9.90959644317627 | Batch Time=14.453125 +GPU:0 | Epoch: 23 | loss=9.837052345275879 | Batch Time=16.015625 +GPU:0 | Epoch: 23 | loss=9.91159725189209 | Batch Time=12.5 +GPU:0 | Epoch: 23 | loss=9.80545425415039 | Batch Time=13.28125 +GPU:0 | Epoch: 23 | loss=9.739110946655273 | Batch Time=19.140625 +GPU:0 | Epoch: 23 | loss=9.631867408752441 | Batch Time=19.921875 +GPU:0 | Epoch: 23 | loss=9.822427749633789 | Batch Time=12.5 +GPU:0 | Epoch: 23 | loss=9.771982192993164 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.649852752685547 | Batch Time=21.09375 +GPU:0 | Epoch: 23 | loss=9.41531753540039 | Batch Time=24.21875 +GPU:0 | Epoch: 23 | loss=9.69025993347168 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.899284362792969 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=9.957320213317871 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.762834548950195 | Batch Time=16.40625 +GPU:0 | Epoch: 23 | loss=9.729833602905273 | Batch Time=16.796875 +GPU:0 | Epoch: 23 | loss=9.990942001342773 | Batch Time=11.71875 +GPU:0 | Epoch: 23 | loss=9.715436935424805 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.47335147857666 | Batch Time=21.09375 +GPU:0 | Epoch: 23 | loss=9.894683837890625 | Batch Time=16.796875 +GPU:0 | Epoch: 23 | loss=9.947534561157227 | Batch Time=16.40625 +GPU:0 | Epoch: 23 | loss=9.88542652130127 | Batch Time=13.28125 +GPU:0 | Epoch: 23 | loss=9.806012153625488 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=9.658720970153809 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=9.9139986038208 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.54831600189209 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.599237442016602 | Batch Time=19.921875 +GPU:0 | Epoch: 23 | loss=9.93941879272461 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=10.029581069946289 | Batch Time=19.140625 +GPU:0 | Epoch: 23 | loss=9.968717575073242 | Batch Time=16.015625 +GPU:0 | Epoch: 23 | loss=9.936392784118652 | Batch Time=13.28125 +GPU:0 | Epoch: 23 | loss=9.901449203491211 | Batch Time=14.0625 +GPU:0 | Epoch: 23 | loss=9.789804458618164 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=9.846384048461914 | Batch Time=14.84375 +GPU:0 | Epoch: 23 | loss=10.05142593383789 | Batch Time=15.234375 +GPU:0 | Epoch: 23 | loss=9.644662857055664 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=9.861801147460938 | Batch Time=16.40625 +GPU:0 | Epoch: 23 | loss=10.001310348510742 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=9.874429702758789 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=9.518983840942383 | Batch Time=19.921875 +GPU:0 | Epoch: 23 | loss=9.739202499389648 | Batch Time=19.140625 +GPU:0 | Epoch: 23 | loss=9.67017650604248 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.607452392578125 | Batch Time=21.875 +GPU:0 | Epoch: 23 | loss=9.95709228515625 | Batch Time=16.796875 +GPU:0 | Epoch: 23 | loss=9.791703224182129 | Batch Time=16.796875 +GPU:0 | Epoch: 23 | loss=9.816228866577148 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=9.716157913208008 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.771162033081055 | Batch Time=14.453125 +GPU:0 | Epoch: 23 | loss=9.736518859863281 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=9.815668106079102 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.748266220092773 | Batch Time=19.53125 +GPU:0 | Epoch: 23 | loss=9.566312789916992 | Batch Time=21.484375 +GPU:0 | Epoch: 23 | loss=10.027884483337402 | Batch Time=14.0625 +GPU:0 | Epoch: 23 | loss=9.992918014526367 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.67648696899414 | Batch Time=17.1875 +GPU:0 | Epoch: 23 | loss=9.780465126037598 | Batch Time=14.0625 +GPU:0 | Epoch: 23 | loss=9.539117813110352 | Batch Time=22.65625 +GPU:0 | Epoch: 23 | loss=9.498878479003906 | Batch Time=20.703125 +GPU:0 | Epoch: 23 | loss=9.618646621704102 | Batch Time=23.828125 +GPU:0 | Epoch: 23 | loss=9.630369186401367 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=9.827507019042969 | Batch Time=14.0625 +GPU:0 | Epoch: 23 | loss=9.697347640991211 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=9.816129684448242 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=9.697052001953125 | Batch Time=17.96875 +GPU:0 | Epoch: 23 | loss=9.559355735778809 | Batch Time=23.046875 +GPU:0 | Epoch: 23 | loss=9.939651489257812 | Batch Time=9.375 +GPU:0 | Epoch: 23 | loss=9.86550521850586 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=9.799176216125488 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=9.653486251831055 | Batch Time=16.40625 +GPU:0 | Epoch: 23 | loss=9.650468826293945 | Batch Time=16.40625 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587769344, percent=77.6, used=192066674688, free=17112887296, active=200410615808, inactive=35415764992, buffers=392871936, cached=47995650048, shared=5662158848, slab=2053603328) +AFTER TRAIN LOOP: Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +STARTING TRAINING: Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +BEFORE TRAIN LOOP: Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587769344, percent=77.6, used=192066674688, free=17112887296, active=200410615808, inactive=35415764992, buffers=392871936, cached=47995650048, shared=5662158848, slab=2053603328) +AFTER TRAIN LOOP: Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +STARTING TRAINING: Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +BEFORE TRAIN LOOP: Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587769344, percent=77.6, used=192066674688, free=17112887296, active=200410615808, inactive=35415764992, buffers=392871936, cached=47995650048, shared=5662158848, slab=2053603328) +AFTER TRAIN LOOP: Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +BEFORE VAL LOOP: GPU: 0 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587769344, percent=77.6, used=192066674688, free=17112887296, active=200410615808, inactive=35415764992, buffers=392871936, cached=47995650048, shared=5662158848, slab=2053603328) +AFTER TRAIN LOOP: Epoch 23 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +STARTING TRAINING: Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +BEFORE TRAIN LOOP: Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=57587777536, percent=77.6, used=192066748416, free=17112895488, active=200410566656, inactive=35415732224, buffers=392871936, cached=47995568128, shared=5662076928, slab=2053595136) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 23 | loss=3.4673845767974854 | Batch Time=28.515625 +GPU:0 | Epoch: 23 | loss=4.289470195770264 | Batch Time=12.890625 +GPU:0 | Epoch: 23 | loss=4.3586812019348145 | Batch Time=21.484375 +GPU:0 | Epoch: 23 | loss=4.089710235595703 | Batch Time=12.890625 +GPU:0 | Epoch: 23 | loss=5.044375896453857 | Batch Time=4.296875 +GPU:0 | Epoch: 23 | loss=4.341996669769287 | Batch Time=9.765625 +GPU:0 | Epoch: 23 | loss=4.4026713371276855 | Batch Time=6.25 +GPU:0 | Epoch: 23 | loss=5.406241416931152 | Batch Time=5.859375 +GPU:0 | Epoch: 23 | loss=5.143721103668213 | Batch Time=6.640625 +GPU:0 | Epoch: 23 | loss=5.714439392089844 | Batch Time=1.5625 +GPU:0 | Epoch: 23 | loss=5.168574810028076 | Batch Time=4.296875 +GPU:0 | Epoch: 23 | loss=4.3860931396484375 | Batch Time=22.65625 +GPU:0 | Epoch: 23 | loss=4.971851825714111 | Batch Time=16.015625 +GPU:0 | Epoch: 23 | loss=4.352755069732666 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=4.359926223754883 | Batch Time=13.28125 +GPU:0 | Epoch: 23 | loss=4.8488383293151855 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=4.315492630004883 | Batch Time=14.453125 +GPU:0 | Epoch: 23 | loss=3.790790319442749 | Batch Time=22.65625 +GPU:0 | Epoch: 23 | loss=3.749937057495117 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=3.2843856811523438 | Batch Time=36.328125 +Model top1 Accuracy: 15.928 +Acc before rounding: 15.928 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 23 | loss=3.4673845767974854 | Batch Time=28.515625 +GPU:0 | Epoch: 23 | loss=4.289470195770264 | Batch Time=12.890625 +GPU:0 | Epoch: 23 | loss=4.3586812019348145 | Batch Time=21.484375 +GPU:0 | Epoch: 23 | loss=4.089710235595703 | Batch Time=12.890625 +GPU:0 | Epoch: 23 | loss=5.044375896453857 | Batch Time=4.296875 +GPU:0 | Epoch: 23 | loss=4.341996669769287 | Batch Time=9.765625 +GPU:0 | Epoch: 23 | loss=4.4026713371276855 | Batch Time=6.25 +GPU:0 | Epoch: 23 | loss=5.406241416931152 | Batch Time=5.859375 +GPU:0 | Epoch: 23 | loss=5.143721103668213 | Batch Time=6.640625 +GPU:0 | Epoch: 23 | loss=5.714439392089844 | Batch Time=1.5625 +GPU:0 | Epoch: 23 | loss=5.168574810028076 | Batch Time=4.296875 +GPU:0 | Epoch: 23 | loss=4.3860931396484375 | Batch Time=22.65625 +GPU:0 | Epoch: 23 | loss=4.971851825714111 | Batch Time=16.015625 +GPU:0 | Epoch: 23 | loss=4.352755069732666 | Batch Time=18.359375 +GPU:0 | Epoch: 23 | loss=4.359926223754883 | Batch Time=13.28125 +GPU:0 | Epoch: 23 | loss=4.8488383293151855 | Batch Time=15.625 +GPU:0 | Epoch: 23 | loss=4.315492630004883 | Batch Time=14.453125 +GPU:0 | Epoch: 23 | loss=3.790790319442749 | Batch Time=22.65625 +GPU:0 | Epoch: 23 | loss=3.749937057495117 | Batch Time=18.75 +GPU:0 | Epoch: 23 | loss=3.2843856811523438 | Batch Time=36.328125 +Model top1 Accuracy: 15.928 +Acc after rounding: 15.928 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 23 | loss=4.73309850692749 | Batch Time=15.234375 +GPU:0 | Epoch: 23 | loss=4.472805976867676 | Batch Time=17.578125 +GPU:0 | Epoch: 23 | loss=4.53880500793457 | Batch Time=14.0625 +GPU:0 | Epoch: 23 | loss=4.660302639007568 | Batch Time=14.0625 +Model top1 Accuracy: 14.12 +Validation Acc after rounding: 14.12 +AFTER VAL LOOP: GPU: 0 | Epoch 23 | Memory Usage: svmem(total=257568083968, available=39798833152, percent=84.5, used=209854988288, free=5136904192, active=222385844224, inactive=25306857472, buffers=427835392, cached=42148356096, shared=5662740480, slab=2043523072) +Rounding model with scheme: naive +Model avg sparsity: 37.19159870389816 +GPU:0 | Epoch: 23 | Acc=15.928 | Epoch Time=21.14523075024287 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 23 | Memory Usage: svmem(total=257568083968, available=39798833152, percent=84.5, used=209855250432, free=5135872000, active=222385827840, inactive=25307889664, buffers=427843584, cached=42149117952, shared=5662740480, slab=2043523072) +STARTING TRAINING: Epoch 24 | Memory Usage: svmem(total=257568083968, available=39798833152, percent=84.5, used=209855250432, free=5135872000, active=222385827840, inactive=25307889664, buffers=427843584, cached=42149117952, shared=5662740480, slab=2043523072) +BEFORE TRAIN LOOP: Epoch 24 | Memory Usage: svmem(total=257568083968, available=39798833152, percent=84.5, used=209855250432, free=5135872000, active=222385827840, inactive=25307889664, buffers=427843584, cached=42149117952, shared=5662740480, slab=2043523072) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=39798833152, percent=84.5, used=209855250432, free=5135872000, active=222385827840, inactive=25307889664, buffers=427843584, cached=42149117952, shared=5662740480, slab=2043523072) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 24 | loss=9.760095596313477 | Batch Time=19.921875 +GPU:0 | Epoch: 24 | loss=9.749470710754395 | Batch Time=18.359375 +GPU:0 | Epoch: 24 | loss=9.621770858764648 | Batch Time=18.75 +GPU:0 | Epoch: 24 | loss=9.616861343383789 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.708541870117188 | Batch Time=21.09375 +GPU:0 | Epoch: 24 | loss=9.881372451782227 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.872228622436523 | Batch Time=14.0625 +GPU:0 | Epoch: 24 | loss=9.760149002075195 | Batch Time=15.625 +GPU:0 | Epoch: 24 | loss=9.615423202514648 | Batch Time=21.09375 +GPU:0 | Epoch: 24 | loss=9.638691902160645 | Batch Time=17.96875 +GPU:0 | Epoch: 24 | loss=9.714155197143555 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.617538452148438 | Batch Time=18.75 +GPU:0 | Epoch: 24 | loss=9.726095199584961 | Batch Time=21.484375 +GPU:0 | Epoch: 24 | loss=9.659008026123047 | Batch Time=19.53125 +GPU:0 | Epoch: 24 | loss=9.770025253295898 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.619094848632812 | Batch Time=23.4375 +GPU:0 | Epoch: 24 | loss=9.652181625366211 | Batch Time=19.921875 +GPU:0 | Epoch: 24 | loss=9.772241592407227 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.525065422058105 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.558531761169434 | Batch Time=21.09375 +GPU:0 | Epoch: 24 | loss=9.666635513305664 | Batch Time=19.921875 +GPU:0 | Epoch: 24 | loss=9.716856002807617 | Batch Time=20.3125 +GPU:0 | Epoch: 24 | loss=9.765087127685547 | Batch Time=18.359375 +GPU:0 | Epoch: 24 | loss=9.839448928833008 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.781639099121094 | Batch Time=16.40625 +GPU:0 | Epoch: 24 | loss=9.835415840148926 | Batch Time=13.671875 +GPU:0 | Epoch: 24 | loss=9.775012969970703 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.715360641479492 | Batch Time=19.53125 +GPU:0 | Epoch: 24 | loss=9.773704528808594 | Batch Time=15.625 +GPU:0 | Epoch: 24 | loss=9.87905502319336 | Batch Time=18.75 +GPU:0 | Epoch: 24 | loss=9.81787109375 | Batch Time=16.015625 +GPU:0 | Epoch: 24 | loss=9.64492416381836 | Batch Time=21.484375 +GPU:0 | Epoch: 24 | loss=9.578262329101562 | Batch Time=20.703125 +GPU:0 | Epoch: 24 | loss=9.714792251586914 | Batch Time=19.140625 +GPU:0 | Epoch: 24 | loss=9.729669570922852 | Batch Time=16.40625 +GPU:0 | Epoch: 24 | loss=9.839677810668945 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.639432907104492 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.675777435302734 | Batch Time=23.046875 +GPU:0 | Epoch: 24 | loss=9.59930419921875 | Batch Time=21.875 +GPU:0 | Epoch: 24 | loss=9.561548233032227 | Batch Time=22.265625 +GPU:0 | Epoch: 24 | loss=9.62942123413086 | Batch Time=18.75 +GPU:0 | Epoch: 24 | loss=9.842569351196289 | Batch Time=18.75 +GPU:0 | Epoch: 24 | loss=9.668313980102539 | Batch Time=19.140625 +GPU:0 | Epoch: 24 | loss=9.82003402709961 | Batch Time=14.453125 +GPU:0 | Epoch: 24 | loss=9.772758483886719 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.665925979614258 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.744691848754883 | Batch Time=19.140625 +GPU:0 | Epoch: 24 | loss=9.892187118530273 | Batch Time=20.703125 +GPU:0 | Epoch: 24 | loss=9.666906356811523 | Batch Time=18.359375 +GPU:0 | Epoch: 24 | loss=9.799798965454102 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.705049514770508 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.82470989227295 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.805709838867188 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.694866180419922 | Batch Time=19.53125 +GPU:0 | Epoch: 24 | loss=9.544994354248047 | Batch Time=22.265625 +GPU:0 | Epoch: 24 | loss=9.801811218261719 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.511301040649414 | Batch Time=21.484375 +GPU:0 | Epoch: 24 | loss=9.616561889648438 | Batch Time=21.484375 +GPU:0 | Epoch: 24 | loss=9.907118797302246 | Batch Time=14.0625 +GPU:0 | Epoch: 24 | loss=9.49709701538086 | Batch Time=21.09375 +GPU:0 | Epoch: 24 | loss=9.796501159667969 | Batch Time=19.140625 +GPU:0 | Epoch: 24 | loss=9.730915069580078 | Batch Time=13.28125 +GPU:0 | Epoch: 24 | loss=9.510786056518555 | Batch Time=19.53125 +GPU:0 | Epoch: 24 | loss=9.574478149414062 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.945493698120117 | Batch Time=13.28125 +GPU:0 | Epoch: 24 | loss=9.616073608398438 | Batch Time=19.53125 +GPU:0 | Epoch: 24 | loss=9.596612930297852 | Batch Time=21.875 +GPU:0 | Epoch: 24 | loss=9.950811386108398 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.663077354431152 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.686246871948242 | Batch Time=20.703125 +GPU:0 | Epoch: 24 | loss=9.646257400512695 | Batch Time=19.921875 +GPU:0 | Epoch: 24 | loss=9.680381774902344 | Batch Time=20.703125 +GPU:0 | Epoch: 24 | loss=9.889472007751465 | Batch Time=13.671875 +GPU:0 | Epoch: 24 | loss=9.635431289672852 | Batch Time=14.84375 +GPU:0 | Epoch: 24 | loss=9.890368461608887 | Batch Time=15.625 +GPU:0 | Epoch: 24 | loss=9.878385543823242 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.542821884155273 | Batch Time=16.015625 +GPU:0 | Epoch: 24 | loss=9.654870986938477 | Batch Time=17.96875 +GPU:0 | Epoch: 24 | loss=9.34312629699707 | Batch Time=24.21875 +GPU:0 | Epoch: 24 | loss=9.64187240600586 | Batch Time=22.265625 +GPU:0 | Epoch: 24 | loss=9.658824920654297 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.583586692810059 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.557319641113281 | Batch Time=19.53125 +GPU:0 | Epoch: 24 | loss=9.540889739990234 | Batch Time=17.96875 +GPU:0 | Epoch: 24 | loss=9.675796508789062 | Batch Time=12.890625 +GPU:0 | Epoch: 24 | loss=9.511259078979492 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.661031723022461 | Batch Time=21.875 +GPU:0 | Epoch: 24 | loss=9.560407638549805 | Batch Time=18.75 +GPU:0 | Epoch: 24 | loss=9.632671356201172 | Batch Time=18.359375 +GPU:0 | Epoch: 24 | loss=9.516196250915527 | Batch Time=14.84375 +GPU:0 | Epoch: 24 | loss=9.510660171508789 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.678068161010742 | Batch Time=17.96875 +GPU:0 | Epoch: 24 | loss=9.682687759399414 | Batch Time=15.625 +GPU:0 | Epoch: 24 | loss=9.782646179199219 | Batch Time=21.09375 +GPU:0 | Epoch: 24 | loss=9.540021896362305 | Batch Time=21.09375 +GPU:0 | Epoch: 24 | loss=9.719725608825684 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.655872344970703 | Batch Time=19.921875 +GPU:0 | Epoch: 24 | loss=9.563833236694336 | Batch Time=19.921875 +GPU:0 | Epoch: 24 | loss=9.83384895324707 | Batch Time=16.015625 +GPU:0 | Epoch: 24 | loss=9.508621215820312 | Batch Time=19.53125 +GPU:0 | Epoch: 24 | loss=9.775534629821777 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.600648880004883 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.490006446838379 | Batch Time=19.921875 +GPU:0 | Epoch: 24 | loss=9.706795692443848 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.654157638549805 | Batch Time=23.046875 +GPU:0 | Epoch: 24 | loss=9.76860237121582 | Batch Time=14.84375 +GPU:0 | Epoch: 24 | loss=9.697500228881836 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.687585830688477 | Batch Time=14.0625 +GPU:0 | Epoch: 24 | loss=9.657658576965332 | Batch Time=17.96875 +GPU:0 | Epoch: 24 | loss=9.525402069091797 | Batch Time=18.75 +GPU:0 | Epoch: 24 | loss=9.583261489868164 | Batch Time=18.75 +GPU:0 | Epoch: 24 | loss=9.558350563049316 | Batch Time=16.40625 +GPU:0 | Epoch: 24 | loss=9.386131286621094 | Batch Time=21.09375 +GPU:0 | Epoch: 24 | loss=9.817546844482422 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.788837432861328 | Batch Time=14.453125 +GPU:0 | Epoch: 24 | loss=9.563199996948242 | Batch Time=17.1875 +GPU:0 | Epoch: 24 | loss=9.455667495727539 | Batch Time=21.875 +GPU:0 | Epoch: 24 | loss=9.478323936462402 | Batch Time=16.796875 +GPU:0 | Epoch: 24 | loss=9.937606811523438 | Batch Time=14.0625 +GPU:0 | Epoch: 24 | loss=9.449594497680664 | Batch Time=22.265625 +GPU:0 | Epoch: 24 | loss=9.742921829223633 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=9.552425384521484 | Batch Time=19.921875 +GPU:0 | Epoch: 24 | loss=9.38852310180664 | Batch Time=24.609375 +GPU:0 | Epoch: 24 | loss=9.466485977172852 | Batch Time=20.703125 +GPU:0 | Epoch: 24 | loss=9.515968322753906 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.60551929473877 | Batch Time=19.53125 +GPU:2 | Epoch: 24 | loss=9.540666580200195 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.781972885131836 | Batch Time=16.40625 +GPU:2 | Epoch: 24 | loss=9.578625679016113 | Batch Time=17.578125 +GPU:2 | Epoch: 24 | loss=9.7876615524292 | Batch Time=17.1875 +GPU:2 | Epoch: 24 | loss=9.751602172851562 | Batch Time=20.703125 +GPU:2 | Epoch: 24 | loss=9.903474807739258 | Batch Time=12.5 +GPU:2 | Epoch: 24 | loss=9.553321838378906 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.696415901184082 | Batch Time=17.578125 +GPU:2 | Epoch: 24 | loss=9.534954071044922 | Batch Time=19.140625 +GPU:2 | Epoch: 24 | loss=9.901158332824707 | Batch Time=10.9375 +GPU:2 | Epoch: 24 | loss=9.575475692749023 | Batch Time=22.265625 +GPU:2 | Epoch: 24 | loss=9.809640884399414 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.682746887207031 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.813745498657227 | Batch Time=17.578125 +GPU:2 | Epoch: 24 | loss=9.738649368286133 | Batch Time=16.015625 +GPU:2 | Epoch: 24 | loss=9.783395767211914 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.6306791305542 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.601271629333496 | Batch Time=17.1875 +GPU:2 | Epoch: 24 | loss=9.789055824279785 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.746442794799805 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.677905082702637 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.60783576965332 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.61372184753418 | Batch Time=25.390625 +GPU:2 | Epoch: 24 | loss=9.911352157592773 | Batch Time=14.0625 +GPU:2 | Epoch: 24 | loss=9.737321853637695 | Batch Time=16.796875 +GPU:2 | Epoch: 24 | loss=9.82383918762207 | Batch Time=17.1875 +GPU:2 | Epoch: 24 | loss=9.608926773071289 | Batch Time=20.3125 +GPU:2 | Epoch: 24 | loss=9.765907287597656 | Batch Time=16.40625 +GPU:2 | Epoch: 24 | loss=9.819597244262695 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.785736083984375 | Batch Time=19.140625 +GPU:2 | Epoch: 24 | loss=9.579723358154297 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.569701194763184 | Batch Time=21.09375 +GPU:2 | Epoch: 24 | loss=9.492523193359375 | Batch Time=23.828125 +GPU:2 | Epoch: 24 | loss=9.793511390686035 | Batch Time=16.015625 +GPU:2 | Epoch: 24 | loss=9.778230667114258 | Batch Time=14.0625 +GPU:2 | Epoch: 24 | loss=9.758846282958984 | Batch Time=14.84375 +GPU:2 | Epoch: 24 | loss=9.693506240844727 | Batch Time=21.484375 +GPU:2 | Epoch: 24 | loss=9.599992752075195 | Batch Time=20.703125 +GPU:2 | Epoch: 24 | loss=9.473371505737305 | Batch Time=22.65625 +GPU:2 | Epoch: 24 | loss=9.644543647766113 | Batch Time=17.578125 +GPU:2 | Epoch: 24 | loss=9.813986778259277 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.536528587341309 | Batch Time=16.40625 +GPU:2 | Epoch: 24 | loss=9.565767288208008 | Batch Time=19.53125 +GPU:2 | Epoch: 24 | loss=9.929553985595703 | Batch Time=14.453125 +GPU:2 | Epoch: 24 | loss=9.846092224121094 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.556814193725586 | Batch Time=21.484375 +GPU:2 | Epoch: 24 | loss=9.718528747558594 | Batch Time=15.625 +GPU:2 | Epoch: 24 | loss=9.750565528869629 | Batch Time=16.796875 +GPU:2 | Epoch: 24 | loss=9.636713027954102 | Batch Time=16.015625 +GPU:2 | Epoch: 24 | loss=9.87093734741211 | Batch Time=16.015625 +GPU:2 | Epoch: 24 | loss=9.598457336425781 | Batch Time=19.140625 +GPU:2 | Epoch: 24 | loss=9.479679107666016 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.817995071411133 | Batch Time=17.578125 +GPU:2 | Epoch: 24 | loss=9.913402557373047 | Batch Time=12.890625 +GPU:2 | Epoch: 24 | loss=9.558418273925781 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.797683715820312 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.752961158752441 | Batch Time=21.09375 +GPU:2 | Epoch: 24 | loss=9.626445770263672 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.586612701416016 | Batch Time=21.484375 +GPU:2 | Epoch: 24 | loss=9.828054428100586 | Batch Time=15.625 +GPU:2 | Epoch: 24 | loss=9.547280311584473 | Batch Time=20.703125 +GPU:2 | Epoch: 24 | loss=9.692490577697754 | Batch Time=17.1875 +GPU:2 | Epoch: 24 | loss=9.572416305541992 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.769824981689453 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.629838943481445 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.795812606811523 | Batch Time=12.5 +GPU:2 | Epoch: 24 | loss=9.565110206604004 | Batch Time=23.046875 +GPU:2 | Epoch: 24 | loss=9.597143173217773 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.755882263183594 | Batch Time=16.015625 +GPU:2 | Epoch: 24 | loss=9.4708251953125 | Batch Time=23.046875 +GPU:2 | Epoch: 24 | loss=9.752015113830566 | Batch Time=14.0625 +GPU:2 | Epoch: 24 | loss=9.672597885131836 | Batch Time=16.40625 +GPU:2 | Epoch: 24 | loss=9.581563949584961 | Batch Time=21.09375 +GPU:2 | Epoch: 24 | loss=9.556425094604492 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.775487899780273 | Batch Time=17.1875 +GPU:2 | Epoch: 24 | loss=9.72406005859375 | Batch Time=21.484375 +GPU:2 | Epoch: 24 | loss=9.705787658691406 | Batch Time=20.3125 +GPU:2 | Epoch: 24 | loss=9.587549209594727 | Batch Time=16.40625 +GPU:2 | Epoch: 24 | loss=9.900077819824219 | Batch Time=14.453125 +GPU:2 | Epoch: 24 | loss=9.736226081848145 | Batch Time=14.453125 +GPU:2 | Epoch: 24 | loss=9.680603981018066 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.511516571044922 | Batch Time=21.875 +GPU:2 | Epoch: 24 | loss=9.57819938659668 | Batch Time=23.828125 +GPU:2 | Epoch: 24 | loss=9.677395820617676 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.742411613464355 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.58675765991211 | Batch Time=20.3125 +GPU:2 | Epoch: 24 | loss=9.88751220703125 | Batch Time=13.671875 +GPU:2 | Epoch: 24 | loss=9.515026092529297 | Batch Time=16.796875 +GPU:2 | Epoch: 24 | loss=9.68513011932373 | Batch Time=16.796875 +GPU:2 | Epoch: 24 | loss=9.477762222290039 | Batch Time=25.0 +GPU:2 | Epoch: 24 | loss=9.798556327819824 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.713525772094727 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.598712921142578 | Batch Time=17.578125 +GPU:2 | Epoch: 24 | loss=9.410453796386719 | Batch Time=19.53125 +GPU:2 | Epoch: 24 | loss=9.678688049316406 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.294053077697754 | Batch Time=23.4375 +GPU:2 | Epoch: 24 | loss=9.608086585998535 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.514970779418945 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.714618682861328 | Batch Time=16.796875 +GPU:2 | Epoch: 24 | loss=9.775551795959473 | Batch Time=16.015625 +GPU:2 | Epoch: 24 | loss=9.411834716796875 | Batch Time=19.53125 +GPU:2 | Epoch: 24 | loss=9.479747772216797 | Batch Time=19.53125 +GPU:2 | Epoch: 24 | loss=9.617944717407227 | Batch Time=19.140625 +GPU:2 | Epoch: 24 | loss=9.766009330749512 | Batch Time=15.234375 +GPU:2 | Epoch: 24 | loss=9.523835182189941 | Batch Time=18.359375 +GPU:2 | Epoch: 24 | loss=9.566783905029297 | Batch Time=23.4375 +GPU:2 | Epoch: 24 | loss=9.594139099121094 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.817508697509766 | Batch Time=14.0625 +GPU:2 | Epoch: 24 | loss=9.297920227050781 | Batch Time=25.0 +GPU:2 | Epoch: 24 | loss=9.609694480895996 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.631044387817383 | Batch Time=20.3125 +GPU:2 | Epoch: 24 | loss=9.569061279296875 | Batch Time=17.578125 +GPU:2 | Epoch: 24 | loss=9.614604949951172 | Batch Time=19.921875 +GPU:2 | Epoch: 24 | loss=9.582653999328613 | Batch Time=20.3125 +GPU:2 | Epoch: 24 | loss=9.582841873168945 | Batch Time=15.625 +GPU:2 | Epoch: 24 | loss=9.630790710449219 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.425893783569336 | Batch Time=20.703125 +GPU:2 | Epoch: 24 | loss=9.675483703613281 | Batch Time=17.578125 +GPU:2 | Epoch: 24 | loss=9.57551097869873 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.559364318847656 | Batch Time=19.53125 +GPU:2 | Epoch: 24 | loss=9.703083038330078 | Batch Time=18.75 +GPU:2 | Epoch: 24 | loss=9.569732666015625 | Batch Time=16.796875 +GPU:2 | Epoch: 24 | loss=9.555822372436523 | Batch Time=17.96875 +GPU:2 | Epoch: 24 | loss=9.58108139038086 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.923089981079102 | Batch Time=16.796875 +GPU:3 | Epoch: 24 | loss=9.903022766113281 | Batch Time=14.84375 +GPU:3 | Epoch: 24 | loss=9.579221725463867 | Batch Time=21.875 +GPU:3 | Epoch: 24 | loss=9.541933059692383 | Batch Time=20.3125 +GPU:3 | Epoch: 24 | loss=9.708897590637207 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.682748794555664 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.63990592956543 | Batch Time=19.921875 +GPU:3 | Epoch: 24 | loss=9.67318344116211 | Batch Time=17.1875 +GPU:3 | Epoch: 24 | loss=9.582566261291504 | Batch Time=20.703125 +GPU:3 | Epoch: 24 | loss=9.708797454833984 | Batch Time=17.578125 +GPU:3 | Epoch: 24 | loss=9.577046394348145 | Batch Time=20.703125 +GPU:3 | Epoch: 24 | loss=9.941621780395508 | Batch Time=17.578125 +GPU:3 | Epoch: 24 | loss=9.804654121398926 | Batch Time=14.0625 +GPU:3 | Epoch: 24 | loss=9.753103256225586 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.819053649902344 | Batch Time=15.234375 +GPU:3 | Epoch: 24 | loss=9.666936874389648 | Batch Time=20.703125 +GPU:3 | Epoch: 24 | loss=9.774173736572266 | Batch Time=20.3125 +GPU:3 | Epoch: 24 | loss=9.681640625 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.828401565551758 | Batch Time=17.1875 +GPU:3 | Epoch: 24 | loss=9.810930252075195 | Batch Time=16.796875 +GPU:3 | Epoch: 24 | loss=9.630170822143555 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.600793838500977 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.491178512573242 | Batch Time=19.53125 +GPU:3 | Epoch: 24 | loss=9.666727066040039 | Batch Time=21.875 +GPU:3 | Epoch: 24 | loss=9.604524612426758 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.82301139831543 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.62840461730957 | Batch Time=16.796875 +GPU:3 | Epoch: 24 | loss=9.867828369140625 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.759805679321289 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.751062393188477 | Batch Time=17.1875 +GPU:3 | Epoch: 24 | loss=9.729848861694336 | Batch Time=21.875 +GPU:3 | Epoch: 24 | loss=9.778838157653809 | Batch Time=18.75 +GPU:3 | Epoch: 24 | loss=9.881967544555664 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.901334762573242 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.587387084960938 | Batch Time=21.875 +GPU:3 | Epoch: 24 | loss=9.759379386901855 | Batch Time=14.453125 +GPU:3 | Epoch: 24 | loss=9.609369277954102 | Batch Time=21.484375 +GPU:3 | Epoch: 24 | loss=9.634946823120117 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.774248123168945 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.649587631225586 | Batch Time=17.96875 +GPU:3 | Epoch: 24 | loss=9.606118202209473 | Batch Time=20.3125 +GPU:3 | Epoch: 24 | loss=9.852010726928711 | Batch Time=15.234375 +GPU:3 | Epoch: 24 | loss=9.733258247375488 | Batch Time=18.75 +GPU:3 | Epoch: 24 | loss=9.704019546508789 | Batch Time=16.796875 +GPU:3 | Epoch: 24 | loss=9.806265830993652 | Batch Time=14.84375 +GPU:3 | Epoch: 24 | loss=9.53205680847168 | Batch Time=21.875 +GPU:3 | Epoch: 24 | loss=9.754026412963867 | Batch Time=17.1875 +GPU:3 | Epoch: 24 | loss=9.510029792785645 | Batch Time=19.921875 +GPU:3 | Epoch: 24 | loss=9.712465286254883 | Batch Time=19.53125 +GPU:3 | Epoch: 24 | loss=10.014030456542969 | Batch Time=16.796875 +GPU:3 | Epoch: 24 | loss=9.737149238586426 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.65176773071289 | Batch Time=20.703125 +GPU:3 | Epoch: 24 | loss=9.563596725463867 | Batch Time=17.578125 +GPU:3 | Epoch: 24 | loss=9.775136947631836 | Batch Time=20.3125 +GPU:3 | Epoch: 24 | loss=9.679323196411133 | Batch Time=19.53125 +GPU:3 | Epoch: 24 | loss=9.83413314819336 | Batch Time=17.578125 +GPU:3 | Epoch: 24 | loss=9.781938552856445 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.648688316345215 | Batch Time=19.921875 +GPU:3 | Epoch: 24 | loss=9.603357315063477 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.796177864074707 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.790098190307617 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.637487411499023 | Batch Time=15.234375 +GPU:3 | Epoch: 24 | loss=9.606963157653809 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.56828498840332 | Batch Time=18.75 +GPU:3 | Epoch: 24 | loss=9.702051162719727 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.606832504272461 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.860885620117188 | Batch Time=14.453125 +GPU:3 | Epoch: 24 | loss=9.474861145019531 | Batch Time=21.09375 +GPU:3 | Epoch: 24 | loss=9.57961368560791 | Batch Time=21.09375 +GPU:3 | Epoch: 24 | loss=9.737839698791504 | Batch Time=16.015625 +GPU:3 | Epoch: 24 | loss=9.906996726989746 | Batch Time=14.0625 +GPU:3 | Epoch: 24 | loss=9.476587295532227 | Batch Time=17.578125 +GPU:3 | Epoch: 24 | loss=9.651902198791504 | Batch Time=19.921875 +GPU:3 | Epoch: 24 | loss=9.636789321899414 | Batch Time=21.09375 +GPU:3 | Epoch: 24 | loss=9.692291259765625 | Batch Time=15.625 +GPU:3 | Epoch: 24 | loss=9.715868949890137 | Batch Time=17.96875 +GPU:3 | Epoch: 24 | loss=9.744176864624023 | Batch Time=14.84375 +GPU:3 | Epoch: 24 | loss=9.58066177368164 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.83069896697998 | Batch Time=14.0625 +GPU:3 | Epoch: 24 | loss=9.56923770904541 | Batch Time=21.09375 +GPU:3 | Epoch: 24 | loss=9.906283378601074 | Batch Time=16.796875 +GPU:3 | Epoch: 24 | loss=9.70358657836914 | Batch Time=17.578125 +GPU:3 | Epoch: 24 | loss=9.584272384643555 | Batch Time=17.1875 +GPU:3 | Epoch: 24 | loss=9.677778244018555 | Batch Time=19.53125 +GPU:3 | Epoch: 24 | loss=9.591562271118164 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.535234451293945 | Batch Time=23.828125 +GPU:3 | Epoch: 24 | loss=9.5338773727417 | Batch Time=17.1875 +GPU:3 | Epoch: 24 | loss=9.385263442993164 | Batch Time=21.875 +GPU:3 | Epoch: 24 | loss=9.547079086303711 | Batch Time=16.015625 +GPU:3 | Epoch: 24 | loss=9.792181015014648 | Batch Time=17.1875 +GPU:3 | Epoch: 24 | loss=9.650590896606445 | Batch Time=18.75 +GPU:3 | Epoch: 24 | loss=9.723219871520996 | Batch Time=16.015625 +GPU:3 | Epoch: 24 | loss=9.503833770751953 | Batch Time=26.5625 +GPU:3 | Epoch: 24 | loss=9.824145317077637 | Batch Time=14.84375 +GPU:3 | Epoch: 24 | loss=9.636575698852539 | Batch Time=16.015625 +GPU:3 | Epoch: 24 | loss=9.683470726013184 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.688247680664062 | Batch Time=18.75 +GPU:3 | Epoch: 24 | loss=9.584579467773438 | Batch Time=17.578125 +GPU:3 | Epoch: 24 | loss=9.531728744506836 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.50490951538086 | Batch Time=19.921875 +GPU:3 | Epoch: 24 | loss=9.771848678588867 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.566282272338867 | Batch Time=21.875 +GPU:3 | Epoch: 24 | loss=9.445411682128906 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.747756958007812 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.64247989654541 | Batch Time=21.09375 +GPU:3 | Epoch: 24 | loss=9.518305778503418 | Batch Time=16.796875 +GPU:3 | Epoch: 24 | loss=9.795045852661133 | Batch Time=19.53125 +GPU:3 | Epoch: 24 | loss=9.720836639404297 | Batch Time=18.75 +GPU:3 | Epoch: 24 | loss=9.534868240356445 | Batch Time=18.359375 +GPU:3 | Epoch: 24 | loss=9.402711868286133 | Batch Time=21.09375 +GPU:3 | Epoch: 24 | loss=9.595939636230469 | Batch Time=17.578125 +GPU:3 | Epoch: 24 | loss=9.543352127075195 | Batch Time=19.140625 +GPU:3 | Epoch: 24 | loss=9.466615676879883 | Batch Time=19.921875 +GPU:3 | Epoch: 24 | loss=9.366864204406738 | Batch Time=22.265625 +GPU:3 | Epoch: 24 | loss=9.754387855529785 | Batch Time=21.09375 +GPU:3 | Epoch: 24 | loss=9.612872123718262 | Batch Time=18.75 +GPU:3 | Epoch: 24 | loss=9.829358100891113 | Batch Time=17.1875 +GPU:3 | Epoch: 24 | loss=9.289081573486328 | Batch Time=21.484375 +GPU:3 | Epoch: 24 | loss=9.515913009643555 | Batch Time=19.921875 +GPU:3 | Epoch: 24 | loss=9.601762771606445 | Batch Time=21.484375 +GPU:3 | Epoch: 24 | loss=9.43004035949707 | Batch Time=21.484375 +GPU:3 | Epoch: 24 | loss=9.455656051635742 | Batch Time=17.96875 +GPU:3 | Epoch: 24 | loss=9.585660934448242 | Batch Time=20.703125 +GPU:3 | Epoch: 24 | loss=9.620792388916016 | Batch Time=16.40625 +GPU:3 | Epoch: 24 | loss=9.60380744934082 | Batch Time=18.359375 +GPU:1 | Epoch: 24 | loss=9.817760467529297 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.890047073364258 | Batch Time=13.671875 +GPU:1 | Epoch: 24 | loss=9.627866744995117 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.70973014831543 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.846866607666016 | Batch Time=16.015625 +GPU:1 | Epoch: 24 | loss=9.625367164611816 | Batch Time=21.875 +GPU:1 | Epoch: 24 | loss=9.85103988647461 | Batch Time=14.84375 +GPU:1 | Epoch: 24 | loss=9.781448364257812 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.488316535949707 | Batch Time=21.875 +GPU:1 | Epoch: 24 | loss=9.77271842956543 | Batch Time=17.578125 +GPU:1 | Epoch: 24 | loss=9.664291381835938 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.75540828704834 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=9.892844200134277 | Batch Time=15.234375 +GPU:1 | Epoch: 24 | loss=9.472315788269043 | Batch Time=18.359375 +GPU:1 | Epoch: 24 | loss=9.569805145263672 | Batch Time=20.703125 +GPU:1 | Epoch: 24 | loss=9.739080429077148 | Batch Time=15.625 +GPU:1 | Epoch: 24 | loss=9.71333122253418 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.77074146270752 | Batch Time=17.96875 +GPU:1 | Epoch: 24 | loss=9.874155044555664 | Batch Time=17.1875 +GPU:1 | Epoch: 24 | loss=9.559650421142578 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.84186840057373 | Batch Time=15.625 +GPU:1 | Epoch: 24 | loss=9.676015853881836 | Batch Time=17.96875 +GPU:1 | Epoch: 24 | loss=9.966811180114746 | Batch Time=16.015625 +GPU:1 | Epoch: 24 | loss=9.66586685180664 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.898061752319336 | Batch Time=17.1875 +GPU:1 | Epoch: 24 | loss=9.631271362304688 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.74496841430664 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.862800598144531 | Batch Time=15.625 +GPU:1 | Epoch: 24 | loss=9.550710678100586 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.777019500732422 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.787818908691406 | Batch Time=17.1875 +GPU:1 | Epoch: 24 | loss=9.883615493774414 | Batch Time=17.578125 +GPU:1 | Epoch: 24 | loss=9.829273223876953 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.863761901855469 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=10.011272430419922 | Batch Time=14.453125 +GPU:1 | Epoch: 24 | loss=9.798144340515137 | Batch Time=15.234375 +GPU:1 | Epoch: 24 | loss=9.834074020385742 | Batch Time=18.359375 +GPU:1 | Epoch: 24 | loss=9.767566680908203 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=9.677282333374023 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.818172454833984 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.504949569702148 | Batch Time=20.3125 +GPU:1 | Epoch: 24 | loss=9.653623580932617 | Batch Time=15.625 +GPU:1 | Epoch: 24 | loss=9.640460968017578 | Batch Time=22.265625 +GPU:1 | Epoch: 24 | loss=9.67436408996582 | Batch Time=18.359375 +GPU:1 | Epoch: 24 | loss=9.769908905029297 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.819000244140625 | Batch Time=15.625 +GPU:1 | Epoch: 24 | loss=9.492963790893555 | Batch Time=18.75 +GPU:1 | Epoch: 24 | loss=9.733427047729492 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.81205940246582 | Batch Time=17.1875 +GPU:1 | Epoch: 24 | loss=9.560464859008789 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.500911712646484 | Batch Time=25.0 +GPU:1 | Epoch: 24 | loss=9.734891891479492 | Batch Time=15.234375 +GPU:1 | Epoch: 24 | loss=9.853240966796875 | Batch Time=17.1875 +GPU:1 | Epoch: 24 | loss=9.60484504699707 | Batch Time=18.359375 +GPU:1 | Epoch: 24 | loss=9.464559555053711 | Batch Time=20.703125 +GPU:1 | Epoch: 24 | loss=9.469510078430176 | Batch Time=25.0 +GPU:1 | Epoch: 24 | loss=9.560518264770508 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.568069458007812 | Batch Time=20.703125 +GPU:1 | Epoch: 24 | loss=9.881402969360352 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.717340469360352 | Batch Time=20.3125 +GPU:1 | Epoch: 24 | loss=9.713190078735352 | Batch Time=17.1875 +GPU:1 | Epoch: 24 | loss=9.586713790893555 | Batch Time=15.625 +GPU:1 | Epoch: 24 | loss=9.705072402954102 | Batch Time=21.09375 +GPU:1 | Epoch: 24 | loss=9.868927001953125 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.668670654296875 | Batch Time=17.578125 +GPU:1 | Epoch: 24 | loss=9.54454231262207 | Batch Time=22.265625 +GPU:1 | Epoch: 24 | loss=9.373659133911133 | Batch Time=22.65625 +GPU:1 | Epoch: 24 | loss=9.671348571777344 | Batch Time=15.625 +GPU:1 | Epoch: 24 | loss=9.528251647949219 | Batch Time=20.3125 +GPU:1 | Epoch: 24 | loss=9.694757461547852 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.543617248535156 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.407652854919434 | Batch Time=22.265625 +GPU:1 | Epoch: 24 | loss=9.723224639892578 | Batch Time=17.578125 +GPU:1 | Epoch: 24 | loss=9.7236328125 | Batch Time=17.578125 +GPU:1 | Epoch: 24 | loss=9.640162467956543 | Batch Time=16.015625 +GPU:1 | Epoch: 24 | loss=9.774209976196289 | Batch Time=15.234375 +GPU:1 | Epoch: 24 | loss=9.62755012512207 | Batch Time=18.75 +GPU:1 | Epoch: 24 | loss=9.36192512512207 | Batch Time=24.21875 +GPU:1 | Epoch: 24 | loss=9.653411865234375 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=9.65919303894043 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.479568481445312 | Batch Time=18.359375 +GPU:1 | Epoch: 24 | loss=9.808774948120117 | Batch Time=13.28125 +GPU:1 | Epoch: 24 | loss=9.776647567749023 | Batch Time=18.75 +GPU:1 | Epoch: 24 | loss=9.577495574951172 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.533975601196289 | Batch Time=18.75 +GPU:1 | Epoch: 24 | loss=9.714078903198242 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.46548080444336 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.724031448364258 | Batch Time=17.578125 +GPU:1 | Epoch: 24 | loss=9.660943031311035 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.686038970947266 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.563508033752441 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=9.632150650024414 | Batch Time=18.75 +GPU:1 | Epoch: 24 | loss=9.397356986999512 | Batch Time=20.703125 +GPU:1 | Epoch: 24 | loss=9.877935409545898 | Batch Time=14.0625 +GPU:1 | Epoch: 24 | loss=9.491004943847656 | Batch Time=22.65625 +GPU:1 | Epoch: 24 | loss=9.66676139831543 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=9.475211143493652 | Batch Time=21.09375 +GPU:1 | Epoch: 24 | loss=9.456972122192383 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.81292724609375 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.625236511230469 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.464507102966309 | Batch Time=21.484375 +GPU:1 | Epoch: 24 | loss=9.94395637512207 | Batch Time=17.96875 +GPU:1 | Epoch: 24 | loss=9.452388763427734 | Batch Time=17.1875 +GPU:1 | Epoch: 24 | loss=9.655773162841797 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.634478569030762 | Batch Time=16.015625 +GPU:1 | Epoch: 24 | loss=9.701075553894043 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=9.72413444519043 | Batch Time=16.015625 +GPU:1 | Epoch: 24 | loss=9.58053970336914 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.645675659179688 | Batch Time=17.96875 +GPU:1 | Epoch: 24 | loss=9.44786262512207 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.633472442626953 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.592338562011719 | Batch Time=16.796875 +GPU:1 | Epoch: 24 | loss=9.504007339477539 | Batch Time=19.921875 +GPU:1 | Epoch: 24 | loss=9.708623886108398 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=9.547306060791016 | Batch Time=18.359375 +GPU:1 | Epoch: 24 | loss=9.397741317749023 | Batch Time=22.65625 +GPU:1 | Epoch: 24 | loss=9.778861999511719 | Batch Time=17.1875 +GPU:1 | Epoch: 24 | loss=9.648702621459961 | Batch Time=18.75 +GPU:1 | Epoch: 24 | loss=9.79793643951416 | Batch Time=11.71875 +GPU:1 | Epoch: 24 | loss=9.529319763183594 | Batch Time=19.53125 +GPU:1 | Epoch: 24 | loss=9.388327598571777 | Batch Time=20.703125 +GPU:1 | Epoch: 24 | loss=9.734834671020508 | Batch Time=14.453125 +GPU:1 | Epoch: 24 | loss=9.694290161132812 | Batch Time=19.140625 +GPU:1 | Epoch: 24 | loss=9.796777725219727 | Batch Time=16.40625 +GPU:1 | Epoch: 24 | loss=9.706317901611328 | Batch Time=16.796875 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=50034348032, percent=80.6, used=199619747840, free=16885796864, active=204198891520, inactive=32071020544, buffers=391098368, cached=40671440896, shared=5662453760, slab=1757052928) +AFTER TRAIN LOOP: Epoch 24 | Memory Usage: svmem(total=257568083968, available=50035220480, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070979584, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +BEFORE VAL LOOP: GPU: 0 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=50035220480, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070979584, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +AFTER TRAIN LOOP: Epoch 24 | Memory Usage: svmem(total=257568083968, available=50035220480, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070979584, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +STARTING TRAINING: Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +BEFORE TRAIN LOOP: Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=50035220480, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070979584, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +AFTER TRAIN LOOP: Epoch 24 | Memory Usage: svmem(total=257568083968, available=50035220480, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070979584, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +STARTING TRAINING: Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +BEFORE TRAIN LOOP: Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=50034348032, percent=80.6, used=199619747840, free=16885796864, active=204198891520, inactive=32071020544, buffers=391098368, cached=40671440896, shared=5662453760, slab=1757052928) +AFTER TRAIN LOOP: Epoch 24 | Memory Usage: svmem(total=257568083968, available=50035220480, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070979584, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +STARTING TRAINING: Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +BEFORE TRAIN LOOP: Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=50035232768, percent=80.6, used=199619145728, free=16886620160, active=204198420480, inactive=32070991872, buffers=391098368, cached=40671219712, shared=5662232576, slab=1757011968) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 24 | loss=3.364086151123047 | Batch Time=30.078125 +GPU:0 | Epoch: 24 | loss=4.472740173339844 | Batch Time=12.109375 +GPU:0 | Epoch: 24 | loss=4.223179340362549 | Batch Time=27.34375 +GPU:0 | Epoch: 24 | loss=4.232739448547363 | Batch Time=7.03125 +GPU:0 | Epoch: 24 | loss=5.0835723876953125 | Batch Time=4.6875 +GPU:0 | Epoch: 24 | loss=4.691171646118164 | Batch Time=8.59375 +GPU:0 | Epoch: 24 | loss=4.079068183898926 | Batch Time=10.546875 +GPU:0 | Epoch: 24 | loss=5.102906227111816 | Batch Time=7.03125 +GPU:0 | Epoch: 24 | loss=4.9520745277404785 | Batch Time=11.328125 +GPU:0 | Epoch: 24 | loss=5.789401054382324 | Batch Time=2.34375 +GPU:0 | Epoch: 24 | loss=5.281717777252197 | Batch Time=3.90625 +GPU:0 | Epoch: 24 | loss=4.212261199951172 | Batch Time=23.828125 +GPU:0 | Epoch: 24 | loss=5.0013628005981445 | Batch Time=13.28125 +GPU:0 | Epoch: 24 | loss=4.357899188995361 | Batch Time=20.3125 +GPU:0 | Epoch: 24 | loss=4.26167106628418 | Batch Time=11.328125 +GPU:0 | Epoch: 24 | loss=4.627721786499023 | Batch Time=13.671875 +GPU:0 | Epoch: 24 | loss=4.404160499572754 | Batch Time=9.765625 +GPU:0 | Epoch: 24 | loss=3.477104663848877 | Batch Time=27.34375 +GPU:0 | Epoch: 24 | loss=3.6561508178710938 | Batch Time=26.5625 +GPU:0 | Epoch: 24 | loss=3.2287681102752686 | Batch Time=32.8125 +Model top1 Accuracy: 16.558 +Acc before rounding: 16.558 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 24 | loss=3.364086151123047 | Batch Time=30.078125 +GPU:0 | Epoch: 24 | loss=4.472740173339844 | Batch Time=12.109375 +GPU:0 | Epoch: 24 | loss=4.223179340362549 | Batch Time=27.34375 +GPU:0 | Epoch: 24 | loss=4.232739448547363 | Batch Time=7.03125 +GPU:0 | Epoch: 24 | loss=5.0835723876953125 | Batch Time=4.6875 +GPU:0 | Epoch: 24 | loss=4.691171646118164 | Batch Time=8.59375 +GPU:0 | Epoch: 24 | loss=4.079068183898926 | Batch Time=10.546875 +GPU:0 | Epoch: 24 | loss=5.102906227111816 | Batch Time=7.03125 +GPU:0 | Epoch: 24 | loss=4.9520745277404785 | Batch Time=11.328125 +GPU:0 | Epoch: 24 | loss=5.789401054382324 | Batch Time=2.34375 +GPU:0 | Epoch: 24 | loss=5.281717777252197 | Batch Time=3.90625 +GPU:0 | Epoch: 24 | loss=4.212261199951172 | Batch Time=23.828125 +GPU:0 | Epoch: 24 | loss=5.0013628005981445 | Batch Time=13.28125 +GPU:0 | Epoch: 24 | loss=4.357899188995361 | Batch Time=20.3125 +GPU:0 | Epoch: 24 | loss=4.26167106628418 | Batch Time=11.328125 +GPU:0 | Epoch: 24 | loss=4.627721786499023 | Batch Time=13.671875 +GPU:0 | Epoch: 24 | loss=4.404160499572754 | Batch Time=9.765625 +GPU:0 | Epoch: 24 | loss=3.477104663848877 | Batch Time=27.34375 +GPU:0 | Epoch: 24 | loss=3.6561508178710938 | Batch Time=26.5625 +GPU:0 | Epoch: 24 | loss=3.2287681102752686 | Batch Time=32.8125 +Model top1 Accuracy: 16.558 +Acc after rounding: 16.558 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 24 | loss=4.742581367492676 | Batch Time=12.5 +GPU:0 | Epoch: 24 | loss=4.500916481018066 | Batch Time=14.0625 +GPU:0 | Epoch: 24 | loss=4.490696430206299 | Batch Time=17.578125 +GPU:0 | Epoch: 24 | loss=4.725709438323975 | Batch Time=12.109375 +Model top1 Accuracy: 14.01 +Validation Acc after rounding: 14.01 +AFTER VAL LOOP: GPU: 0 | Epoch 24 | Memory Usage: svmem(total=257568083968, available=32224595968, percent=87.5, used=217429303296, free=5366001664, active=225919291392, inactive=21770231808, buffers=427769856, cached=34345009152, shared=5662601216, slab=1742499840) +Rounding model with scheme: naive +Model avg sparsity: 36.66199144049164 +GPU:0 | Epoch: 24 | Acc=16.558 | Epoch Time=21.786791590849557 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 24 | Memory Usage: svmem(total=257568083968, available=32224755712, percent=87.5, used=217429221376, free=5364703232, active=225919242240, inactive=21771636736, buffers=427786240, cached=34346373120, shared=5662519296, slab=1742508032) +STARTING TRAINING: Epoch 25 | Memory Usage: svmem(total=257568083968, available=32224755712, percent=87.5, used=217429221376, free=5364703232, active=225919242240, inactive=21771636736, buffers=427786240, cached=34346373120, shared=5662519296, slab=1742508032) +BEFORE TRAIN LOOP: Epoch 25 | Memory Usage: svmem(total=257568083968, available=32224755712, percent=87.5, used=217429221376, free=5364703232, active=225919242240, inactive=21771636736, buffers=427786240, cached=34346373120, shared=5662519296, slab=1742508032) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=32224755712, percent=87.5, used=217429221376, free=5364703232, active=225919242240, inactive=21771636736, buffers=427786240, cached=34346373120, shared=5662519296, slab=1742508032) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:3 | Epoch: 25 | loss=9.722232818603516 | Batch Time=19.140625 +GPU:3 | Epoch: 25 | loss=9.51770305633545 | Batch Time=22.65625 +GPU:3 | Epoch: 25 | loss=9.695542335510254 | Batch Time=20.3125 +GPU:3 | Epoch: 25 | loss=9.43324089050293 | Batch Time=21.484375 +GPU:3 | Epoch: 25 | loss=9.74470043182373 | Batch Time=14.84375 +GPU:3 | Epoch: 25 | loss=9.860928535461426 | Batch Time=13.28125 +GPU:3 | Epoch: 25 | loss=9.342668533325195 | Batch Time=25.390625 +GPU:3 | Epoch: 25 | loss=9.752864837646484 | Batch Time=16.40625 +GPU:3 | Epoch: 25 | loss=9.599266052246094 | Batch Time=15.625 +GPU:3 | Epoch: 25 | loss=9.467247009277344 | Batch Time=19.921875 +GPU:3 | Epoch: 25 | loss=9.741588592529297 | Batch Time=15.625 +GPU:3 | Epoch: 25 | loss=9.604966163635254 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.782615661621094 | Batch Time=12.890625 +GPU:3 | Epoch: 25 | loss=9.4911527633667 | Batch Time=22.65625 +GPU:3 | Epoch: 25 | loss=9.350046157836914 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.625636100769043 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.627120971679688 | Batch Time=17.578125 +GPU:3 | Epoch: 25 | loss=9.325132369995117 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.384349822998047 | Batch Time=20.3125 +GPU:3 | Epoch: 25 | loss=9.481937408447266 | Batch Time=21.09375 +GPU:3 | Epoch: 25 | loss=9.464652061462402 | Batch Time=19.921875 +GPU:3 | Epoch: 25 | loss=9.797285079956055 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.510136604309082 | Batch Time=20.3125 +GPU:3 | Epoch: 25 | loss=9.742036819458008 | Batch Time=16.796875 +GPU:3 | Epoch: 25 | loss=9.317235946655273 | Batch Time=21.09375 +GPU:3 | Epoch: 25 | loss=9.42677116394043 | Batch Time=21.484375 +GPU:3 | Epoch: 25 | loss=9.71694564819336 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.443281173706055 | Batch Time=22.65625 +GPU:3 | Epoch: 25 | loss=9.502115249633789 | Batch Time=20.703125 +GPU:3 | Epoch: 25 | loss=9.581517219543457 | Batch Time=17.578125 +GPU:3 | Epoch: 25 | loss=9.614173889160156 | Batch Time=17.578125 +GPU:3 | Epoch: 25 | loss=9.591650009155273 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.456380844116211 | Batch Time=18.359375 +GPU:3 | Epoch: 25 | loss=9.554464340209961 | Batch Time=21.484375 +GPU:3 | Epoch: 25 | loss=9.485620498657227 | Batch Time=20.3125 +GPU:3 | Epoch: 25 | loss=9.765182495117188 | Batch Time=14.453125 +GPU:3 | Epoch: 25 | loss=9.659852981567383 | Batch Time=16.796875 +GPU:3 | Epoch: 25 | loss=9.634998321533203 | Batch Time=15.234375 +GPU:3 | Epoch: 25 | loss=9.419336318969727 | Batch Time=17.578125 +GPU:3 | Epoch: 25 | loss=9.667470932006836 | Batch Time=17.578125 +GPU:3 | Epoch: 25 | loss=9.64498519897461 | Batch Time=13.28125 +GPU:3 | Epoch: 25 | loss=9.40351390838623 | Batch Time=20.703125 +GPU:3 | Epoch: 25 | loss=9.458386421203613 | Batch Time=17.96875 +GPU:3 | Epoch: 25 | loss=9.378570556640625 | Batch Time=25.0 +GPU:3 | Epoch: 25 | loss=9.677391052246094 | Batch Time=17.96875 +GPU:3 | Epoch: 25 | loss=9.625622749328613 | Batch Time=16.796875 +GPU:3 | Epoch: 25 | loss=9.44405746459961 | Batch Time=23.046875 +GPU:3 | Epoch: 25 | loss=9.312477111816406 | Batch Time=21.484375 +GPU:3 | Epoch: 25 | loss=9.713109970092773 | Batch Time=18.75 +GPU:3 | Epoch: 25 | loss=9.569754600524902 | Batch Time=16.40625 +GPU:3 | Epoch: 25 | loss=9.46839427947998 | Batch Time=16.796875 +GPU:3 | Epoch: 25 | loss=9.509880065917969 | Batch Time=16.40625 +GPU:3 | Epoch: 25 | loss=9.625131607055664 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.801729202270508 | Batch Time=17.578125 +GPU:3 | Epoch: 25 | loss=9.689193725585938 | Batch Time=15.625 +GPU:3 | Epoch: 25 | loss=9.542791366577148 | Batch Time=17.578125 +GPU:3 | Epoch: 25 | loss=9.451348304748535 | Batch Time=17.96875 +GPU:3 | Epoch: 25 | loss=9.460994720458984 | Batch Time=21.09375 +GPU:3 | Epoch: 25 | loss=9.268648147583008 | Batch Time=23.828125 +GPU:3 | Epoch: 25 | loss=9.65493392944336 | Batch Time=15.625 +GPU:3 | Epoch: 25 | loss=9.697281837463379 | Batch Time=18.75 +GPU:3 | Epoch: 25 | loss=9.608619689941406 | Batch Time=18.75 +GPU:3 | Epoch: 25 | loss=9.467103958129883 | Batch Time=20.3125 +GPU:3 | Epoch: 25 | loss=9.358980178833008 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.481504440307617 | Batch Time=20.3125 +GPU:3 | Epoch: 25 | loss=9.715784072875977 | Batch Time=18.75 +GPU:3 | Epoch: 25 | loss=9.435205459594727 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.36214542388916 | Batch Time=19.921875 +GPU:3 | Epoch: 25 | loss=9.578682899475098 | Batch Time=20.3125 +GPU:3 | Epoch: 25 | loss=9.62604808807373 | Batch Time=19.921875 +GPU:3 | Epoch: 25 | loss=9.522709846496582 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.614267349243164 | Batch Time=17.96875 +GPU:3 | Epoch: 25 | loss=9.619291305541992 | Batch Time=16.40625 +GPU:3 | Epoch: 25 | loss=9.542365074157715 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.487680435180664 | Batch Time=19.921875 +GPU:3 | Epoch: 25 | loss=9.609495162963867 | Batch Time=20.3125 +GPU:3 | Epoch: 25 | loss=9.530309677124023 | Batch Time=17.96875 +GPU:3 | Epoch: 25 | loss=9.530900001525879 | Batch Time=21.484375 +GPU:3 | Epoch: 25 | loss=9.488821029663086 | Batch Time=18.75 +GPU:3 | Epoch: 25 | loss=9.566702842712402 | Batch Time=13.28125 +GPU:3 | Epoch: 25 | loss=9.739051818847656 | Batch Time=14.0625 +GPU:3 | Epoch: 25 | loss=9.7381010055542 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.738356590270996 | Batch Time=15.625 +GPU:3 | Epoch: 25 | loss=9.47392463684082 | Batch Time=18.75 +GPU:3 | Epoch: 25 | loss=9.67685317993164 | Batch Time=16.015625 +GPU:3 | Epoch: 25 | loss=9.501961708068848 | Batch Time=22.65625 +GPU:3 | Epoch: 25 | loss=9.503721237182617 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.579704284667969 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.792186737060547 | Batch Time=14.0625 +GPU:3 | Epoch: 25 | loss=9.383170127868652 | Batch Time=20.703125 +GPU:3 | Epoch: 25 | loss=9.38357925415039 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.518244743347168 | Batch Time=18.75 +GPU:3 | Epoch: 25 | loss=9.620054244995117 | Batch Time=16.796875 +GPU:3 | Epoch: 25 | loss=9.334489822387695 | Batch Time=19.921875 +GPU:3 | Epoch: 25 | loss=9.42210578918457 | Batch Time=19.140625 +GPU:3 | Epoch: 25 | loss=9.667186737060547 | Batch Time=13.28125 +GPU:3 | Epoch: 25 | loss=9.792835235595703 | Batch Time=16.015625 +GPU:3 | Epoch: 25 | loss=9.41109848022461 | Batch Time=19.140625 +GPU:3 | Epoch: 25 | loss=9.59543514251709 | Batch Time=15.234375 +GPU:3 | Epoch: 25 | loss=9.373685836791992 | Batch Time=19.140625 +GPU:3 | Epoch: 25 | loss=9.652116775512695 | Batch Time=14.453125 +GPU:3 | Epoch: 25 | loss=9.404505729675293 | Batch Time=16.015625 +GPU:3 | Epoch: 25 | loss=9.525209426879883 | Batch Time=22.65625 +GPU:3 | Epoch: 25 | loss=9.414737701416016 | Batch Time=22.265625 +GPU:3 | Epoch: 25 | loss=9.529769897460938 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.437003135681152 | Batch Time=18.75 +GPU:3 | Epoch: 25 | loss=9.70321273803711 | Batch Time=12.5 +GPU:3 | Epoch: 25 | loss=9.322604179382324 | Batch Time=20.703125 +GPU:3 | Epoch: 25 | loss=9.350067138671875 | Batch Time=17.1875 +GPU:3 | Epoch: 25 | loss=9.430719375610352 | Batch Time=21.875 +GPU:3 | Epoch: 25 | loss=9.459930419921875 | Batch Time=23.046875 +GPU:3 | Epoch: 25 | loss=9.675958633422852 | Batch Time=16.796875 +GPU:3 | Epoch: 25 | loss=9.575238227844238 | Batch Time=17.578125 +GPU:3 | Epoch: 25 | loss=9.372557640075684 | Batch Time=18.359375 +GPU:3 | Epoch: 25 | loss=9.611312866210938 | Batch Time=20.703125 +GPU:3 | Epoch: 25 | loss=9.53580093383789 | Batch Time=16.796875 +GPU:3 | Epoch: 25 | loss=9.380138397216797 | Batch Time=18.359375 +GPU:3 | Epoch: 25 | loss=9.73881721496582 | Batch Time=16.796875 +GPU:3 | Epoch: 25 | loss=9.606917381286621 | Batch Time=16.015625 +GPU:3 | Epoch: 25 | loss=9.418424606323242 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.393321990966797 | Batch Time=22.265625 +GPU:3 | Epoch: 25 | loss=9.47313117980957 | Batch Time=19.53125 +GPU:3 | Epoch: 25 | loss=9.71750259399414 | Batch Time=18.359375 +GPU:3 | Epoch: 25 | loss=9.4344482421875 | Batch Time=21.484375 +GPU:3 | Epoch: 25 | loss=9.367727279663086 | Batch Time=19.140625 +GPU:2 | Epoch: 25 | loss=9.773635864257812 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.646244049072266 | Batch Time=19.53125 +GPU:2 | Epoch: 25 | loss=9.670612335205078 | Batch Time=16.796875 +GPU:2 | Epoch: 25 | loss=9.564295768737793 | Batch Time=17.96875 +GPU:2 | Epoch: 25 | loss=9.651817321777344 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.65776252746582 | Batch Time=17.578125 +GPU:2 | Epoch: 25 | loss=9.386529922485352 | Batch Time=23.828125 +GPU:2 | Epoch: 25 | loss=9.661548614501953 | Batch Time=17.96875 +GPU:2 | Epoch: 25 | loss=9.760597229003906 | Batch Time=19.140625 +GPU:2 | Epoch: 25 | loss=9.58056640625 | Batch Time=17.1875 +GPU:2 | Epoch: 25 | loss=9.462345123291016 | Batch Time=18.359375 +GPU:2 | Epoch: 25 | loss=9.422586441040039 | Batch Time=21.875 +GPU:2 | Epoch: 25 | loss=9.555680274963379 | Batch Time=14.0625 +GPU:2 | Epoch: 25 | loss=9.789132118225098 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.254716873168945 | Batch Time=26.171875 +GPU:2 | Epoch: 25 | loss=9.552545547485352 | Batch Time=19.921875 +GPU:2 | Epoch: 25 | loss=9.571125984191895 | Batch Time=17.578125 +GPU:2 | Epoch: 25 | loss=9.657964706420898 | Batch Time=19.921875 +GPU:2 | Epoch: 25 | loss=9.56485366821289 | Batch Time=15.625 +GPU:2 | Epoch: 25 | loss=9.673685073852539 | Batch Time=21.09375 +GPU:2 | Epoch: 25 | loss=9.356887817382812 | Batch Time=22.65625 +GPU:2 | Epoch: 25 | loss=9.809473037719727 | Batch Time=17.578125 +GPU:2 | Epoch: 25 | loss=9.69696044921875 | Batch Time=20.703125 +GPU:2 | Epoch: 25 | loss=9.512511253356934 | Batch Time=19.140625 +GPU:2 | Epoch: 25 | loss=9.475443840026855 | Batch Time=15.234375 +GPU:2 | Epoch: 25 | loss=9.333462715148926 | Batch Time=20.703125 +GPU:2 | Epoch: 25 | loss=9.4290771484375 | Batch Time=21.875 +GPU:2 | Epoch: 25 | loss=9.50439739227295 | Batch Time=20.3125 +GPU:2 | Epoch: 25 | loss=9.540488243103027 | Batch Time=21.875 +GPU:2 | Epoch: 25 | loss=9.574567794799805 | Batch Time=21.09375 +GPU:2 | Epoch: 25 | loss=9.774116516113281 | Batch Time=15.234375 +GPU:2 | Epoch: 25 | loss=9.307226181030273 | Batch Time=20.703125 +GPU:2 | Epoch: 25 | loss=9.608627319335938 | Batch Time=16.015625 +GPU:2 | Epoch: 25 | loss=9.63011360168457 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.895386695861816 | Batch Time=16.015625 +GPU:2 | Epoch: 25 | loss=9.703597068786621 | Batch Time=17.578125 +GPU:2 | Epoch: 25 | loss=9.486671447753906 | Batch Time=17.1875 +GPU:2 | Epoch: 25 | loss=9.709756851196289 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.581300735473633 | Batch Time=20.3125 +GPU:2 | Epoch: 25 | loss=9.26878547668457 | Batch Time=23.4375 +GPU:2 | Epoch: 25 | loss=9.318319320678711 | Batch Time=23.4375 +GPU:2 | Epoch: 25 | loss=9.598915100097656 | Batch Time=16.796875 +GPU:2 | Epoch: 25 | loss=9.66989517211914 | Batch Time=17.1875 +GPU:2 | Epoch: 25 | loss=9.698598861694336 | Batch Time=18.359375 +GPU:2 | Epoch: 25 | loss=9.631516456604004 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.619376182556152 | Batch Time=21.875 +GPU:2 | Epoch: 25 | loss=9.437082290649414 | Batch Time=17.1875 +GPU:2 | Epoch: 25 | loss=9.636457443237305 | Batch Time=19.53125 +GPU:2 | Epoch: 25 | loss=9.551877975463867 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.454167366027832 | Batch Time=19.53125 +GPU:2 | Epoch: 25 | loss=9.566717147827148 | Batch Time=14.84375 +GPU:2 | Epoch: 25 | loss=9.499906539916992 | Batch Time=18.359375 +GPU:2 | Epoch: 25 | loss=9.408754348754883 | Batch Time=19.921875 +GPU:2 | Epoch: 25 | loss=9.373650550842285 | Batch Time=17.96875 +GPU:2 | Epoch: 25 | loss=9.41887092590332 | Batch Time=21.484375 +GPU:2 | Epoch: 25 | loss=9.648505210876465 | Batch Time=19.53125 +GPU:2 | Epoch: 25 | loss=9.572856903076172 | Batch Time=19.140625 +GPU:2 | Epoch: 25 | loss=9.577230453491211 | Batch Time=20.703125 +GPU:2 | Epoch: 25 | loss=9.604206085205078 | Batch Time=16.796875 +GPU:2 | Epoch: 25 | loss=9.559769630432129 | Batch Time=19.140625 +GPU:2 | Epoch: 25 | loss=9.57335090637207 | Batch Time=17.96875 +GPU:2 | Epoch: 25 | loss=9.475153923034668 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.558420181274414 | Batch Time=16.796875 +GPU:2 | Epoch: 25 | loss=9.543670654296875 | Batch Time=15.234375 +GPU:2 | Epoch: 25 | loss=9.368599891662598 | Batch Time=22.265625 +GPU:2 | Epoch: 25 | loss=9.713165283203125 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.733427047729492 | Batch Time=19.140625 +GPU:2 | Epoch: 25 | loss=9.709146499633789 | Batch Time=17.578125 +GPU:2 | Epoch: 25 | loss=9.547101974487305 | Batch Time=16.015625 +GPU:2 | Epoch: 25 | loss=9.609138488769531 | Batch Time=17.1875 +GPU:2 | Epoch: 25 | loss=9.681157112121582 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.73641586303711 | Batch Time=17.96875 +GPU:2 | Epoch: 25 | loss=9.603288650512695 | Batch Time=19.140625 +GPU:2 | Epoch: 25 | loss=9.764274597167969 | Batch Time=15.625 +GPU:2 | Epoch: 25 | loss=9.360933303833008 | Batch Time=23.046875 +GPU:2 | Epoch: 25 | loss=9.46658992767334 | Batch Time=21.09375 +GPU:2 | Epoch: 25 | loss=9.755210876464844 | Batch Time=15.234375 +GPU:2 | Epoch: 25 | loss=9.559776306152344 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.70205307006836 | Batch Time=14.0625 +GPU:2 | Epoch: 25 | loss=9.244794845581055 | Batch Time=25.390625 +GPU:2 | Epoch: 25 | loss=9.566495895385742 | Batch Time=21.09375 +GPU:2 | Epoch: 25 | loss=9.607858657836914 | Batch Time=16.015625 +GPU:2 | Epoch: 25 | loss=9.482123374938965 | Batch Time=21.875 +GPU:2 | Epoch: 25 | loss=9.642594337463379 | Batch Time=14.0625 +GPU:2 | Epoch: 25 | loss=9.461875915527344 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.400985717773438 | Batch Time=17.96875 +GPU:2 | Epoch: 25 | loss=9.64419174194336 | Batch Time=16.796875 +GPU:2 | Epoch: 25 | loss=9.480069160461426 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.52285099029541 | Batch Time=25.78125 +GPU:2 | Epoch: 25 | loss=9.81240463256836 | Batch Time=16.796875 +GPU:2 | Epoch: 25 | loss=9.554032325744629 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.578312873840332 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.589900970458984 | Batch Time=16.40625 +GPU:2 | Epoch: 25 | loss=9.462135314941406 | Batch Time=20.703125 +GPU:2 | Epoch: 25 | loss=9.234703063964844 | Batch Time=20.703125 +GPU:2 | Epoch: 25 | loss=9.59249496459961 | Batch Time=18.359375 +GPU:2 | Epoch: 25 | loss=9.504289627075195 | Batch Time=20.3125 +GPU:2 | Epoch: 25 | loss=9.63733196258545 | Batch Time=19.140625 +GPU:2 | Epoch: 25 | loss=9.720771789550781 | Batch Time=16.796875 +GPU:2 | Epoch: 25 | loss=9.425161361694336 | Batch Time=19.921875 +GPU:2 | Epoch: 25 | loss=9.540359497070312 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.556455612182617 | Batch Time=20.3125 +GPU:2 | Epoch: 25 | loss=9.405292510986328 | Batch Time=24.609375 +GPU:2 | Epoch: 25 | loss=9.66834831237793 | Batch Time=15.625 +GPU:2 | Epoch: 25 | loss=9.181119918823242 | Batch Time=23.4375 +GPU:2 | Epoch: 25 | loss=9.371183395385742 | Batch Time=18.359375 +GPU:2 | Epoch: 25 | loss=9.680305480957031 | Batch Time=16.796875 +GPU:2 | Epoch: 25 | loss=9.691372871398926 | Batch Time=19.921875 +GPU:2 | Epoch: 25 | loss=9.48771858215332 | Batch Time=18.359375 +GPU:2 | Epoch: 25 | loss=9.478177070617676 | Batch Time=19.53125 +GPU:2 | Epoch: 25 | loss=9.32553768157959 | Batch Time=19.921875 +GPU:2 | Epoch: 25 | loss=9.401317596435547 | Batch Time=20.3125 +GPU:2 | Epoch: 25 | loss=9.421318054199219 | Batch Time=19.921875 +GPU:2 | Epoch: 25 | loss=9.435312271118164 | Batch Time=18.359375 +GPU:2 | Epoch: 25 | loss=9.357709884643555 | Batch Time=23.828125 +GPU:2 | Epoch: 25 | loss=9.449363708496094 | Batch Time=17.96875 +GPU:2 | Epoch: 25 | loss=9.558537483215332 | Batch Time=18.359375 +GPU:2 | Epoch: 25 | loss=9.555143356323242 | Batch Time=19.53125 +GPU:2 | Epoch: 25 | loss=9.353927612304688 | Batch Time=22.265625 +GPU:2 | Epoch: 25 | loss=9.195819854736328 | Batch Time=25.390625 +GPU:2 | Epoch: 25 | loss=9.38776969909668 | Batch Time=23.046875 +GPU:2 | Epoch: 25 | loss=9.534942626953125 | Batch Time=17.96875 +GPU:2 | Epoch: 25 | loss=9.569177627563477 | Batch Time=14.84375 +GPU:2 | Epoch: 25 | loss=9.641254425048828 | Batch Time=18.75 +GPU:2 | Epoch: 25 | loss=9.455596923828125 | Batch Time=21.875 +GPU:1 | Epoch: 25 | loss=9.68139934539795 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.485784530639648 | Batch Time=22.65625 +GPU:1 | Epoch: 25 | loss=9.944188117980957 | Batch Time=15.234375 +GPU:1 | Epoch: 25 | loss=9.534669876098633 | Batch Time=20.3125 +GPU:1 | Epoch: 25 | loss=9.521931648254395 | Batch Time=22.265625 +GPU:1 | Epoch: 25 | loss=9.649391174316406 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.664015769958496 | Batch Time=19.53125 +GPU:1 | Epoch: 25 | loss=9.596329689025879 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.45478630065918 | Batch Time=15.234375 +GPU:1 | Epoch: 25 | loss=9.780926704406738 | Batch Time=17.578125 +GPU:1 | Epoch: 25 | loss=9.529407501220703 | Batch Time=20.703125 +GPU:1 | Epoch: 25 | loss=9.36172103881836 | Batch Time=20.703125 +GPU:1 | Epoch: 25 | loss=9.455429077148438 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.487342834472656 | Batch Time=21.484375 +GPU:1 | Epoch: 25 | loss=9.510494232177734 | Batch Time=21.09375 +GPU:1 | Epoch: 25 | loss=9.674888610839844 | Batch Time=17.1875 +GPU:1 | Epoch: 25 | loss=9.570391654968262 | Batch Time=17.1875 +GPU:1 | Epoch: 25 | loss=9.746953964233398 | Batch Time=17.1875 +GPU:1 | Epoch: 25 | loss=9.438589096069336 | Batch Time=21.09375 +GPU:1 | Epoch: 25 | loss=9.599966049194336 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.582542419433594 | Batch Time=21.484375 +GPU:1 | Epoch: 25 | loss=9.706220626831055 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.360325813293457 | Batch Time=23.046875 +GPU:1 | Epoch: 25 | loss=9.689668655395508 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.61571979522705 | Batch Time=19.140625 +GPU:1 | Epoch: 25 | loss=9.529647827148438 | Batch Time=17.578125 +GPU:1 | Epoch: 25 | loss=9.605340003967285 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.389776229858398 | Batch Time=22.65625 +GPU:1 | Epoch: 25 | loss=9.445512771606445 | Batch Time=24.21875 +GPU:1 | Epoch: 25 | loss=9.501124382019043 | Batch Time=17.578125 +GPU:1 | Epoch: 25 | loss=9.636576652526855 | Batch Time=15.234375 +GPU:1 | Epoch: 25 | loss=9.460416793823242 | Batch Time=22.65625 +GPU:1 | Epoch: 25 | loss=9.660289764404297 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.658775329589844 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.600876808166504 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.437545776367188 | Batch Time=17.1875 +GPU:1 | Epoch: 25 | loss=9.753335952758789 | Batch Time=18.359375 +GPU:1 | Epoch: 25 | loss=9.539339065551758 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.437810897827148 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.533437728881836 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.741514205932617 | Batch Time=20.3125 +GPU:1 | Epoch: 25 | loss=9.447216987609863 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.458184242248535 | Batch Time=21.484375 +GPU:1 | Epoch: 25 | loss=9.786083221435547 | Batch Time=13.28125 +GPU:1 | Epoch: 25 | loss=9.509654998779297 | Batch Time=17.578125 +GPU:1 | Epoch: 25 | loss=9.477455139160156 | Batch Time=19.140625 +GPU:1 | Epoch: 25 | loss=9.638069152832031 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.792791366577148 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.728264808654785 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.410619735717773 | Batch Time=20.3125 +GPU:1 | Epoch: 25 | loss=9.454290390014648 | Batch Time=17.578125 +GPU:1 | Epoch: 25 | loss=9.639421463012695 | Batch Time=16.40625 +GPU:1 | Epoch: 25 | loss=9.690024375915527 | Batch Time=19.140625 +GPU:1 | Epoch: 25 | loss=9.441655158996582 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.298116683959961 | Batch Time=20.3125 +GPU:1 | Epoch: 25 | loss=9.453283309936523 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.481306076049805 | Batch Time=20.3125 +GPU:1 | Epoch: 25 | loss=9.574789047241211 | Batch Time=16.40625 +GPU:1 | Epoch: 25 | loss=9.651544570922852 | Batch Time=20.3125 +GPU:1 | Epoch: 25 | loss=9.653545379638672 | Batch Time=19.53125 +GPU:1 | Epoch: 25 | loss=9.701825141906738 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.498115539550781 | Batch Time=19.140625 +GPU:1 | Epoch: 25 | loss=9.631237030029297 | Batch Time=18.359375 +GPU:1 | Epoch: 25 | loss=9.515363693237305 | Batch Time=18.359375 +GPU:1 | Epoch: 25 | loss=9.572874069213867 | Batch Time=19.53125 +GPU:1 | Epoch: 25 | loss=9.468416213989258 | Batch Time=19.140625 +GPU:1 | Epoch: 25 | loss=9.524520874023438 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.485170364379883 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.495044708251953 | Batch Time=17.578125 +GPU:1 | Epoch: 25 | loss=9.450057029724121 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.464335441589355 | Batch Time=19.140625 +GPU:1 | Epoch: 25 | loss=9.547269821166992 | Batch Time=22.65625 +GPU:1 | Epoch: 25 | loss=9.356193542480469 | Batch Time=21.484375 +GPU:1 | Epoch: 25 | loss=9.634233474731445 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.442228317260742 | Batch Time=19.140625 +GPU:1 | Epoch: 25 | loss=9.514802932739258 | Batch Time=20.703125 +GPU:1 | Epoch: 25 | loss=9.545360565185547 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.784116744995117 | Batch Time=16.40625 +GPU:1 | Epoch: 25 | loss=9.842672348022461 | Batch Time=15.234375 +GPU:1 | Epoch: 25 | loss=9.354484558105469 | Batch Time=22.265625 +GPU:1 | Epoch: 25 | loss=9.652292251586914 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.54202651977539 | Batch Time=18.359375 +GPU:1 | Epoch: 25 | loss=9.468530654907227 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.593599319458008 | Batch Time=16.40625 +GPU:1 | Epoch: 25 | loss=9.610886573791504 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.459203720092773 | Batch Time=22.265625 +GPU:1 | Epoch: 25 | loss=9.644678115844727 | Batch Time=17.578125 +GPU:1 | Epoch: 25 | loss=9.593669891357422 | Batch Time=16.796875 +GPU:1 | Epoch: 25 | loss=9.736555099487305 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.453042984008789 | Batch Time=16.40625 +GPU:1 | Epoch: 25 | loss=9.492988586425781 | Batch Time=20.703125 +GPU:1 | Epoch: 25 | loss=9.548103332519531 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.404029846191406 | Batch Time=21.09375 +GPU:1 | Epoch: 25 | loss=9.279584884643555 | Batch Time=21.484375 +GPU:1 | Epoch: 25 | loss=9.354936599731445 | Batch Time=20.703125 +GPU:1 | Epoch: 25 | loss=9.335418701171875 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.436361312866211 | Batch Time=21.875 +GPU:1 | Epoch: 25 | loss=9.756427764892578 | Batch Time=16.015625 +GPU:1 | Epoch: 25 | loss=9.36170768737793 | Batch Time=20.703125 +GPU:1 | Epoch: 25 | loss=9.559069633483887 | Batch Time=21.875 +GPU:1 | Epoch: 25 | loss=9.361194610595703 | Batch Time=21.875 +GPU:1 | Epoch: 25 | loss=9.611614227294922 | Batch Time=16.015625 +GPU:1 | Epoch: 25 | loss=9.454696655273438 | Batch Time=23.4375 +GPU:1 | Epoch: 25 | loss=9.33713150024414 | Batch Time=24.609375 +GPU:1 | Epoch: 25 | loss=9.667865753173828 | Batch Time=16.40625 +GPU:1 | Epoch: 25 | loss=9.644645690917969 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.428949356079102 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.44547176361084 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.535858154296875 | Batch Time=14.453125 +GPU:1 | Epoch: 25 | loss=9.503124237060547 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.440008163452148 | Batch Time=17.96875 +GPU:1 | Epoch: 25 | loss=9.558158874511719 | Batch Time=16.40625 +GPU:1 | Epoch: 25 | loss=9.467277526855469 | Batch Time=19.53125 +GPU:1 | Epoch: 25 | loss=9.6038179397583 | Batch Time=16.015625 +GPU:1 | Epoch: 25 | loss=9.4052152633667 | Batch Time=26.5625 +GPU:1 | Epoch: 25 | loss=9.379343032836914 | Batch Time=19.53125 +GPU:1 | Epoch: 25 | loss=9.522608757019043 | Batch Time=18.359375 +GPU:1 | Epoch: 25 | loss=9.552517890930176 | Batch Time=18.75 +GPU:1 | Epoch: 25 | loss=9.639162063598633 | Batch Time=18.359375 +GPU:1 | Epoch: 25 | loss=9.425460815429688 | Batch Time=19.140625 +GPU:1 | Epoch: 25 | loss=9.278294563293457 | Batch Time=20.703125 +GPU:1 | Epoch: 25 | loss=9.288778305053711 | Batch Time=20.703125 +GPU:1 | Epoch: 25 | loss=9.403705596923828 | Batch Time=18.359375 +GPU:1 | Epoch: 25 | loss=9.351700782775879 | Batch Time=19.921875 +GPU:1 | Epoch: 25 | loss=9.361579895019531 | Batch Time=16.015625 +GPU:0 | Epoch: 25 | loss=9.768224716186523 | Batch Time=14.0625 +GPU:0 | Epoch: 25 | loss=9.713720321655273 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.763887405395508 | Batch Time=16.40625 +GPU:0 | Epoch: 25 | loss=9.68679428100586 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.382223129272461 | Batch Time=22.65625 +GPU:0 | Epoch: 25 | loss=9.618555068969727 | Batch Time=16.015625 +GPU:0 | Epoch: 25 | loss=9.524750709533691 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=9.56468677520752 | Batch Time=14.453125 +GPU:0 | Epoch: 25 | loss=9.645343780517578 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.45129108428955 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=9.606962203979492 | Batch Time=19.921875 +GPU:0 | Epoch: 25 | loss=9.319576263427734 | Batch Time=25.0 +GPU:0 | Epoch: 25 | loss=9.589155197143555 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.786799430847168 | Batch Time=17.1875 +GPU:0 | Epoch: 25 | loss=9.490358352661133 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=9.759808540344238 | Batch Time=13.671875 +GPU:0 | Epoch: 25 | loss=9.562942504882812 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.790225982666016 | Batch Time=18.359375 +GPU:0 | Epoch: 25 | loss=9.619429588317871 | Batch Time=14.0625 +GPU:0 | Epoch: 25 | loss=9.557548522949219 | Batch Time=21.875 +GPU:0 | Epoch: 25 | loss=9.41290283203125 | Batch Time=21.484375 +GPU:0 | Epoch: 25 | loss=9.555962562561035 | Batch Time=17.96875 +GPU:0 | Epoch: 25 | loss=9.53446102142334 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.763216018676758 | Batch Time=16.40625 +GPU:0 | Epoch: 25 | loss=9.533685684204102 | Batch Time=17.1875 +GPU:0 | Epoch: 25 | loss=9.765695571899414 | Batch Time=15.625 +GPU:0 | Epoch: 25 | loss=9.782489776611328 | Batch Time=18.359375 +GPU:0 | Epoch: 25 | loss=9.645174026489258 | Batch Time=14.84375 +GPU:0 | Epoch: 25 | loss=9.613429069519043 | Batch Time=21.484375 +GPU:0 | Epoch: 25 | loss=9.400123596191406 | Batch Time=20.3125 +GPU:0 | Epoch: 25 | loss=9.456979751586914 | Batch Time=23.828125 +GPU:0 | Epoch: 25 | loss=9.48105239868164 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.797853469848633 | Batch Time=16.40625 +GPU:0 | Epoch: 25 | loss=9.719877243041992 | Batch Time=16.796875 +GPU:0 | Epoch: 25 | loss=9.40415096282959 | Batch Time=18.359375 +GPU:0 | Epoch: 25 | loss=9.485897064208984 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.491865158081055 | Batch Time=21.484375 +GPU:0 | Epoch: 25 | loss=9.542048454284668 | Batch Time=16.796875 +GPU:0 | Epoch: 25 | loss=9.584436416625977 | Batch Time=16.40625 +GPU:0 | Epoch: 25 | loss=9.52275276184082 | Batch Time=18.359375 +GPU:0 | Epoch: 25 | loss=9.6004056930542 | Batch Time=20.3125 +GPU:0 | Epoch: 25 | loss=9.57898998260498 | Batch Time=20.3125 +GPU:0 | Epoch: 25 | loss=9.74908447265625 | Batch Time=14.0625 +GPU:0 | Epoch: 25 | loss=9.485270500183105 | Batch Time=19.921875 +GPU:0 | Epoch: 25 | loss=9.77920150756836 | Batch Time=15.625 +GPU:0 | Epoch: 25 | loss=9.5279541015625 | Batch Time=21.875 +GPU:0 | Epoch: 25 | loss=9.543109893798828 | Batch Time=15.625 +GPU:0 | Epoch: 25 | loss=9.473825454711914 | Batch Time=18.75 +GPU:0 | Epoch: 25 | loss=9.763894081115723 | Batch Time=13.671875 +GPU:0 | Epoch: 25 | loss=9.622221946716309 | Batch Time=17.96875 +GPU:0 | Epoch: 25 | loss=9.441585540771484 | Batch Time=24.21875 +GPU:0 | Epoch: 25 | loss=9.701889038085938 | Batch Time=17.1875 +GPU:0 | Epoch: 25 | loss=9.537342071533203 | Batch Time=21.09375 +GPU:0 | Epoch: 25 | loss=9.59769058227539 | Batch Time=18.75 +GPU:0 | Epoch: 25 | loss=9.48151969909668 | Batch Time=22.265625 +GPU:0 | Epoch: 25 | loss=9.533346176147461 | Batch Time=21.484375 +GPU:0 | Epoch: 25 | loss=9.84085464477539 | Batch Time=16.40625 +GPU:0 | Epoch: 25 | loss=9.568866729736328 | Batch Time=14.84375 +GPU:0 | Epoch: 25 | loss=9.567818641662598 | Batch Time=19.921875 +GPU:0 | Epoch: 25 | loss=9.474215507507324 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.560009002685547 | Batch Time=17.96875 +GPU:0 | Epoch: 25 | loss=9.6240816116333 | Batch Time=16.796875 +GPU:0 | Epoch: 25 | loss=9.78740119934082 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=9.670251846313477 | Batch Time=14.84375 +GPU:0 | Epoch: 25 | loss=9.583393096923828 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=9.669824600219727 | Batch Time=12.890625 +GPU:0 | Epoch: 25 | loss=9.488563537597656 | Batch Time=21.09375 +GPU:0 | Epoch: 25 | loss=9.518282890319824 | Batch Time=13.28125 +GPU:0 | Epoch: 25 | loss=9.464043617248535 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.584949493408203 | Batch Time=16.015625 +GPU:0 | Epoch: 25 | loss=9.498757362365723 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.650251388549805 | Batch Time=17.96875 +GPU:0 | Epoch: 25 | loss=9.548591613769531 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.717681884765625 | Batch Time=14.84375 +GPU:0 | Epoch: 25 | loss=9.518291473388672 | Batch Time=23.4375 +GPU:0 | Epoch: 25 | loss=9.604009628295898 | Batch Time=13.671875 +GPU:0 | Epoch: 25 | loss=9.567334175109863 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.529220581054688 | Batch Time=17.1875 +GPU:0 | Epoch: 25 | loss=9.684171676635742 | Batch Time=16.796875 +GPU:0 | Epoch: 25 | loss=9.433414459228516 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.450525283813477 | Batch Time=22.65625 +GPU:0 | Epoch: 25 | loss=9.533206939697266 | Batch Time=16.015625 +GPU:0 | Epoch: 25 | loss=9.429128646850586 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.832466125488281 | Batch Time=15.625 +GPU:0 | Epoch: 25 | loss=9.695907592773438 | Batch Time=16.796875 +GPU:0 | Epoch: 25 | loss=9.494781494140625 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.565620422363281 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.40489387512207 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=9.71200180053711 | Batch Time=15.625 +GPU:0 | Epoch: 25 | loss=9.443145751953125 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.233596801757812 | Batch Time=25.0 +GPU:0 | Epoch: 25 | loss=9.52352523803711 | Batch Time=18.75 +GPU:0 | Epoch: 25 | loss=9.563565254211426 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.496395111083984 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=9.506924629211426 | Batch Time=17.1875 +GPU:0 | Epoch: 25 | loss=9.281048774719238 | Batch Time=20.3125 +GPU:0 | Epoch: 25 | loss=9.654987335205078 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.642753601074219 | Batch Time=15.625 +GPU:0 | Epoch: 25 | loss=9.469245910644531 | Batch Time=18.359375 +GPU:0 | Epoch: 25 | loss=9.641767501831055 | Batch Time=15.234375 +GPU:0 | Epoch: 25 | loss=9.380680084228516 | Batch Time=21.09375 +GPU:0 | Epoch: 25 | loss=9.469552993774414 | Batch Time=18.359375 +GPU:0 | Epoch: 25 | loss=9.487140655517578 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.397405624389648 | Batch Time=21.875 +GPU:0 | Epoch: 25 | loss=9.662731170654297 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.622424125671387 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.483007431030273 | Batch Time=15.625 +GPU:0 | Epoch: 25 | loss=9.51923656463623 | Batch Time=16.796875 +GPU:0 | Epoch: 25 | loss=9.652618408203125 | Batch Time=17.96875 +GPU:0 | Epoch: 25 | loss=9.857051849365234 | Batch Time=14.84375 +GPU:0 | Epoch: 25 | loss=9.469318389892578 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.404409408569336 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.3131685256958 | Batch Time=18.75 +GPU:0 | Epoch: 25 | loss=9.54263687133789 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=9.558393478393555 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.382774353027344 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.3140869140625 | Batch Time=18.359375 +GPU:0 | Epoch: 25 | loss=9.709619522094727 | Batch Time=14.0625 +GPU:0 | Epoch: 25 | loss=9.505094528198242 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.425939559936523 | Batch Time=19.53125 +GPU:0 | Epoch: 25 | loss=9.537442207336426 | Batch Time=18.75 +GPU:0 | Epoch: 25 | loss=9.562692642211914 | Batch Time=18.75 +GPU:0 | Epoch: 25 | loss=9.638635635375977 | Batch Time=19.140625 +GPU:0 | Epoch: 25 | loss=9.40615463256836 | Batch Time=23.046875 +GPU:0 | Epoch: 25 | loss=9.605670928955078 | Batch Time=21.09375 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160184832, free=16912035840, active=208844206080, inactive=27606253568, buffers=390750208, cached=33105113088, shared=5662564352, slab=1482473472) +AFTER TRAIN LOOP: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160229888, free=16912035840, active=208844124160, inactive=27606220800, buffers=390750208, cached=33105068032, shared=5662482432, slab=1482473472) +BEFORE VAL LOOP: GPU: 0 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160229888, free=16912035840, active=208844124160, inactive=27606220800, buffers=390750208, cached=33105068032, shared=5662482432, slab=1482473472) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160184832, free=16912035840, active=208844206080, inactive=27606253568, buffers=390750208, cached=33105113088, shared=5662564352, slab=1482473472) +AFTER TRAIN LOOP: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160229888, free=16912035840, active=208844124160, inactive=27606220800, buffers=390750208, cached=33105068032, shared=5662482432, slab=1482473472) +BEFORE PRUNE: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160229888, free=16912035840, active=208844124160, inactive=27606220800, buffers=390750208, cached=33105068032, shared=5662482432, slab=1482473472) +Pruning Model: +AFTER PRUNE: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +STARTING TRAINING: Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +BEFORE TRAIN LOOP: Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160184832, free=16912035840, active=208844206080, inactive=27606253568, buffers=390750208, cached=33105113088, shared=5662564352, slab=1482473472) +AFTER TRAIN LOOP: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160229888, free=16912035840, active=208844124160, inactive=27606220800, buffers=390750208, cached=33105068032, shared=5662482432, slab=1482473472) +BEFORE PRUNE: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160229888, free=16912035840, active=208844124160, inactive=27606220800, buffers=390750208, cached=33105068032, shared=5662482432, slab=1482473472) +Pruning Model: +AFTER PRUNE: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +STARTING TRAINING: Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +BEFORE TRAIN LOOP: Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160184832, free=16912035840, active=208844206080, inactive=27606253568, buffers=390750208, cached=33105113088, shared=5662564352, slab=1482473472) +AFTER TRAIN LOOP: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160229888, free=16912035840, active=208844124160, inactive=27606220800, buffers=390750208, cached=33105068032, shared=5662482432, slab=1482473472) +BEFORE PRUNE: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42493845504, percent=83.5, used=207160229888, free=16912035840, active=208844124160, inactive=27606220800, buffers=390750208, cached=33105068032, shared=5662482432, slab=1482473472) +Pruning Model: +AFTER PRUNE: Epoch 25 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +STARTING TRAINING: Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +BEFORE TRAIN LOOP: Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=42492919808, percent=83.5, used=207160664064, free=16905129984, active=208843587584, inactive=27612463104, buffers=390750208, cached=33111539712, shared=5662400512, slab=1482170368) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 25 | loss=3.2577860355377197 | Batch Time=35.546875 +GPU:0 | Epoch: 25 | loss=4.571620464324951 | Batch Time=12.5 +GPU:0 | Epoch: 25 | loss=3.896333694458008 | Batch Time=28.515625 +GPU:0 | Epoch: 25 | loss=3.4386847019195557 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=4.722518444061279 | Batch Time=12.109375 +GPU:0 | Epoch: 25 | loss=3.867584705352783 | Batch Time=13.28125 +GPU:0 | Epoch: 25 | loss=4.370175838470459 | Batch Time=12.890625 +GPU:0 | Epoch: 25 | loss=4.673081398010254 | Batch Time=8.984375 +GPU:0 | Epoch: 25 | loss=5.1088738441467285 | Batch Time=8.203125 +GPU:0 | Epoch: 25 | loss=5.725987434387207 | Batch Time=3.515625 +GPU:0 | Epoch: 25 | loss=5.202845573425293 | Batch Time=5.078125 +GPU:0 | Epoch: 25 | loss=4.221409797668457 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=4.911551475524902 | Batch Time=14.84375 +GPU:0 | Epoch: 25 | loss=4.134039402008057 | Batch Time=23.046875 +GPU:0 | Epoch: 25 | loss=4.364226341247559 | Batch Time=13.671875 +GPU:0 | Epoch: 25 | loss=4.7992658615112305 | Batch Time=14.0625 +GPU:0 | Epoch: 25 | loss=4.401010036468506 | Batch Time=10.9375 +GPU:0 | Epoch: 25 | loss=3.6822729110717773 | Batch Time=21.484375 +GPU:0 | Epoch: 25 | loss=3.3046531677246094 | Batch Time=22.65625 +GPU:0 | Epoch: 25 | loss=3.5593814849853516 | Batch Time=26.5625 +Model top1 Accuracy: 18.198 +Acc before rounding: 18.198 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 25 | loss=3.2577860355377197 | Batch Time=35.546875 +GPU:0 | Epoch: 25 | loss=4.571620464324951 | Batch Time=12.5 +GPU:0 | Epoch: 25 | loss=3.896333694458008 | Batch Time=28.515625 +GPU:0 | Epoch: 25 | loss=3.4386847019195557 | Batch Time=17.578125 +GPU:0 | Epoch: 25 | loss=4.722518444061279 | Batch Time=12.109375 +GPU:0 | Epoch: 25 | loss=3.867584705352783 | Batch Time=13.28125 +GPU:0 | Epoch: 25 | loss=4.370175838470459 | Batch Time=12.890625 +GPU:0 | Epoch: 25 | loss=4.673081398010254 | Batch Time=8.984375 +GPU:0 | Epoch: 25 | loss=5.1088738441467285 | Batch Time=8.203125 +GPU:0 | Epoch: 25 | loss=5.725987434387207 | Batch Time=3.515625 +GPU:0 | Epoch: 25 | loss=5.202845573425293 | Batch Time=5.078125 +GPU:0 | Epoch: 25 | loss=4.221409797668457 | Batch Time=20.703125 +GPU:0 | Epoch: 25 | loss=4.911551475524902 | Batch Time=14.84375 +GPU:0 | Epoch: 25 | loss=4.134039402008057 | Batch Time=23.046875 +GPU:0 | Epoch: 25 | loss=4.364226341247559 | Batch Time=13.671875 +GPU:0 | Epoch: 25 | loss=4.7992658615112305 | Batch Time=14.0625 +GPU:0 | Epoch: 25 | loss=4.401010036468506 | Batch Time=10.9375 +GPU:0 | Epoch: 25 | loss=3.6822729110717773 | Batch Time=21.484375 +GPU:0 | Epoch: 25 | loss=3.3046531677246094 | Batch Time=22.65625 +GPU:0 | Epoch: 25 | loss=3.5593814849853516 | Batch Time=26.5625 +Model top1 Accuracy: 18.198 +Acc after rounding: 18.198 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 25 | loss=4.495368003845215 | Batch Time=14.0625 +GPU:0 | Epoch: 25 | loss=4.478463649749756 | Batch Time=13.671875 +GPU:0 | Epoch: 25 | loss=4.355901718139648 | Batch Time=18.359375 +GPU:0 | Epoch: 25 | loss=4.615606307983398 | Batch Time=13.28125 +Model top1 Accuracy: 15.23 +Validation Acc after rounding: 15.23 +AFTER VAL LOOP: GPU: 0 | Epoch 25 | Memory Usage: svmem(total=257568083968, available=24683495424, percent=90.4, used=224970244096, free=5166149632, active=230921363456, inactive=17169965056, buffers=419880960, cached=27011809280, shared=5662711808, slab=1467572224) +BEFORE PRUNE: Epoch 25 | Memory Usage: svmem(total=257568083968, available=24683495424, percent=90.4, used=224970244096, free=5166149632, active=230921363456, inactive=17169965056, buffers=419880960, cached=27011809280, shared=5662711808, slab=1467572224) +Pruning Model: +AFTER PRUNE: Epoch 25 | Memory Usage: svmem(total=257568083968, available=24683753472, percent=90.4, used=224969986048, free=5166407680, active=230921334784, inactive=17169965056, buffers=419880960, cached=27011809280, shared=5662711808, slab=1467572224) +Rounding model with scheme: naive +Model avg sparsity: 36.13962003017856 +GPU:0 | Epoch: 25 | Acc=18.198 | Epoch Time=22.425518107414245 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 25 | Memory Usage: svmem(total=257568083968, available=24683868160, percent=90.4, used=224969949184, free=5165109248, active=230921297920, inactive=17171324928, buffers=419897344, cached=27013128192, shared=5662629888, slab=1467580416) +STARTING TRAINING: Epoch 26 | Memory Usage: svmem(total=257568083968, available=24683868160, percent=90.4, used=224969949184, free=5165109248, active=230921297920, inactive=17171324928, buffers=419897344, cached=27013128192, shared=5662629888, slab=1467580416) +BEFORE TRAIN LOOP: Epoch 26 | Memory Usage: svmem(total=257568083968, available=24683868160, percent=90.4, used=224969949184, free=5165109248, active=230921297920, inactive=17171324928, buffers=419897344, cached=27013128192, shared=5662629888, slab=1467580416) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=24683868160, percent=90.4, used=224969949184, free=5165109248, active=230921297920, inactive=17171324928, buffers=419897344, cached=27013128192, shared=5662629888, slab=1467580416) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 26 | loss=9.41850471496582 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.546377182006836 | Batch Time=17.578125 +GPU:0 | Epoch: 26 | loss=9.578221321105957 | Batch Time=19.140625 +GPU:0 | Epoch: 26 | loss=9.518295288085938 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.428890228271484 | Batch Time=22.65625 +GPU:0 | Epoch: 26 | loss=9.434326171875 | Batch Time=20.703125 +GPU:0 | Epoch: 26 | loss=9.455881118774414 | Batch Time=19.140625 +GPU:0 | Epoch: 26 | loss=9.514976501464844 | Batch Time=17.1875 +GPU:0 | Epoch: 26 | loss=9.360969543457031 | Batch Time=20.703125 +GPU:0 | Epoch: 26 | loss=9.571854591369629 | Batch Time=17.578125 +GPU:0 | Epoch: 26 | loss=9.626309394836426 | Batch Time=18.359375 +GPU:0 | Epoch: 26 | loss=9.332982063293457 | Batch Time=24.609375 +GPU:0 | Epoch: 26 | loss=9.268867492675781 | Batch Time=19.140625 +GPU:0 | Epoch: 26 | loss=9.418108940124512 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.408376693725586 | Batch Time=19.53125 +GPU:0 | Epoch: 26 | loss=9.518144607543945 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.659128189086914 | Batch Time=15.625 +GPU:0 | Epoch: 26 | loss=9.273317337036133 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.39381217956543 | Batch Time=17.578125 +GPU:0 | Epoch: 26 | loss=9.373466491699219 | Batch Time=16.40625 +GPU:0 | Epoch: 26 | loss=9.382416725158691 | Batch Time=19.140625 +GPU:0 | Epoch: 26 | loss=9.480567932128906 | Batch Time=15.625 +GPU:0 | Epoch: 26 | loss=9.34524154663086 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.423080444335938 | Batch Time=18.75 +GPU:0 | Epoch: 26 | loss=9.340713500976562 | Batch Time=22.265625 +GPU:0 | Epoch: 26 | loss=9.422836303710938 | Batch Time=19.53125 +GPU:0 | Epoch: 26 | loss=9.290555953979492 | Batch Time=26.953125 +GPU:0 | Epoch: 26 | loss=9.440154075622559 | Batch Time=16.015625 +GPU:0 | Epoch: 26 | loss=9.69085693359375 | Batch Time=16.015625 +GPU:0 | Epoch: 26 | loss=9.401018142700195 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.413402557373047 | Batch Time=21.09375 +GPU:0 | Epoch: 26 | loss=9.371086120605469 | Batch Time=25.0 +GPU:0 | Epoch: 26 | loss=9.255857467651367 | Batch Time=21.875 +GPU:0 | Epoch: 26 | loss=9.311126708984375 | Batch Time=25.78125 +GPU:0 | Epoch: 26 | loss=9.676214218139648 | Batch Time=18.359375 +GPU:0 | Epoch: 26 | loss=9.40172004699707 | Batch Time=17.96875 +GPU:0 | Epoch: 26 | loss=9.24412727355957 | Batch Time=22.65625 +GPU:0 | Epoch: 26 | loss=9.3434476852417 | Batch Time=27.34375 +GPU:0 | Epoch: 26 | loss=9.360695838928223 | Batch Time=19.53125 +GPU:0 | Epoch: 26 | loss=9.212242126464844 | Batch Time=18.359375 +GPU:0 | Epoch: 26 | loss=9.425910949707031 | Batch Time=21.484375 +GPU:0 | Epoch: 26 | loss=9.328211784362793 | Batch Time=19.140625 +GPU:0 | Epoch: 26 | loss=9.346900939941406 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.33491325378418 | Batch Time=18.359375 +GPU:0 | Epoch: 26 | loss=9.651079177856445 | Batch Time=18.75 +GPU:0 | Epoch: 26 | loss=9.480020523071289 | Batch Time=22.65625 +GPU:0 | Epoch: 26 | loss=9.59885311126709 | Batch Time=18.75 +GPU:0 | Epoch: 26 | loss=9.422626495361328 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.453497886657715 | Batch Time=17.578125 +GPU:0 | Epoch: 26 | loss=9.404611587524414 | Batch Time=22.65625 +GPU:0 | Epoch: 26 | loss=9.410137176513672 | Batch Time=17.1875 +GPU:0 | Epoch: 26 | loss=9.413755416870117 | Batch Time=21.875 +GPU:0 | Epoch: 26 | loss=9.537803649902344 | Batch Time=21.09375 +GPU:0 | Epoch: 26 | loss=9.55593204498291 | Batch Time=18.359375 +GPU:0 | Epoch: 26 | loss=9.45615005493164 | Batch Time=18.359375 +GPU:0 | Epoch: 26 | loss=9.370731353759766 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.161377906799316 | Batch Time=21.484375 +GPU:0 | Epoch: 26 | loss=9.386103630065918 | Batch Time=16.40625 +GPU:0 | Epoch: 26 | loss=9.237768173217773 | Batch Time=21.09375 +GPU:0 | Epoch: 26 | loss=9.406625747680664 | Batch Time=21.875 +GPU:0 | Epoch: 26 | loss=9.431044578552246 | Batch Time=13.671875 +GPU:0 | Epoch: 26 | loss=9.276288986206055 | Batch Time=17.578125 +GPU:0 | Epoch: 26 | loss=9.556001663208008 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.54378890991211 | Batch Time=13.671875 +GPU:0 | Epoch: 26 | loss=9.42038345336914 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.463714599609375 | Batch Time=24.21875 +GPU:0 | Epoch: 26 | loss=9.33827018737793 | Batch Time=20.703125 +GPU:0 | Epoch: 26 | loss=9.525566101074219 | Batch Time=18.75 +GPU:0 | Epoch: 26 | loss=9.21905517578125 | Batch Time=22.265625 +GPU:0 | Epoch: 26 | loss=9.313316345214844 | Batch Time=17.578125 +GPU:0 | Epoch: 26 | loss=9.042830467224121 | Batch Time=24.21875 +GPU:0 | Epoch: 26 | loss=9.401594161987305 | Batch Time=21.875 +GPU:0 | Epoch: 26 | loss=9.489681243896484 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.652557373046875 | Batch Time=17.1875 +GPU:0 | Epoch: 26 | loss=9.341208457946777 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.415277481079102 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.255114555358887 | Batch Time=21.875 +GPU:0 | Epoch: 26 | loss=9.383280754089355 | Batch Time=21.09375 +GPU:0 | Epoch: 26 | loss=9.482243537902832 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.383191108703613 | Batch Time=22.65625 +GPU:0 | Epoch: 26 | loss=9.562599182128906 | Batch Time=16.015625 +GPU:0 | Epoch: 26 | loss=9.063423156738281 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.272153854370117 | Batch Time=22.265625 +GPU:0 | Epoch: 26 | loss=9.31907844543457 | Batch Time=17.578125 +GPU:0 | Epoch: 26 | loss=9.35230827331543 | Batch Time=19.53125 +GPU:0 | Epoch: 26 | loss=9.464763641357422 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.64596939086914 | Batch Time=14.84375 +GPU:0 | Epoch: 26 | loss=9.556649208068848 | Batch Time=18.359375 +GPU:0 | Epoch: 26 | loss=9.599028587341309 | Batch Time=16.796875 +GPU:0 | Epoch: 26 | loss=9.419004440307617 | Batch Time=15.625 +GPU:0 | Epoch: 26 | loss=9.255346298217773 | Batch Time=21.484375 +GPU:0 | Epoch: 26 | loss=9.42356014251709 | Batch Time=16.40625 +GPU:0 | Epoch: 26 | loss=9.38099479675293 | Batch Time=18.75 +GPU:0 | Epoch: 26 | loss=9.38473129272461 | Batch Time=15.625 +GPU:0 | Epoch: 26 | loss=9.294663429260254 | Batch Time=23.828125 +GPU:0 | Epoch: 26 | loss=9.143317222595215 | Batch Time=21.484375 +GPU:0 | Epoch: 26 | loss=9.375826835632324 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.145587921142578 | Batch Time=20.703125 +GPU:0 | Epoch: 26 | loss=9.314957618713379 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.287739753723145 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.24754524230957 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.514020919799805 | Batch Time=16.796875 +GPU:0 | Epoch: 26 | loss=9.631187438964844 | Batch Time=12.5 +GPU:0 | Epoch: 26 | loss=9.26498031616211 | Batch Time=19.53125 +GPU:0 | Epoch: 26 | loss=9.464302062988281 | Batch Time=16.796875 +GPU:0 | Epoch: 26 | loss=9.196573257446289 | Batch Time=22.265625 +GPU:0 | Epoch: 26 | loss=9.360448837280273 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.30040168762207 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.321492195129395 | Batch Time=23.828125 +GPU:0 | Epoch: 26 | loss=9.273200988769531 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.1300048828125 | Batch Time=19.53125 +GPU:0 | Epoch: 26 | loss=9.324690818786621 | Batch Time=22.65625 +GPU:0 | Epoch: 26 | loss=9.408809661865234 | Batch Time=19.140625 +GPU:0 | Epoch: 26 | loss=9.22914981842041 | Batch Time=24.609375 +GPU:0 | Epoch: 26 | loss=9.291984558105469 | Batch Time=19.921875 +GPU:0 | Epoch: 26 | loss=9.448400497436523 | Batch Time=20.703125 +GPU:0 | Epoch: 26 | loss=9.233906745910645 | Batch Time=21.484375 +GPU:0 | Epoch: 26 | loss=9.358875274658203 | Batch Time=18.75 +GPU:0 | Epoch: 26 | loss=9.473470687866211 | Batch Time=19.140625 +GPU:0 | Epoch: 26 | loss=9.40189266204834 | Batch Time=16.796875 +GPU:0 | Epoch: 26 | loss=9.792716026306152 | Batch Time=17.578125 +GPU:0 | Epoch: 26 | loss=9.366436004638672 | Batch Time=21.09375 +GPU:0 | Epoch: 26 | loss=9.336580276489258 | Batch Time=20.3125 +GPU:0 | Epoch: 26 | loss=9.209587097167969 | Batch Time=21.484375 +GPU:0 | Epoch: 26 | loss=9.410911560058594 | Batch Time=19.140625 +GPU:3 | Epoch: 26 | loss=9.528249740600586 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.467273712158203 | Batch Time=22.65625 +GPU:3 | Epoch: 26 | loss=9.335871696472168 | Batch Time=18.75 +GPU:3 | Epoch: 26 | loss=9.343669891357422 | Batch Time=23.046875 +GPU:3 | Epoch: 26 | loss=9.625617980957031 | Batch Time=15.625 +GPU:3 | Epoch: 26 | loss=9.670880317687988 | Batch Time=16.40625 +GPU:3 | Epoch: 26 | loss=9.467693328857422 | Batch Time=22.65625 +GPU:3 | Epoch: 26 | loss=9.24847412109375 | Batch Time=21.484375 +GPU:3 | Epoch: 26 | loss=9.467004776000977 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.366622924804688 | Batch Time=16.015625 +GPU:3 | Epoch: 26 | loss=9.308149337768555 | Batch Time=23.4375 +GPU:3 | Epoch: 26 | loss=9.521251678466797 | Batch Time=18.75 +GPU:3 | Epoch: 26 | loss=9.494453430175781 | Batch Time=20.3125 +GPU:3 | Epoch: 26 | loss=9.310287475585938 | Batch Time=17.96875 +GPU:3 | Epoch: 26 | loss=9.398069381713867 | Batch Time=20.703125 +GPU:3 | Epoch: 26 | loss=9.337356567382812 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.521742820739746 | Batch Time=16.40625 +GPU:3 | Epoch: 26 | loss=9.390453338623047 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.390308380126953 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.356090545654297 | Batch Time=23.4375 +GPU:3 | Epoch: 26 | loss=9.223278045654297 | Batch Time=22.65625 +GPU:3 | Epoch: 26 | loss=9.341156005859375 | Batch Time=19.140625 +GPU:3 | Epoch: 26 | loss=9.316864013671875 | Batch Time=24.21875 +GPU:3 | Epoch: 26 | loss=9.282061576843262 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.563470840454102 | Batch Time=18.359375 +GPU:3 | Epoch: 26 | loss=9.215055465698242 | Batch Time=18.359375 +GPU:3 | Epoch: 26 | loss=9.486242294311523 | Batch Time=17.1875 +GPU:3 | Epoch: 26 | loss=9.311739921569824 | Batch Time=18.359375 +GPU:3 | Epoch: 26 | loss=9.584760665893555 | Batch Time=16.40625 +GPU:3 | Epoch: 26 | loss=9.385335922241211 | Batch Time=19.53125 +GPU:3 | Epoch: 26 | loss=9.422749519348145 | Batch Time=19.53125 +GPU:3 | Epoch: 26 | loss=9.60129165649414 | Batch Time=17.96875 +GPU:3 | Epoch: 26 | loss=9.431939125061035 | Batch Time=20.703125 +GPU:3 | Epoch: 26 | loss=9.460320472717285 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.375822067260742 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.386964797973633 | Batch Time=19.140625 +GPU:3 | Epoch: 26 | loss=9.236507415771484 | Batch Time=22.65625 +GPU:3 | Epoch: 26 | loss=9.498764038085938 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.266227722167969 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.792740821838379 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.630176544189453 | Batch Time=16.015625 +GPU:3 | Epoch: 26 | loss=9.434219360351562 | Batch Time=20.3125 +GPU:3 | Epoch: 26 | loss=9.38821792602539 | Batch Time=22.265625 +GPU:3 | Epoch: 26 | loss=9.509453773498535 | Batch Time=16.796875 +GPU:3 | Epoch: 26 | loss=9.564714431762695 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.266948699951172 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.388472557067871 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.382159233093262 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.457221031188965 | Batch Time=23.046875 +GPU:3 | Epoch: 26 | loss=9.419097900390625 | Batch Time=21.484375 +GPU:3 | Epoch: 26 | loss=9.41352653503418 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.517644882202148 | Batch Time=16.40625 +GPU:3 | Epoch: 26 | loss=9.308584213256836 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.262250900268555 | Batch Time=22.65625 +GPU:3 | Epoch: 26 | loss=9.48901653289795 | Batch Time=19.140625 +GPU:3 | Epoch: 26 | loss=9.386573791503906 | Batch Time=17.1875 +GPU:3 | Epoch: 26 | loss=9.169015884399414 | Batch Time=23.4375 +GPU:3 | Epoch: 26 | loss=9.520694732666016 | Batch Time=13.28125 +GPU:3 | Epoch: 26 | loss=9.317285537719727 | Batch Time=22.65625 +GPU:3 | Epoch: 26 | loss=9.326213836669922 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.358518600463867 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.20545768737793 | Batch Time=19.53125 +GPU:3 | Epoch: 26 | loss=9.33092212677002 | Batch Time=21.484375 +GPU:3 | Epoch: 26 | loss=9.240274429321289 | Batch Time=22.265625 +GPU:3 | Epoch: 26 | loss=9.25516128540039 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.530967712402344 | Batch Time=17.578125 +GPU:3 | Epoch: 26 | loss=9.434014320373535 | Batch Time=20.703125 +GPU:3 | Epoch: 26 | loss=9.3098783493042 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.379082679748535 | Batch Time=24.609375 +GPU:3 | Epoch: 26 | loss=9.654031753540039 | Batch Time=16.40625 +GPU:3 | Epoch: 26 | loss=9.426216125488281 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.405654907226562 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.434431076049805 | Batch Time=18.75 +GPU:3 | Epoch: 26 | loss=9.265783309936523 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.275055885314941 | Batch Time=20.703125 +GPU:3 | Epoch: 26 | loss=9.50506591796875 | Batch Time=18.359375 +GPU:3 | Epoch: 26 | loss=9.194011688232422 | Batch Time=22.265625 +GPU:3 | Epoch: 26 | loss=9.445108413696289 | Batch Time=21.484375 +GPU:3 | Epoch: 26 | loss=9.375864028930664 | Batch Time=21.484375 +GPU:3 | Epoch: 26 | loss=9.220941543579102 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.60810661315918 | Batch Time=18.75 +GPU:3 | Epoch: 26 | loss=9.429437637329102 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.366052627563477 | Batch Time=18.359375 +GPU:3 | Epoch: 26 | loss=9.386442184448242 | Batch Time=19.53125 +GPU:3 | Epoch: 26 | loss=9.192338943481445 | Batch Time=23.046875 +GPU:3 | Epoch: 26 | loss=9.581119537353516 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.525680541992188 | Batch Time=15.234375 +GPU:3 | Epoch: 26 | loss=9.225456237792969 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.239700317382812 | Batch Time=23.4375 +GPU:3 | Epoch: 26 | loss=9.447982788085938 | Batch Time=18.75 +GPU:3 | Epoch: 26 | loss=9.331049919128418 | Batch Time=21.875 +GPU:3 | Epoch: 26 | loss=9.328084945678711 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.232064247131348 | Batch Time=16.796875 +GPU:3 | Epoch: 26 | loss=9.23565673828125 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.28692626953125 | Batch Time=20.3125 +GPU:3 | Epoch: 26 | loss=9.552364349365234 | Batch Time=18.75 +GPU:3 | Epoch: 26 | loss=9.386384963989258 | Batch Time=17.96875 +GPU:3 | Epoch: 26 | loss=9.46650505065918 | Batch Time=17.1875 +GPU:3 | Epoch: 26 | loss=9.410440444946289 | Batch Time=17.1875 +GPU:3 | Epoch: 26 | loss=9.357386589050293 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.258333206176758 | Batch Time=23.828125 +GPU:3 | Epoch: 26 | loss=9.452503204345703 | Batch Time=18.359375 +GPU:3 | Epoch: 26 | loss=9.549077033996582 | Batch Time=16.40625 +GPU:3 | Epoch: 26 | loss=9.582332611083984 | Batch Time=16.40625 +GPU:3 | Epoch: 26 | loss=9.599105834960938 | Batch Time=16.015625 +GPU:3 | Epoch: 26 | loss=9.200826644897461 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.467554092407227 | Batch Time=19.53125 +GPU:3 | Epoch: 26 | loss=9.481389999389648 | Batch Time=17.578125 +GPU:3 | Epoch: 26 | loss=9.279458045959473 | Batch Time=20.3125 +GPU:3 | Epoch: 26 | loss=9.360061645507812 | Batch Time=16.796875 +GPU:3 | Epoch: 26 | loss=9.420671463012695 | Batch Time=18.75 +GPU:3 | Epoch: 26 | loss=9.363897323608398 | Batch Time=19.140625 +GPU:3 | Epoch: 26 | loss=9.210712432861328 | Batch Time=23.046875 +GPU:3 | Epoch: 26 | loss=9.320671081542969 | Batch Time=20.703125 +GPU:3 | Epoch: 26 | loss=9.51191520690918 | Batch Time=19.53125 +GPU:3 | Epoch: 26 | loss=9.164392471313477 | Batch Time=24.609375 +GPU:3 | Epoch: 26 | loss=9.37539291381836 | Batch Time=17.96875 +GPU:3 | Epoch: 26 | loss=9.224387168884277 | Batch Time=20.3125 +GPU:3 | Epoch: 26 | loss=9.324527740478516 | Batch Time=19.53125 +GPU:3 | Epoch: 26 | loss=9.311500549316406 | Batch Time=19.921875 +GPU:3 | Epoch: 26 | loss=9.295186996459961 | Batch Time=21.09375 +GPU:3 | Epoch: 26 | loss=9.356823921203613 | Batch Time=19.53125 +GPU:3 | Epoch: 26 | loss=9.201155662536621 | Batch Time=23.4375 +GPU:3 | Epoch: 26 | loss=9.215476989746094 | Batch Time=19.140625 +GPU:3 | Epoch: 26 | loss=9.343074798583984 | Batch Time=22.265625 +GPU:2 | Epoch: 26 | loss=9.525056838989258 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.525875091552734 | Batch Time=17.1875 +GPU:2 | Epoch: 26 | loss=9.766592025756836 | Batch Time=14.0625 +GPU:2 | Epoch: 26 | loss=9.610692977905273 | Batch Time=18.75 +GPU:2 | Epoch: 26 | loss=9.564546585083008 | Batch Time=16.796875 +GPU:2 | Epoch: 26 | loss=9.475528717041016 | Batch Time=21.09375 +GPU:2 | Epoch: 26 | loss=9.320737838745117 | Batch Time=20.703125 +GPU:2 | Epoch: 26 | loss=9.699512481689453 | Batch Time=18.75 +GPU:2 | Epoch: 26 | loss=9.56663703918457 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.430763244628906 | Batch Time=19.140625 +GPU:2 | Epoch: 26 | loss=9.508655548095703 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.506017684936523 | Batch Time=17.1875 +GPU:2 | Epoch: 26 | loss=9.497566223144531 | Batch Time=21.09375 +GPU:2 | Epoch: 26 | loss=9.291228294372559 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.393621444702148 | Batch Time=21.484375 +GPU:2 | Epoch: 26 | loss=9.321735382080078 | Batch Time=21.875 +GPU:2 | Epoch: 26 | loss=9.32630443572998 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.557812690734863 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.601445198059082 | Batch Time=19.140625 +GPU:2 | Epoch: 26 | loss=9.326929092407227 | Batch Time=21.875 +GPU:2 | Epoch: 26 | loss=9.44891357421875 | Batch Time=18.75 +GPU:2 | Epoch: 26 | loss=9.327531814575195 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.403353691101074 | Batch Time=18.359375 +GPU:2 | Epoch: 26 | loss=9.554193496704102 | Batch Time=16.015625 +GPU:2 | Epoch: 26 | loss=9.510089874267578 | Batch Time=16.40625 +GPU:2 | Epoch: 26 | loss=9.486589431762695 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.37367057800293 | Batch Time=18.75 +GPU:2 | Epoch: 26 | loss=9.482186317443848 | Batch Time=21.484375 +GPU:2 | Epoch: 26 | loss=9.287887573242188 | Batch Time=21.09375 +GPU:2 | Epoch: 26 | loss=9.536266326904297 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.302062034606934 | Batch Time=20.3125 +GPU:2 | Epoch: 26 | loss=9.396858215332031 | Batch Time=18.359375 +GPU:2 | Epoch: 26 | loss=9.40429973602295 | Batch Time=14.453125 +GPU:2 | Epoch: 26 | loss=9.405601501464844 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.485481262207031 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.469001770019531 | Batch Time=21.484375 +GPU:2 | Epoch: 26 | loss=9.544227600097656 | Batch Time=19.53125 +GPU:2 | Epoch: 26 | loss=9.33334732055664 | Batch Time=18.359375 +GPU:2 | Epoch: 26 | loss=9.460307121276855 | Batch Time=14.453125 +GPU:2 | Epoch: 26 | loss=9.429588317871094 | Batch Time=20.3125 +GPU:2 | Epoch: 26 | loss=9.344844818115234 | Batch Time=19.53125 +GPU:2 | Epoch: 26 | loss=9.340816497802734 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.54000473022461 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.430865287780762 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.557561874389648 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.454227447509766 | Batch Time=18.359375 +GPU:2 | Epoch: 26 | loss=9.51108169555664 | Batch Time=17.1875 +GPU:2 | Epoch: 26 | loss=9.419265747070312 | Batch Time=21.484375 +GPU:2 | Epoch: 26 | loss=9.42991828918457 | Batch Time=17.1875 +GPU:2 | Epoch: 26 | loss=9.349838256835938 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.520610809326172 | Batch Time=17.1875 +GPU:2 | Epoch: 26 | loss=9.461771011352539 | Batch Time=16.796875 +GPU:2 | Epoch: 26 | loss=9.434618949890137 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.256845474243164 | Batch Time=21.09375 +GPU:2 | Epoch: 26 | loss=9.218618392944336 | Batch Time=21.484375 +GPU:2 | Epoch: 26 | loss=9.363550186157227 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.248584747314453 | Batch Time=19.140625 +GPU:2 | Epoch: 26 | loss=9.293288230895996 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.436532974243164 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.15321159362793 | Batch Time=23.828125 +GPU:2 | Epoch: 26 | loss=9.492267608642578 | Batch Time=19.53125 +GPU:2 | Epoch: 26 | loss=9.507543563842773 | Batch Time=20.703125 +GPU:2 | Epoch: 26 | loss=9.402791023254395 | Batch Time=18.75 +GPU:2 | Epoch: 26 | loss=9.386453628540039 | Batch Time=18.359375 +GPU:2 | Epoch: 26 | loss=9.556184768676758 | Batch Time=16.796875 +GPU:2 | Epoch: 26 | loss=9.652560234069824 | Batch Time=16.015625 +GPU:2 | Epoch: 26 | loss=9.273359298706055 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.531265258789062 | Batch Time=15.234375 +GPU:2 | Epoch: 26 | loss=9.425324440002441 | Batch Time=18.359375 +GPU:2 | Epoch: 26 | loss=9.435813903808594 | Batch Time=18.359375 +GPU:2 | Epoch: 26 | loss=9.39735221862793 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.455469131469727 | Batch Time=19.140625 +GPU:2 | Epoch: 26 | loss=9.56089973449707 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.408500671386719 | Batch Time=19.140625 +GPU:2 | Epoch: 26 | loss=9.604896545410156 | Batch Time=16.40625 +GPU:2 | Epoch: 26 | loss=9.532600402832031 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.563753128051758 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.441091537475586 | Batch Time=15.234375 +GPU:2 | Epoch: 26 | loss=9.57826042175293 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.405974388122559 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.31770133972168 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.471885681152344 | Batch Time=17.96875 +GPU:2 | Epoch: 26 | loss=9.4548921585083 | Batch Time=21.09375 +GPU:2 | Epoch: 26 | loss=9.438870429992676 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.170797348022461 | Batch Time=25.78125 +GPU:2 | Epoch: 26 | loss=9.333576202392578 | Batch Time=19.53125 +GPU:2 | Epoch: 26 | loss=9.51765251159668 | Batch Time=16.40625 +GPU:2 | Epoch: 26 | loss=9.413192749023438 | Batch Time=19.53125 +GPU:2 | Epoch: 26 | loss=9.434951782226562 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.45433235168457 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.297098159790039 | Batch Time=19.53125 +GPU:2 | Epoch: 26 | loss=9.465452194213867 | Batch Time=20.703125 +GPU:2 | Epoch: 26 | loss=9.668464660644531 | Batch Time=15.625 +GPU:2 | Epoch: 26 | loss=9.542659759521484 | Batch Time=18.75 +GPU:2 | Epoch: 26 | loss=9.433425903320312 | Batch Time=18.75 +GPU:2 | Epoch: 26 | loss=9.198863983154297 | Batch Time=19.140625 +GPU:2 | Epoch: 26 | loss=9.546548843383789 | Batch Time=16.015625 +GPU:2 | Epoch: 26 | loss=9.122979164123535 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.208086013793945 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.50521469116211 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.350113868713379 | Batch Time=22.265625 +GPU:2 | Epoch: 26 | loss=9.276878356933594 | Batch Time=22.265625 +GPU:2 | Epoch: 26 | loss=9.363810539245605 | Batch Time=21.09375 +GPU:2 | Epoch: 26 | loss=9.32213020324707 | Batch Time=23.046875 +GPU:2 | Epoch: 26 | loss=9.458908081054688 | Batch Time=15.234375 +GPU:2 | Epoch: 26 | loss=9.439018249511719 | Batch Time=23.4375 +GPU:2 | Epoch: 26 | loss=9.341371536254883 | Batch Time=18.75 +GPU:2 | Epoch: 26 | loss=9.56076431274414 | Batch Time=19.921875 +GPU:2 | Epoch: 26 | loss=9.117758750915527 | Batch Time=24.21875 +GPU:2 | Epoch: 26 | loss=9.330816268920898 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.388267517089844 | Batch Time=23.046875 +GPU:2 | Epoch: 26 | loss=9.399227142333984 | Batch Time=21.09375 +GPU:2 | Epoch: 26 | loss=9.395164489746094 | Batch Time=18.359375 +GPU:2 | Epoch: 26 | loss=9.38279914855957 | Batch Time=22.265625 +GPU:2 | Epoch: 26 | loss=9.400613784790039 | Batch Time=22.265625 +GPU:2 | Epoch: 26 | loss=9.384979248046875 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.174259185791016 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.21610164642334 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.481172561645508 | Batch Time=19.140625 +GPU:2 | Epoch: 26 | loss=9.376152992248535 | Batch Time=17.578125 +GPU:2 | Epoch: 26 | loss=9.302282333374023 | Batch Time=19.140625 +GPU:2 | Epoch: 26 | loss=9.34599781036377 | Batch Time=22.65625 +GPU:2 | Epoch: 26 | loss=9.188385963439941 | Batch Time=22.265625 +GPU:2 | Epoch: 26 | loss=9.458593368530273 | Batch Time=21.09375 +GPU:2 | Epoch: 26 | loss=9.463272094726562 | Batch Time=16.796875 +GPU:1 | Epoch: 26 | loss=9.834553718566895 | Batch Time=19.921875 +GPU:1 | Epoch: 26 | loss=9.653823852539062 | Batch Time=16.40625 +GPU:1 | Epoch: 26 | loss=9.529434204101562 | Batch Time=20.3125 +GPU:1 | Epoch: 26 | loss=9.673774719238281 | Batch Time=17.96875 +GPU:1 | Epoch: 26 | loss=9.63534927368164 | Batch Time=16.015625 +GPU:1 | Epoch: 26 | loss=9.45316219329834 | Batch Time=20.3125 +GPU:1 | Epoch: 26 | loss=9.72995376586914 | Batch Time=14.0625 +GPU:1 | Epoch: 26 | loss=9.550283432006836 | Batch Time=18.359375 +GPU:1 | Epoch: 26 | loss=9.483604431152344 | Batch Time=21.875 +GPU:1 | Epoch: 26 | loss=9.604227066040039 | Batch Time=16.796875 +GPU:1 | Epoch: 26 | loss=9.571820259094238 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.441035270690918 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.596952438354492 | Batch Time=16.015625 +GPU:1 | Epoch: 26 | loss=9.289206504821777 | Batch Time=21.484375 +GPU:1 | Epoch: 26 | loss=9.533465385437012 | Batch Time=19.921875 +GPU:1 | Epoch: 26 | loss=9.34434700012207 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.327302932739258 | Batch Time=22.65625 +GPU:1 | Epoch: 26 | loss=9.415453910827637 | Batch Time=22.65625 +GPU:1 | Epoch: 26 | loss=9.231363296508789 | Batch Time=23.828125 +GPU:1 | Epoch: 26 | loss=9.387776374816895 | Batch Time=18.359375 +GPU:1 | Epoch: 26 | loss=9.540998458862305 | Batch Time=18.359375 +GPU:1 | Epoch: 26 | loss=9.402228355407715 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.383014678955078 | Batch Time=18.359375 +GPU:1 | Epoch: 26 | loss=9.727895736694336 | Batch Time=17.1875 +GPU:1 | Epoch: 26 | loss=9.408761024475098 | Batch Time=17.96875 +GPU:1 | Epoch: 26 | loss=9.535982131958008 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.38838005065918 | Batch Time=25.78125 +GPU:1 | Epoch: 26 | loss=9.372550964355469 | Batch Time=22.65625 +GPU:1 | Epoch: 26 | loss=9.939420700073242 | Batch Time=13.28125 +GPU:1 | Epoch: 26 | loss=9.197798728942871 | Batch Time=22.65625 +GPU:1 | Epoch: 26 | loss=9.461877822875977 | Batch Time=18.75 +GPU:1 | Epoch: 26 | loss=9.535957336425781 | Batch Time=18.75 +GPU:1 | Epoch: 26 | loss=9.361042022705078 | Batch Time=18.75 +GPU:1 | Epoch: 26 | loss=9.455607414245605 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.496026992797852 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.375641822814941 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.640464782714844 | Batch Time=20.3125 +GPU:1 | Epoch: 26 | loss=9.54606819152832 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.5509614944458 | Batch Time=16.40625 +GPU:1 | Epoch: 26 | loss=9.419705390930176 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.240413665771484 | Batch Time=21.484375 +GPU:1 | Epoch: 26 | loss=9.284132957458496 | Batch Time=21.09375 +GPU:1 | Epoch: 26 | loss=9.37790584564209 | Batch Time=21.484375 +GPU:1 | Epoch: 26 | loss=9.493032455444336 | Batch Time=16.796875 +GPU:1 | Epoch: 26 | loss=9.531917572021484 | Batch Time=17.578125 +GPU:1 | Epoch: 26 | loss=9.464059829711914 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.383581161499023 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.352542877197266 | Batch Time=24.609375 +GPU:1 | Epoch: 26 | loss=9.476726531982422 | Batch Time=20.3125 +GPU:1 | Epoch: 26 | loss=9.245048522949219 | Batch Time=23.4375 +GPU:1 | Epoch: 26 | loss=9.331122398376465 | Batch Time=21.484375 +GPU:1 | Epoch: 26 | loss=9.529565811157227 | Batch Time=17.578125 +GPU:1 | Epoch: 26 | loss=9.438623428344727 | Batch Time=17.578125 +GPU:1 | Epoch: 26 | loss=9.478418350219727 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.48797607421875 | Batch Time=21.09375 +GPU:1 | Epoch: 26 | loss=9.301992416381836 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.390371322631836 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.351436614990234 | Batch Time=17.1875 +GPU:1 | Epoch: 26 | loss=9.647626876831055 | Batch Time=18.359375 +GPU:1 | Epoch: 26 | loss=9.531584739685059 | Batch Time=17.578125 +GPU:1 | Epoch: 26 | loss=9.545982360839844 | Batch Time=16.40625 +GPU:1 | Epoch: 26 | loss=9.389457702636719 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.245580673217773 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.643694877624512 | Batch Time=13.671875 +GPU:1 | Epoch: 26 | loss=9.62596321105957 | Batch Time=15.625 +GPU:1 | Epoch: 26 | loss=9.565827369689941 | Batch Time=16.40625 +GPU:1 | Epoch: 26 | loss=9.33545970916748 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.451070785522461 | Batch Time=19.921875 +GPU:1 | Epoch: 26 | loss=9.704465866088867 | Batch Time=17.578125 +GPU:1 | Epoch: 26 | loss=9.363308906555176 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.34037971496582 | Batch Time=22.65625 +GPU:1 | Epoch: 26 | loss=9.327620506286621 | Batch Time=22.65625 +GPU:1 | Epoch: 26 | loss=9.58933162689209 | Batch Time=13.28125 +GPU:1 | Epoch: 26 | loss=9.548088073730469 | Batch Time=17.1875 +GPU:1 | Epoch: 26 | loss=9.443717956542969 | Batch Time=22.65625 +GPU:1 | Epoch: 26 | loss=9.348154067993164 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.435201644897461 | Batch Time=18.359375 +GPU:1 | Epoch: 26 | loss=9.450425148010254 | Batch Time=18.359375 +GPU:1 | Epoch: 26 | loss=9.510706901550293 | Batch Time=16.015625 +GPU:1 | Epoch: 26 | loss=9.46826457977295 | Batch Time=14.84375 +GPU:1 | Epoch: 26 | loss=9.229477882385254 | Batch Time=23.046875 +GPU:1 | Epoch: 26 | loss=9.346696853637695 | Batch Time=21.484375 +GPU:1 | Epoch: 26 | loss=9.757424354553223 | Batch Time=16.015625 +GPU:1 | Epoch: 26 | loss=9.268915176391602 | Batch Time=21.09375 +GPU:1 | Epoch: 26 | loss=9.372759819030762 | Batch Time=22.65625 +GPU:1 | Epoch: 26 | loss=9.585546493530273 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.615304946899414 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.298712730407715 | Batch Time=18.75 +GPU:1 | Epoch: 26 | loss=9.32740592956543 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.439275741577148 | Batch Time=16.015625 +GPU:1 | Epoch: 26 | loss=9.293067932128906 | Batch Time=18.75 +GPU:1 | Epoch: 26 | loss=9.394547462463379 | Batch Time=17.96875 +GPU:1 | Epoch: 26 | loss=9.55428695678711 | Batch Time=20.3125 +GPU:1 | Epoch: 26 | loss=9.44192123413086 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.201051712036133 | Batch Time=25.390625 +GPU:1 | Epoch: 26 | loss=9.461036682128906 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.379633903503418 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.440832138061523 | Batch Time=16.40625 +GPU:1 | Epoch: 26 | loss=9.37211799621582 | Batch Time=18.75 +GPU:1 | Epoch: 26 | loss=9.274221420288086 | Batch Time=21.484375 +GPU:1 | Epoch: 26 | loss=9.354472160339355 | Batch Time=17.578125 +GPU:1 | Epoch: 26 | loss=9.472187042236328 | Batch Time=21.09375 +GPU:1 | Epoch: 26 | loss=9.298325538635254 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.564373970031738 | Batch Time=15.625 +GPU:1 | Epoch: 26 | loss=9.385717391967773 | Batch Time=16.40625 +GPU:1 | Epoch: 26 | loss=9.209161758422852 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.520780563354492 | Batch Time=13.671875 +GPU:1 | Epoch: 26 | loss=9.230487823486328 | Batch Time=21.484375 +GPU:1 | Epoch: 26 | loss=9.473501205444336 | Batch Time=17.96875 +GPU:1 | Epoch: 26 | loss=9.288439750671387 | Batch Time=23.828125 +GPU:1 | Epoch: 26 | loss=9.195066452026367 | Batch Time=21.09375 +GPU:1 | Epoch: 26 | loss=9.315058708190918 | Batch Time=21.875 +GPU:1 | Epoch: 26 | loss=9.522510528564453 | Batch Time=17.1875 +GPU:1 | Epoch: 26 | loss=9.515375137329102 | Batch Time=14.84375 +GPU:1 | Epoch: 26 | loss=9.564522743225098 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.186872482299805 | Batch Time=20.703125 +GPU:1 | Epoch: 26 | loss=9.425542831420898 | Batch Time=19.140625 +GPU:1 | Epoch: 26 | loss=9.251424789428711 | Batch Time=18.75 +GPU:1 | Epoch: 26 | loss=9.445409774780273 | Batch Time=18.359375 +GPU:1 | Epoch: 26 | loss=9.095672607421875 | Batch Time=21.484375 +GPU:1 | Epoch: 26 | loss=9.325052261352539 | Batch Time=19.53125 +GPU:1 | Epoch: 26 | loss=9.256725311279297 | Batch Time=21.875 +GPU:1 | Epoch: 26 | loss=9.25533676147461 | Batch Time=20.3125 +GPU:1 | Epoch: 26 | loss=9.101619720458984 | Batch Time=23.046875 +GPU:1 | Epoch: 26 | loss=9.459490776062012 | Batch Time=22.65625 +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +AFTER TRAIN LOOP: Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +BEFORE VAL LOOP: GPU: 0 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +AFTER TRAIN LOOP: Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +STARTING TRAINING: Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +BEFORE TRAIN LOOP: Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +AFTER TRAIN LOOP: Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +STARTING TRAINING: Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +BEFORE TRAIN LOOP: Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +AFTER TRAIN LOOP: Epoch 26 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +STARTING TRAINING: Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +BEFORE TRAIN LOOP: Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=34945064960, percent=86.4, used=214708858880, free=16878804992, active=214209630208, inactive=22475526144, buffers=388927488, cached=25591492608, shared=5662646272, slab=1206751232) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 26 | loss=3.3783013820648193 | Batch Time=30.46875 +GPU:0 | Epoch: 26 | loss=4.498996257781982 | Batch Time=14.84375 +GPU:0 | Epoch: 26 | loss=4.302589416503906 | Batch Time=24.21875 +GPU:0 | Epoch: 26 | loss=3.886561155319214 | Batch Time=10.15625 +GPU:0 | Epoch: 26 | loss=4.549664497375488 | Batch Time=12.5 +GPU:0 | Epoch: 26 | loss=4.066972732543945 | Batch Time=13.671875 +GPU:0 | Epoch: 26 | loss=4.65991735458374 | Batch Time=8.984375 +GPU:0 | Epoch: 26 | loss=5.173993110656738 | Batch Time=8.203125 +GPU:0 | Epoch: 26 | loss=4.701009750366211 | Batch Time=9.765625 +GPU:0 | Epoch: 26 | loss=5.688297748565674 | Batch Time=4.6875 +GPU:0 | Epoch: 26 | loss=5.1900811195373535 | Batch Time=2.734375 +GPU:0 | Epoch: 26 | loss=4.072653770446777 | Batch Time=22.65625 +GPU:0 | Epoch: 26 | loss=5.183501720428467 | Batch Time=10.15625 +GPU:0 | Epoch: 26 | loss=4.090492248535156 | Batch Time=20.703125 +GPU:0 | Epoch: 26 | loss=4.219912528991699 | Batch Time=13.671875 +GPU:0 | Epoch: 26 | loss=4.756687164306641 | Batch Time=14.0625 +GPU:0 | Epoch: 26 | loss=3.971860647201538 | Batch Time=16.796875 +GPU:0 | Epoch: 26 | loss=3.033238172531128 | Batch Time=37.890625 +GPU:0 | Epoch: 26 | loss=3.6591739654541016 | Batch Time=23.4375 +GPU:0 | Epoch: 26 | loss=3.324167251586914 | Batch Time=29.296875 +Model top1 Accuracy: 18.106 +Acc before rounding: 18.106 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 26 | loss=3.3783013820648193 | Batch Time=30.46875 +GPU:0 | Epoch: 26 | loss=4.498996257781982 | Batch Time=14.84375 +GPU:0 | Epoch: 26 | loss=4.302589416503906 | Batch Time=24.21875 +GPU:0 | Epoch: 26 | loss=3.886561155319214 | Batch Time=10.15625 +GPU:0 | Epoch: 26 | loss=4.549664497375488 | Batch Time=12.5 +GPU:0 | Epoch: 26 | loss=4.066972732543945 | Batch Time=13.671875 +GPU:0 | Epoch: 26 | loss=4.65991735458374 | Batch Time=8.984375 +GPU:0 | Epoch: 26 | loss=5.173993110656738 | Batch Time=8.203125 +GPU:0 | Epoch: 26 | loss=4.701009750366211 | Batch Time=9.765625 +GPU:0 | Epoch: 26 | loss=5.688297748565674 | Batch Time=4.6875 +GPU:0 | Epoch: 26 | loss=5.1900811195373535 | Batch Time=2.734375 +GPU:0 | Epoch: 26 | loss=4.072653770446777 | Batch Time=22.65625 +GPU:0 | Epoch: 26 | loss=5.183501720428467 | Batch Time=10.15625 +GPU:0 | Epoch: 26 | loss=4.090492248535156 | Batch Time=20.703125 +GPU:0 | Epoch: 26 | loss=4.219912528991699 | Batch Time=13.671875 +GPU:0 | Epoch: 26 | loss=4.756687164306641 | Batch Time=14.0625 +GPU:0 | Epoch: 26 | loss=3.971860647201538 | Batch Time=16.796875 +GPU:0 | Epoch: 26 | loss=3.033238172531128 | Batch Time=37.890625 +GPU:0 | Epoch: 26 | loss=3.6591739654541016 | Batch Time=23.4375 +GPU:0 | Epoch: 26 | loss=3.324167251586914 | Batch Time=29.296875 +Model top1 Accuracy: 18.106 +Acc after rounding: 18.106 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 26 | loss=4.449991226196289 | Batch Time=18.359375 +GPU:0 | Epoch: 26 | loss=4.568846702575684 | Batch Time=17.1875 +GPU:0 | Epoch: 26 | loss=4.435697078704834 | Batch Time=17.96875 +GPU:0 | Epoch: 26 | loss=4.553837299346924 | Batch Time=12.890625 +Model top1 Accuracy: 15.44 +Validation Acc after rounding: 15.44 +AFTER VAL LOOP: GPU: 0 | Epoch 26 | Memory Usage: svmem(total=257568083968, available=17136197632, percent=93.3, used=232531369984, free=5151920128, active=236835586048, inactive=11482034176, buffers=418443264, cached=19466350592, shared=5662826496, slab=1185157120) +Rounding model with scheme: naive +Model avg sparsity: 35.61869579048418 +GPU:0 | Epoch: 26 | Acc=18.106 | Epoch Time=23.065761987368266 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 26 | Memory Usage: svmem(total=257568083968, available=17136713728, percent=93.3, used=232530345984, free=5151404032, active=236835610624, inactive=11483066368, buffers=418451456, cached=19467882496, shared=5662826496, slab=1185157120) +STARTING TRAINING: Epoch 27 | Memory Usage: svmem(total=257568083968, available=17136713728, percent=93.3, used=232530345984, free=5151404032, active=236835610624, inactive=11483066368, buffers=418451456, cached=19467882496, shared=5662826496, slab=1185157120) +BEFORE TRAIN LOOP: Epoch 27 | Memory Usage: svmem(total=257568083968, available=17136713728, percent=93.3, used=232530345984, free=5151404032, active=236835610624, inactive=11483066368, buffers=418451456, cached=19467882496, shared=5662826496, slab=1185157120) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=17136713728, percent=93.3, used=232530345984, free=5151404032, active=236835610624, inactive=11483066368, buffers=418451456, cached=19467882496, shared=5662826496, slab=1185157120) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 27 | loss=9.359593391418457 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.314566612243652 | Batch Time=17.578125 +GPU:0 | Epoch: 27 | loss=9.434240341186523 | Batch Time=18.75 +GPU:0 | Epoch: 27 | loss=9.304951667785645 | Batch Time=24.609375 +GPU:0 | Epoch: 27 | loss=9.418087005615234 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.415946960449219 | Batch Time=23.046875 +GPU:0 | Epoch: 27 | loss=9.139544486999512 | Batch Time=23.4375 +GPU:0 | Epoch: 27 | loss=9.26517105102539 | Batch Time=21.09375 +GPU:0 | Epoch: 27 | loss=9.307106018066406 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.091011047363281 | Batch Time=22.65625 +GPU:0 | Epoch: 27 | loss=9.169504165649414 | Batch Time=23.046875 +GPU:0 | Epoch: 27 | loss=9.179328918457031 | Batch Time=18.75 +GPU:0 | Epoch: 27 | loss=9.35911750793457 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.332925796508789 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.604663848876953 | Batch Time=15.234375 +GPU:0 | Epoch: 27 | loss=9.353829383850098 | Batch Time=20.703125 +GPU:0 | Epoch: 27 | loss=9.128975868225098 | Batch Time=23.828125 +GPU:0 | Epoch: 27 | loss=9.517694473266602 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.48037338256836 | Batch Time=14.453125 +GPU:0 | Epoch: 27 | loss=9.144429206848145 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.36471176147461 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.260659217834473 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.230960845947266 | Batch Time=23.4375 +GPU:0 | Epoch: 27 | loss=9.34133529663086 | Batch Time=20.703125 +GPU:0 | Epoch: 27 | loss=9.210927963256836 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.242128372192383 | Batch Time=23.4375 +GPU:0 | Epoch: 27 | loss=9.529901504516602 | Batch Time=16.796875 +GPU:0 | Epoch: 27 | loss=9.200376510620117 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.241477966308594 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.106293678283691 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.177770614624023 | Batch Time=23.046875 +GPU:0 | Epoch: 27 | loss=9.393455505371094 | Batch Time=20.703125 +GPU:0 | Epoch: 27 | loss=9.122756958007812 | Batch Time=26.171875 +GPU:0 | Epoch: 27 | loss=9.453954696655273 | Batch Time=14.84375 +GPU:0 | Epoch: 27 | loss=9.266898155212402 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.299530029296875 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.300283432006836 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.201583862304688 | Batch Time=21.09375 +GPU:0 | Epoch: 27 | loss=9.280488967895508 | Batch Time=17.1875 +GPU:0 | Epoch: 27 | loss=9.250449180603027 | Batch Time=20.703125 +GPU:0 | Epoch: 27 | loss=9.374408721923828 | Batch Time=19.53125 +GPU:0 | Epoch: 27 | loss=9.300691604614258 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.292264938354492 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.229207038879395 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.146835327148438 | Batch Time=23.046875 +GPU:0 | Epoch: 27 | loss=9.290054321289062 | Batch Time=18.75 +GPU:0 | Epoch: 27 | loss=9.163564682006836 | Batch Time=20.703125 +GPU:0 | Epoch: 27 | loss=9.192008972167969 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.28979206085205 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.546016693115234 | Batch Time=17.96875 +GPU:0 | Epoch: 27 | loss=9.530073165893555 | Batch Time=16.796875 +GPU:0 | Epoch: 27 | loss=9.20047664642334 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.24885082244873 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.377391815185547 | Batch Time=16.40625 +GPU:0 | Epoch: 27 | loss=9.435287475585938 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.433612823486328 | Batch Time=16.40625 +GPU:0 | Epoch: 27 | loss=9.331803321838379 | Batch Time=16.015625 +GPU:0 | Epoch: 27 | loss=9.273990631103516 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.247060775756836 | Batch Time=22.65625 +GPU:0 | Epoch: 27 | loss=9.302850723266602 | Batch Time=17.96875 +GPU:0 | Epoch: 27 | loss=9.158956527709961 | Batch Time=21.09375 +GPU:0 | Epoch: 27 | loss=9.421735763549805 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.390939712524414 | Batch Time=20.3125 +GPU:0 | Epoch: 27 | loss=9.253381729125977 | Batch Time=22.265625 +GPU:0 | Epoch: 27 | loss=9.553854942321777 | Batch Time=16.015625 +GPU:0 | Epoch: 27 | loss=9.131591796875 | Batch Time=22.65625 +GPU:0 | Epoch: 27 | loss=9.20230484008789 | Batch Time=23.4375 +GPU:0 | Epoch: 27 | loss=9.134449005126953 | Batch Time=22.65625 +GPU:0 | Epoch: 27 | loss=9.35619068145752 | Batch Time=20.3125 +GPU:0 | Epoch: 27 | loss=9.322502136230469 | Batch Time=23.828125 +GPU:0 | Epoch: 27 | loss=9.12720012664795 | Batch Time=24.21875 +GPU:0 | Epoch: 27 | loss=9.200339317321777 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.323783874511719 | Batch Time=20.703125 +GPU:0 | Epoch: 27 | loss=9.108664512634277 | Batch Time=23.828125 +GPU:0 | Epoch: 27 | loss=9.319814682006836 | Batch Time=22.265625 +GPU:0 | Epoch: 27 | loss=9.090681076049805 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.402019500732422 | Batch Time=15.234375 +GPU:0 | Epoch: 27 | loss=9.305063247680664 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.114734649658203 | Batch Time=25.390625 +GPU:0 | Epoch: 27 | loss=9.325695037841797 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.312564849853516 | Batch Time=19.53125 +GPU:0 | Epoch: 27 | loss=9.07748031616211 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=9.236623764038086 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.190580368041992 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.250089645385742 | Batch Time=16.796875 +GPU:0 | Epoch: 27 | loss=9.018303871154785 | Batch Time=24.21875 +GPU:0 | Epoch: 27 | loss=9.214555740356445 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.258455276489258 | Batch Time=22.65625 +GPU:0 | Epoch: 27 | loss=9.4197416305542 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.083724975585938 | Batch Time=23.828125 +GPU:0 | Epoch: 27 | loss=9.101966857910156 | Batch Time=23.4375 +GPU:0 | Epoch: 27 | loss=9.47165298461914 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.042703628540039 | Batch Time=20.3125 +GPU:0 | Epoch: 27 | loss=9.185709953308105 | Batch Time=21.09375 +GPU:0 | Epoch: 27 | loss=9.525411605834961 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.064081192016602 | Batch Time=23.4375 +GPU:0 | Epoch: 27 | loss=8.993877410888672 | Batch Time=22.265625 +GPU:0 | Epoch: 27 | loss=9.482234954833984 | Batch Time=15.234375 +GPU:0 | Epoch: 27 | loss=9.13165283203125 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.167635917663574 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.215855598449707 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.289591789245605 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.312171936035156 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=9.435571670532227 | Batch Time=15.625 +GPU:0 | Epoch: 27 | loss=9.244827270507812 | Batch Time=19.53125 +GPU:0 | Epoch: 27 | loss=9.166820526123047 | Batch Time=26.5625 +GPU:0 | Epoch: 27 | loss=9.412948608398438 | Batch Time=20.3125 +GPU:0 | Epoch: 27 | loss=9.343650817871094 | Batch Time=18.75 +GPU:0 | Epoch: 27 | loss=9.236783981323242 | Batch Time=17.1875 +GPU:0 | Epoch: 27 | loss=9.176896095275879 | Batch Time=19.53125 +GPU:0 | Epoch: 27 | loss=9.157825469970703 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.28831672668457 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.02224349975586 | Batch Time=23.828125 +GPU:0 | Epoch: 27 | loss=9.39764404296875 | Batch Time=22.65625 +GPU:0 | Epoch: 27 | loss=9.229286193847656 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.324687004089355 | Batch Time=19.921875 +GPU:0 | Epoch: 27 | loss=9.235553741455078 | Batch Time=19.53125 +GPU:0 | Epoch: 27 | loss=9.379100799560547 | Batch Time=17.1875 +GPU:0 | Epoch: 27 | loss=9.157089233398438 | Batch Time=21.875 +GPU:0 | Epoch: 27 | loss=9.069016456604004 | Batch Time=23.4375 +GPU:0 | Epoch: 27 | loss=9.162270545959473 | Batch Time=23.828125 +GPU:0 | Epoch: 27 | loss=9.204752922058105 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.27813720703125 | Batch Time=19.140625 +GPU:0 | Epoch: 27 | loss=9.178813934326172 | Batch Time=19.53125 +GPU:0 | Epoch: 27 | loss=8.811363220214844 | Batch Time=28.90625 +GPU:1 | Epoch: 27 | loss=9.334878921508789 | Batch Time=17.578125 +GPU:1 | Epoch: 27 | loss=9.230172157287598 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.270956039428711 | Batch Time=25.0 +GPU:1 | Epoch: 27 | loss=9.370353698730469 | Batch Time=21.09375 +GPU:1 | Epoch: 27 | loss=9.285125732421875 | Batch Time=20.703125 +GPU:1 | Epoch: 27 | loss=9.337539672851562 | Batch Time=19.140625 +GPU:1 | Epoch: 27 | loss=9.265113830566406 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.3282470703125 | Batch Time=18.359375 +GPU:1 | Epoch: 27 | loss=9.375011444091797 | Batch Time=17.578125 +GPU:1 | Epoch: 27 | loss=9.3846435546875 | Batch Time=17.578125 +GPU:1 | Epoch: 27 | loss=9.38585090637207 | Batch Time=16.40625 +GPU:1 | Epoch: 27 | loss=9.364742279052734 | Batch Time=20.703125 +GPU:1 | Epoch: 27 | loss=9.297186851501465 | Batch Time=25.0 +GPU:1 | Epoch: 27 | loss=9.104101181030273 | Batch Time=21.484375 +GPU:1 | Epoch: 27 | loss=9.228116989135742 | Batch Time=19.53125 +GPU:1 | Epoch: 27 | loss=9.426002502441406 | Batch Time=16.015625 +GPU:1 | Epoch: 27 | loss=9.227524757385254 | Batch Time=21.09375 +GPU:1 | Epoch: 27 | loss=9.320377349853516 | Batch Time=17.96875 +GPU:1 | Epoch: 27 | loss=9.37213134765625 | Batch Time=19.53125 +GPU:1 | Epoch: 27 | loss=9.475181579589844 | Batch Time=16.40625 +GPU:1 | Epoch: 27 | loss=9.339370727539062 | Batch Time=20.703125 +GPU:1 | Epoch: 27 | loss=9.251157760620117 | Batch Time=22.65625 +GPU:1 | Epoch: 27 | loss=9.274858474731445 | Batch Time=22.65625 +GPU:1 | Epoch: 27 | loss=9.068439483642578 | Batch Time=21.875 +GPU:1 | Epoch: 27 | loss=9.219655990600586 | Batch Time=19.921875 +GPU:1 | Epoch: 27 | loss=9.167911529541016 | Batch Time=23.4375 +GPU:1 | Epoch: 27 | loss=9.09873104095459 | Batch Time=27.34375 +GPU:1 | Epoch: 27 | loss=9.393325805664062 | Batch Time=21.484375 +GPU:1 | Epoch: 27 | loss=9.464449882507324 | Batch Time=18.75 +GPU:1 | Epoch: 27 | loss=9.221384048461914 | Batch Time=19.53125 +GPU:1 | Epoch: 27 | loss=9.26809310913086 | Batch Time=18.359375 +GPU:1 | Epoch: 27 | loss=9.432182312011719 | Batch Time=18.75 +GPU:1 | Epoch: 27 | loss=9.246902465820312 | Batch Time=19.140625 +GPU:1 | Epoch: 27 | loss=9.24785327911377 | Batch Time=19.53125 +GPU:1 | Epoch: 27 | loss=9.386763572692871 | Batch Time=18.75 +GPU:1 | Epoch: 27 | loss=9.255701065063477 | Batch Time=21.484375 +GPU:1 | Epoch: 27 | loss=9.076953887939453 | Batch Time=28.90625 +GPU:1 | Epoch: 27 | loss=9.322805404663086 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.14638900756836 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.629796981811523 | Batch Time=17.96875 +GPU:1 | Epoch: 27 | loss=9.301301956176758 | Batch Time=19.140625 +GPU:1 | Epoch: 27 | loss=9.342948913574219 | Batch Time=20.703125 +GPU:1 | Epoch: 27 | loss=9.230506896972656 | Batch Time=20.703125 +GPU:1 | Epoch: 27 | loss=9.757295608520508 | Batch Time=17.1875 +GPU:1 | Epoch: 27 | loss=9.43250846862793 | Batch Time=17.96875 +GPU:1 | Epoch: 27 | loss=9.367378234863281 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.165754318237305 | Batch Time=25.0 +GPU:1 | Epoch: 27 | loss=9.324017524719238 | Batch Time=17.1875 +GPU:1 | Epoch: 27 | loss=9.152215003967285 | Batch Time=23.828125 +GPU:1 | Epoch: 27 | loss=9.261697769165039 | Batch Time=21.875 +GPU:1 | Epoch: 27 | loss=9.159002304077148 | Batch Time=18.75 +GPU:1 | Epoch: 27 | loss=9.346460342407227 | Batch Time=17.1875 +GPU:1 | Epoch: 27 | loss=9.371706008911133 | Batch Time=17.96875 +GPU:1 | Epoch: 27 | loss=9.478201866149902 | Batch Time=19.921875 +GPU:1 | Epoch: 27 | loss=9.236011505126953 | Batch Time=21.875 +GPU:1 | Epoch: 27 | loss=9.168119430541992 | Batch Time=24.21875 +GPU:1 | Epoch: 27 | loss=9.423114776611328 | Batch Time=17.96875 +GPU:1 | Epoch: 27 | loss=9.46440315246582 | Batch Time=16.796875 +GPU:1 | Epoch: 27 | loss=9.243169784545898 | Batch Time=23.046875 +GPU:1 | Epoch: 27 | loss=9.164877891540527 | Batch Time=18.359375 +GPU:1 | Epoch: 27 | loss=9.217788696289062 | Batch Time=19.921875 +GPU:1 | Epoch: 27 | loss=9.083063125610352 | Batch Time=24.609375 +GPU:1 | Epoch: 27 | loss=9.249171257019043 | Batch Time=25.390625 +GPU:1 | Epoch: 27 | loss=9.490518569946289 | Batch Time=15.234375 +GPU:1 | Epoch: 27 | loss=9.302517890930176 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.700202941894531 | Batch Time=16.796875 +GPU:1 | Epoch: 27 | loss=9.548891067504883 | Batch Time=17.1875 +GPU:1 | Epoch: 27 | loss=9.197681427001953 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.192216873168945 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.196325302124023 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.17904281616211 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.535306930541992 | Batch Time=14.84375 +GPU:1 | Epoch: 27 | loss=9.302788734436035 | Batch Time=18.359375 +GPU:1 | Epoch: 27 | loss=9.435324668884277 | Batch Time=19.140625 +GPU:1 | Epoch: 27 | loss=9.349164009094238 | Batch Time=19.921875 +GPU:1 | Epoch: 27 | loss=9.126327514648438 | Batch Time=21.484375 +GPU:1 | Epoch: 27 | loss=9.350166320800781 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.467692375183105 | Batch Time=18.359375 +GPU:1 | Epoch: 27 | loss=9.265188217163086 | Batch Time=16.40625 +GPU:1 | Epoch: 27 | loss=9.1054048538208 | Batch Time=21.09375 +GPU:1 | Epoch: 27 | loss=9.215771675109863 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.325138092041016 | Batch Time=19.921875 +GPU:1 | Epoch: 27 | loss=9.39767837524414 | Batch Time=21.09375 +GPU:1 | Epoch: 27 | loss=9.172188758850098 | Batch Time=19.53125 +GPU:1 | Epoch: 27 | loss=9.275702476501465 | Batch Time=15.625 +GPU:1 | Epoch: 27 | loss=9.355366706848145 | Batch Time=19.921875 +GPU:1 | Epoch: 27 | loss=9.173942565917969 | Batch Time=21.484375 +GPU:1 | Epoch: 27 | loss=9.241071701049805 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.445517539978027 | Batch Time=19.53125 +GPU:1 | Epoch: 27 | loss=9.05588150024414 | Batch Time=23.4375 +GPU:1 | Epoch: 27 | loss=9.319485664367676 | Batch Time=18.75 +GPU:1 | Epoch: 27 | loss=9.213253021240234 | Batch Time=20.703125 +GPU:1 | Epoch: 27 | loss=9.24411392211914 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.349991798400879 | Batch Time=16.40625 +GPU:1 | Epoch: 27 | loss=9.226409912109375 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.3387451171875 | Batch Time=18.75 +GPU:1 | Epoch: 27 | loss=9.131864547729492 | Batch Time=24.21875 +GPU:1 | Epoch: 27 | loss=9.282405853271484 | Batch Time=16.796875 +GPU:1 | Epoch: 27 | loss=9.011970520019531 | Batch Time=19.140625 +GPU:1 | Epoch: 27 | loss=9.299924850463867 | Batch Time=21.09375 +GPU:1 | Epoch: 27 | loss=9.449264526367188 | Batch Time=18.75 +GPU:1 | Epoch: 27 | loss=9.387903213500977 | Batch Time=21.09375 +GPU:1 | Epoch: 27 | loss=9.216550827026367 | Batch Time=19.921875 +GPU:1 | Epoch: 27 | loss=9.24197769165039 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.219274520874023 | Batch Time=20.3125 +GPU:1 | Epoch: 27 | loss=9.235210418701172 | Batch Time=23.828125 +GPU:1 | Epoch: 27 | loss=9.071968078613281 | Batch Time=26.5625 +GPU:1 | Epoch: 27 | loss=8.948807716369629 | Batch Time=24.21875 +GPU:1 | Epoch: 27 | loss=9.266347885131836 | Batch Time=19.921875 +GPU:1 | Epoch: 27 | loss=9.173566818237305 | Batch Time=21.09375 +GPU:1 | Epoch: 27 | loss=9.281858444213867 | Batch Time=16.796875 +GPU:1 | Epoch: 27 | loss=8.994895935058594 | Batch Time=21.484375 +GPU:1 | Epoch: 27 | loss=9.244205474853516 | Batch Time=14.453125 +GPU:1 | Epoch: 27 | loss=9.130949974060059 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.204928398132324 | Batch Time=18.75 +GPU:1 | Epoch: 27 | loss=9.276859283447266 | Batch Time=18.359375 +GPU:1 | Epoch: 27 | loss=9.069393157958984 | Batch Time=22.65625 +GPU:1 | Epoch: 27 | loss=9.254825592041016 | Batch Time=14.84375 +GPU:1 | Epoch: 27 | loss=9.393861770629883 | Batch Time=15.625 +GPU:1 | Epoch: 27 | loss=9.201708793640137 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.30685806274414 | Batch Time=22.265625 +GPU:1 | Epoch: 27 | loss=9.181150436401367 | Batch Time=21.875 +GPU:1 | Epoch: 27 | loss=9.08806324005127 | Batch Time=20.703125 +GPU:1 | Epoch: 27 | loss=9.199383735656738 | Batch Time=20.703125 +GPU:1 | Epoch: 27 | loss=9.210000991821289 | Batch Time=19.53125 +GPU:3 | Epoch: 27 | loss=9.35168743133545 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.534011840820312 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.180059432983398 | Batch Time=22.65625 +GPU:3 | Epoch: 27 | loss=9.232542991638184 | Batch Time=22.65625 +GPU:3 | Epoch: 27 | loss=9.403553009033203 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.11745834350586 | Batch Time=25.390625 +GPU:3 | Epoch: 27 | loss=9.311351776123047 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.167951583862305 | Batch Time=23.828125 +GPU:3 | Epoch: 27 | loss=9.457603454589844 | Batch Time=17.578125 +GPU:3 | Epoch: 27 | loss=9.360563278198242 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.370075225830078 | Batch Time=19.921875 +GPU:3 | Epoch: 27 | loss=9.372316360473633 | Batch Time=17.578125 +GPU:3 | Epoch: 27 | loss=9.511608123779297 | Batch Time=18.359375 +GPU:3 | Epoch: 27 | loss=9.431936264038086 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.320590019226074 | Batch Time=19.53125 +GPU:3 | Epoch: 27 | loss=9.353256225585938 | Batch Time=17.1875 +GPU:3 | Epoch: 27 | loss=9.354286193847656 | Batch Time=20.703125 +GPU:3 | Epoch: 27 | loss=9.170387268066406 | Batch Time=19.921875 +GPU:3 | Epoch: 27 | loss=9.272294998168945 | Batch Time=23.4375 +GPU:3 | Epoch: 27 | loss=9.389957427978516 | Batch Time=17.96875 +GPU:3 | Epoch: 27 | loss=9.534910202026367 | Batch Time=17.578125 +GPU:3 | Epoch: 27 | loss=9.206840515136719 | Batch Time=17.1875 +GPU:3 | Epoch: 27 | loss=9.271221160888672 | Batch Time=22.265625 +GPU:3 | Epoch: 27 | loss=9.205257415771484 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.299905776977539 | Batch Time=19.53125 +GPU:3 | Epoch: 27 | loss=9.27493953704834 | Batch Time=20.703125 +GPU:3 | Epoch: 27 | loss=9.101700782775879 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.391091346740723 | Batch Time=17.96875 +GPU:3 | Epoch: 27 | loss=9.221702575683594 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.254810333251953 | Batch Time=20.3125 +GPU:3 | Epoch: 27 | loss=9.427489280700684 | Batch Time=17.96875 +GPU:3 | Epoch: 27 | loss=9.297699928283691 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.142102241516113 | Batch Time=22.65625 +GPU:3 | Epoch: 27 | loss=9.208412170410156 | Batch Time=22.65625 +GPU:3 | Epoch: 27 | loss=9.07667350769043 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.197535514831543 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.316329956054688 | Batch Time=20.3125 +GPU:3 | Epoch: 27 | loss=9.323359489440918 | Batch Time=19.53125 +GPU:3 | Epoch: 27 | loss=9.428865432739258 | Batch Time=16.40625 +GPU:3 | Epoch: 27 | loss=9.312440872192383 | Batch Time=16.015625 +GPU:3 | Epoch: 27 | loss=9.369407653808594 | Batch Time=17.96875 +GPU:3 | Epoch: 27 | loss=9.235851287841797 | Batch Time=21.875 +GPU:3 | Epoch: 27 | loss=9.447755813598633 | Batch Time=16.796875 +GPU:3 | Epoch: 27 | loss=9.415779113769531 | Batch Time=18.359375 +GPU:3 | Epoch: 27 | loss=9.313226699829102 | Batch Time=17.96875 +GPU:3 | Epoch: 27 | loss=9.48539924621582 | Batch Time=17.578125 +GPU:3 | Epoch: 27 | loss=9.256146430969238 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.317357063293457 | Batch Time=22.265625 +GPU:3 | Epoch: 27 | loss=9.264762878417969 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.461019515991211 | Batch Time=16.796875 +GPU:3 | Epoch: 27 | loss=9.032390594482422 | Batch Time=20.703125 +GPU:3 | Epoch: 27 | loss=9.185047149658203 | Batch Time=24.609375 +GPU:3 | Epoch: 27 | loss=9.403593063354492 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.342206954956055 | Batch Time=18.359375 +GPU:3 | Epoch: 27 | loss=9.289857864379883 | Batch Time=22.265625 +GPU:3 | Epoch: 27 | loss=9.26887321472168 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.39862060546875 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.287216186523438 | Batch Time=23.046875 +GPU:3 | Epoch: 27 | loss=9.303718566894531 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.199180603027344 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.212042808532715 | Batch Time=19.921875 +GPU:3 | Epoch: 27 | loss=9.35293960571289 | Batch Time=20.3125 +GPU:3 | Epoch: 27 | loss=9.137654304504395 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.400774002075195 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.458878517150879 | Batch Time=18.359375 +GPU:3 | Epoch: 27 | loss=9.48176383972168 | Batch Time=20.703125 +GPU:3 | Epoch: 27 | loss=9.10373306274414 | Batch Time=23.046875 +GPU:3 | Epoch: 27 | loss=8.981942176818848 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.265840530395508 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.219442367553711 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.336188316345215 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.402722358703613 | Batch Time=20.703125 +GPU:3 | Epoch: 27 | loss=9.231657028198242 | Batch Time=20.3125 +GPU:3 | Epoch: 27 | loss=9.324487686157227 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.143467903137207 | Batch Time=21.875 +GPU:3 | Epoch: 27 | loss=9.449743270874023 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.191621780395508 | Batch Time=22.65625 +GPU:3 | Epoch: 27 | loss=9.50815486907959 | Batch Time=13.671875 +GPU:3 | Epoch: 27 | loss=9.340347290039062 | Batch Time=16.015625 +GPU:3 | Epoch: 27 | loss=9.047229766845703 | Batch Time=23.046875 +GPU:3 | Epoch: 27 | loss=9.404548645019531 | Batch Time=21.875 +GPU:3 | Epoch: 27 | loss=9.297002792358398 | Batch Time=20.703125 +GPU:3 | Epoch: 27 | loss=9.20730209350586 | Batch Time=19.140625 +GPU:3 | Epoch: 27 | loss=9.268921852111816 | Batch Time=24.609375 +GPU:3 | Epoch: 27 | loss=9.167925834655762 | Batch Time=22.265625 +GPU:3 | Epoch: 27 | loss=9.21840763092041 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.326702117919922 | Batch Time=20.3125 +GPU:3 | Epoch: 27 | loss=9.535560607910156 | Batch Time=16.015625 +GPU:3 | Epoch: 27 | loss=9.247361183166504 | Batch Time=19.53125 +GPU:3 | Epoch: 27 | loss=9.105297088623047 | Batch Time=21.875 +GPU:3 | Epoch: 27 | loss=9.134498596191406 | Batch Time=23.046875 +GPU:3 | Epoch: 27 | loss=9.099438667297363 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.270423889160156 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.408897399902344 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.162751197814941 | Batch Time=24.21875 +GPU:3 | Epoch: 27 | loss=9.153230667114258 | Batch Time=22.65625 +GPU:3 | Epoch: 27 | loss=9.063104629516602 | Batch Time=21.875 +GPU:3 | Epoch: 27 | loss=9.173295974731445 | Batch Time=21.875 +GPU:3 | Epoch: 27 | loss=9.056421279907227 | Batch Time=25.0 +GPU:3 | Epoch: 27 | loss=9.201894760131836 | Batch Time=18.359375 +GPU:3 | Epoch: 27 | loss=9.333703994750977 | Batch Time=20.3125 +GPU:3 | Epoch: 27 | loss=9.311729431152344 | Batch Time=15.234375 +GPU:3 | Epoch: 27 | loss=9.085999488830566 | Batch Time=24.21875 +GPU:3 | Epoch: 27 | loss=9.327139854431152 | Batch Time=16.796875 +GPU:3 | Epoch: 27 | loss=9.289135932922363 | Batch Time=19.53125 +GPU:3 | Epoch: 27 | loss=8.983307838439941 | Batch Time=21.484375 +GPU:3 | Epoch: 27 | loss=9.352389335632324 | Batch Time=16.796875 +GPU:3 | Epoch: 27 | loss=9.406570434570312 | Batch Time=17.1875 +GPU:3 | Epoch: 27 | loss=9.399589538574219 | Batch Time=17.96875 +GPU:3 | Epoch: 27 | loss=9.156280517578125 | Batch Time=19.921875 +GPU:3 | Epoch: 27 | loss=9.274723052978516 | Batch Time=17.96875 +GPU:3 | Epoch: 27 | loss=9.094490051269531 | Batch Time=22.265625 +GPU:3 | Epoch: 27 | loss=9.305999755859375 | Batch Time=22.265625 +GPU:3 | Epoch: 27 | loss=9.18820858001709 | Batch Time=19.53125 +GPU:3 | Epoch: 27 | loss=9.398519515991211 | Batch Time=17.96875 +GPU:3 | Epoch: 27 | loss=9.330615997314453 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.11233901977539 | Batch Time=21.09375 +GPU:3 | Epoch: 27 | loss=9.354061126708984 | Batch Time=14.0625 +GPU:3 | Epoch: 27 | loss=9.078438758850098 | Batch Time=24.21875 +GPU:3 | Epoch: 27 | loss=9.563238143920898 | Batch Time=17.578125 +GPU:3 | Epoch: 27 | loss=9.268611907958984 | Batch Time=25.0 +GPU:3 | Epoch: 27 | loss=9.24782943725586 | Batch Time=18.75 +GPU:3 | Epoch: 27 | loss=9.210437774658203 | Batch Time=20.703125 +GPU:3 | Epoch: 27 | loss=8.94155216217041 | Batch Time=24.609375 +GPU:3 | Epoch: 27 | loss=9.001596450805664 | Batch Time=23.828125 +GPU:2 | Epoch: 27 | loss=9.416168212890625 | Batch Time=16.015625 +GPU:2 | Epoch: 27 | loss=9.631011009216309 | Batch Time=17.1875 +GPU:2 | Epoch: 27 | loss=9.33873176574707 | Batch Time=22.265625 +GPU:2 | Epoch: 27 | loss=9.467257499694824 | Batch Time=20.3125 +GPU:2 | Epoch: 27 | loss=9.311969757080078 | Batch Time=23.046875 +GPU:2 | Epoch: 27 | loss=9.186431884765625 | Batch Time=21.09375 +GPU:2 | Epoch: 27 | loss=9.214885711669922 | Batch Time=26.5625 +GPU:2 | Epoch: 27 | loss=9.20612907409668 | Batch Time=19.53125 +GPU:2 | Epoch: 27 | loss=9.357992172241211 | Batch Time=17.578125 +GPU:2 | Epoch: 27 | loss=9.186162948608398 | Batch Time=22.265625 +GPU:2 | Epoch: 27 | loss=9.417814254760742 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.103389739990234 | Batch Time=24.609375 +GPU:2 | Epoch: 27 | loss=9.158906936645508 | Batch Time=25.78125 +GPU:2 | Epoch: 27 | loss=9.304695129394531 | Batch Time=18.359375 +GPU:2 | Epoch: 27 | loss=9.175954818725586 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.462679862976074 | Batch Time=16.796875 +GPU:2 | Epoch: 27 | loss=9.297157287597656 | Batch Time=21.09375 +GPU:2 | Epoch: 27 | loss=9.352151870727539 | Batch Time=18.359375 +GPU:2 | Epoch: 27 | loss=9.519584655761719 | Batch Time=17.578125 +GPU:2 | Epoch: 27 | loss=9.335461616516113 | Batch Time=16.796875 +GPU:2 | Epoch: 27 | loss=9.177136421203613 | Batch Time=22.65625 +GPU:2 | Epoch: 27 | loss=9.30813217163086 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.371089935302734 | Batch Time=21.09375 +GPU:2 | Epoch: 27 | loss=9.3158597946167 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.185976028442383 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.254314422607422 | Batch Time=19.53125 +GPU:2 | Epoch: 27 | loss=9.375432014465332 | Batch Time=17.578125 +GPU:2 | Epoch: 27 | loss=9.225737571716309 | Batch Time=17.1875 +GPU:2 | Epoch: 27 | loss=9.189977645874023 | Batch Time=23.046875 +GPU:2 | Epoch: 27 | loss=9.403971672058105 | Batch Time=15.234375 +GPU:2 | Epoch: 27 | loss=9.222672462463379 | Batch Time=19.921875 +GPU:2 | Epoch: 27 | loss=9.410968780517578 | Batch Time=16.796875 +GPU:2 | Epoch: 27 | loss=9.401859283447266 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.160219192504883 | Batch Time=21.875 +GPU:2 | Epoch: 27 | loss=9.480977058410645 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.225540161132812 | Batch Time=21.484375 +GPU:2 | Epoch: 27 | loss=9.286117553710938 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.224456787109375 | Batch Time=22.265625 +GPU:2 | Epoch: 27 | loss=9.277514457702637 | Batch Time=17.96875 +GPU:2 | Epoch: 27 | loss=9.346254348754883 | Batch Time=18.359375 +GPU:2 | Epoch: 27 | loss=9.506354331970215 | Batch Time=15.625 +GPU:2 | Epoch: 27 | loss=9.30485725402832 | Batch Time=21.484375 +GPU:2 | Epoch: 27 | loss=9.240171432495117 | Batch Time=19.53125 +GPU:2 | Epoch: 27 | loss=9.41606330871582 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.53532600402832 | Batch Time=17.1875 +GPU:2 | Epoch: 27 | loss=9.646848678588867 | Batch Time=14.84375 +GPU:2 | Epoch: 27 | loss=9.391887664794922 | Batch Time=22.65625 +GPU:2 | Epoch: 27 | loss=9.289186477661133 | Batch Time=21.875 +GPU:2 | Epoch: 27 | loss=9.323925971984863 | Batch Time=22.65625 +GPU:2 | Epoch: 27 | loss=9.34534740447998 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.368993759155273 | Batch Time=21.484375 +GPU:2 | Epoch: 27 | loss=9.328291893005371 | Batch Time=17.578125 +GPU:2 | Epoch: 27 | loss=9.145750045776367 | Batch Time=24.609375 +GPU:2 | Epoch: 27 | loss=9.426253318786621 | Batch Time=19.921875 +GPU:2 | Epoch: 27 | loss=9.309123992919922 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.171647071838379 | Batch Time=20.3125 +GPU:2 | Epoch: 27 | loss=9.16248607635498 | Batch Time=17.96875 +GPU:2 | Epoch: 27 | loss=9.4678955078125 | Batch Time=17.96875 +GPU:2 | Epoch: 27 | loss=9.2738037109375 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.220117568969727 | Batch Time=19.921875 +GPU:2 | Epoch: 27 | loss=9.287151336669922 | Batch Time=21.09375 +GPU:2 | Epoch: 27 | loss=9.196308135986328 | Batch Time=25.390625 +GPU:2 | Epoch: 27 | loss=9.294686317443848 | Batch Time=14.84375 +GPU:2 | Epoch: 27 | loss=9.146408081054688 | Batch Time=21.875 +GPU:2 | Epoch: 27 | loss=9.355738639831543 | Batch Time=23.4375 +GPU:2 | Epoch: 27 | loss=9.244928359985352 | Batch Time=17.578125 +GPU:2 | Epoch: 27 | loss=9.137629508972168 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.257766723632812 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.210858345031738 | Batch Time=20.3125 +GPU:2 | Epoch: 27 | loss=9.415510177612305 | Batch Time=18.359375 +GPU:2 | Epoch: 27 | loss=9.458176612854004 | Batch Time=17.1875 +GPU:2 | Epoch: 27 | loss=9.100147247314453 | Batch Time=21.875 +GPU:2 | Epoch: 27 | loss=9.253666877746582 | Batch Time=22.265625 +GPU:2 | Epoch: 27 | loss=9.159811019897461 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.24688720703125 | Batch Time=16.40625 +GPU:2 | Epoch: 27 | loss=9.35367202758789 | Batch Time=21.875 +GPU:2 | Epoch: 27 | loss=9.461821556091309 | Batch Time=19.921875 +GPU:2 | Epoch: 27 | loss=9.364293098449707 | Batch Time=22.65625 +GPU:2 | Epoch: 27 | loss=9.272723197937012 | Batch Time=17.96875 +GPU:2 | Epoch: 27 | loss=9.255026817321777 | Batch Time=19.53125 +GPU:2 | Epoch: 27 | loss=9.401348114013672 | Batch Time=16.015625 +GPU:2 | Epoch: 27 | loss=9.252700805664062 | Batch Time=22.265625 +GPU:2 | Epoch: 27 | loss=9.468254089355469 | Batch Time=17.1875 +GPU:2 | Epoch: 27 | loss=9.024215698242188 | Batch Time=21.875 +GPU:2 | Epoch: 27 | loss=9.030972480773926 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.244852066040039 | Batch Time=19.53125 +GPU:2 | Epoch: 27 | loss=9.20649242401123 | Batch Time=22.265625 +GPU:2 | Epoch: 27 | loss=9.129704475402832 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.130695343017578 | Batch Time=21.09375 +GPU:2 | Epoch: 27 | loss=9.30438232421875 | Batch Time=17.96875 +GPU:2 | Epoch: 27 | loss=9.339838027954102 | Batch Time=16.796875 +GPU:2 | Epoch: 27 | loss=9.224778175354004 | Batch Time=21.484375 +GPU:2 | Epoch: 27 | loss=9.352544784545898 | Batch Time=17.96875 +GPU:2 | Epoch: 27 | loss=9.234313011169434 | Batch Time=21.09375 +GPU:2 | Epoch: 27 | loss=9.230761528015137 | Batch Time=19.53125 +GPU:2 | Epoch: 27 | loss=9.098067283630371 | Batch Time=19.921875 +GPU:2 | Epoch: 27 | loss=9.336990356445312 | Batch Time=19.140625 +GPU:2 | Epoch: 27 | loss=9.17452621459961 | Batch Time=20.3125 +GPU:2 | Epoch: 27 | loss=9.156746864318848 | Batch Time=24.609375 +GPU:2 | Epoch: 27 | loss=9.328106880187988 | Batch Time=20.3125 +GPU:2 | Epoch: 27 | loss=9.137641906738281 | Batch Time=22.265625 +GPU:2 | Epoch: 27 | loss=9.287602424621582 | Batch Time=21.09375 +GPU:2 | Epoch: 27 | loss=9.172346115112305 | Batch Time=17.96875 +GPU:2 | Epoch: 27 | loss=9.241323471069336 | Batch Time=17.96875 +GPU:2 | Epoch: 27 | loss=9.529149055480957 | Batch Time=13.671875 +GPU:2 | Epoch: 27 | loss=9.199857711791992 | Batch Time=21.875 +GPU:2 | Epoch: 27 | loss=9.277153015136719 | Batch Time=25.0 +GPU:2 | Epoch: 27 | loss=9.35460090637207 | Batch Time=19.140625 +GPU:2 | Epoch: 27 | loss=9.132074356079102 | Batch Time=19.53125 +GPU:2 | Epoch: 27 | loss=9.414398193359375 | Batch Time=18.359375 +GPU:2 | Epoch: 27 | loss=9.269243240356445 | Batch Time=21.875 +GPU:2 | Epoch: 27 | loss=9.299386978149414 | Batch Time=21.484375 +GPU:2 | Epoch: 27 | loss=8.9683256149292 | Batch Time=27.34375 +GPU:2 | Epoch: 27 | loss=9.307107925415039 | Batch Time=17.1875 +GPU:2 | Epoch: 27 | loss=9.32933235168457 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.158012390136719 | Batch Time=18.75 +GPU:2 | Epoch: 27 | loss=9.390884399414062 | Batch Time=19.53125 +GPU:2 | Epoch: 27 | loss=9.451326370239258 | Batch Time=16.015625 +GPU:2 | Epoch: 27 | loss=9.481826782226562 | Batch Time=15.234375 +GPU:2 | Epoch: 27 | loss=9.153407096862793 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.145903587341309 | Batch Time=20.3125 +GPU:2 | Epoch: 27 | loss=9.31866455078125 | Batch Time=17.578125 +GPU:2 | Epoch: 27 | loss=9.28734016418457 | Batch Time=20.703125 +GPU:2 | Epoch: 27 | loss=9.323143005371094 | Batch Time=21.09375 +GPU:2 | Epoch: 27 | loss=9.195260047912598 | Batch Time=19.53125 +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=27500691456, percent=89.3, used=222256427008, free=16718983168, active=220228382720, inactive=16746115072, buffers=392712192, cached=18199961600, shared=5662793728, slab=1005043712) +AFTER TRAIN LOOP: Epoch 27 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +STARTING TRAINING: Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +BEFORE TRAIN LOOP: Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=27500691456, percent=89.3, used=222256427008, free=16718983168, active=220228382720, inactive=16746115072, buffers=392712192, cached=18199961600, shared=5662793728, slab=1005043712) +AFTER TRAIN LOOP: Epoch 27 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +STARTING TRAINING: Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +BEFORE TRAIN LOOP: Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=27500691456, percent=89.3, used=222256427008, free=16718983168, active=220228382720, inactive=16746115072, buffers=392712192, cached=18199961600, shared=5662793728, slab=1005043712) +AFTER TRAIN LOOP: Epoch 27 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +BEFORE VAL LOOP: GPU: 0 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +AFTER TRAIN LOOP: Epoch 27 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +STARTING TRAINING: Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +BEFORE TRAIN LOOP: Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=27501199360, percent=89.3, used=222256316416, free=16719298560, active=220227878912, inactive=16746180608, buffers=392712192, cached=18199756800, shared=5662474240, slab=1005010944) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 27 | loss=3.2145121097564697 | Batch Time=29.6875 +GPU:0 | Epoch: 27 | loss=4.2657551765441895 | Batch Time=14.0625 +GPU:0 | Epoch: 27 | loss=3.8739748001098633 | Batch Time=25.78125 +GPU:0 | Epoch: 27 | loss=3.9771933555603027 | Batch Time=9.375 +GPU:0 | Epoch: 27 | loss=4.424023628234863 | Batch Time=11.71875 +GPU:0 | Epoch: 27 | loss=4.2387566566467285 | Batch Time=12.109375 +GPU:0 | Epoch: 27 | loss=4.0739593505859375 | Batch Time=10.9375 +GPU:0 | Epoch: 27 | loss=4.957123279571533 | Batch Time=8.984375 +GPU:0 | Epoch: 27 | loss=4.623513698577881 | Batch Time=10.9375 +GPU:0 | Epoch: 27 | loss=5.191821575164795 | Batch Time=8.203125 +GPU:0 | Epoch: 27 | loss=5.062563896179199 | Batch Time=2.34375 +GPU:0 | Epoch: 27 | loss=4.240389823913574 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=4.963083744049072 | Batch Time=10.546875 +GPU:0 | Epoch: 27 | loss=4.029594421386719 | Batch Time=23.828125 +GPU:0 | Epoch: 27 | loss=4.5380539894104 | Batch Time=8.59375 +GPU:0 | Epoch: 27 | loss=4.549259662628174 | Batch Time=16.796875 +GPU:0 | Epoch: 27 | loss=4.020293712615967 | Batch Time=17.96875 +GPU:0 | Epoch: 27 | loss=2.955559015274048 | Batch Time=37.109375 +GPU:0 | Epoch: 27 | loss=3.7420551776885986 | Batch Time=24.609375 +GPU:0 | Epoch: 27 | loss=3.266084671020508 | Batch Time=30.078125 +Model top1 Accuracy: 20.082 +Acc before rounding: 20.082 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 27 | loss=3.2145121097564697 | Batch Time=29.6875 +GPU:0 | Epoch: 27 | loss=4.2657551765441895 | Batch Time=14.0625 +GPU:0 | Epoch: 27 | loss=3.8739748001098633 | Batch Time=25.78125 +GPU:0 | Epoch: 27 | loss=3.9771933555603027 | Batch Time=9.375 +GPU:0 | Epoch: 27 | loss=4.424023628234863 | Batch Time=11.71875 +GPU:0 | Epoch: 27 | loss=4.2387566566467285 | Batch Time=12.109375 +GPU:0 | Epoch: 27 | loss=4.0739593505859375 | Batch Time=10.9375 +GPU:0 | Epoch: 27 | loss=4.957123279571533 | Batch Time=8.984375 +GPU:0 | Epoch: 27 | loss=4.623513698577881 | Batch Time=10.9375 +GPU:0 | Epoch: 27 | loss=5.191821575164795 | Batch Time=8.203125 +GPU:0 | Epoch: 27 | loss=5.062563896179199 | Batch Time=2.34375 +GPU:0 | Epoch: 27 | loss=4.240389823913574 | Batch Time=21.484375 +GPU:0 | Epoch: 27 | loss=4.963083744049072 | Batch Time=10.546875 +GPU:0 | Epoch: 27 | loss=4.029594421386719 | Batch Time=23.828125 +GPU:0 | Epoch: 27 | loss=4.5380539894104 | Batch Time=8.59375 +GPU:0 | Epoch: 27 | loss=4.549259662628174 | Batch Time=16.796875 +GPU:0 | Epoch: 27 | loss=4.020293712615967 | Batch Time=17.96875 +GPU:0 | Epoch: 27 | loss=2.955559015274048 | Batch Time=37.109375 +GPU:0 | Epoch: 27 | loss=3.7420551776885986 | Batch Time=24.609375 +GPU:0 | Epoch: 27 | loss=3.266084671020508 | Batch Time=30.078125 +Model top1 Accuracy: 20.082 +Acc after rounding: 20.082 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 27 | loss=4.11008358001709 | Batch Time=18.359375 +GPU:0 | Epoch: 27 | loss=4.467299461364746 | Batch Time=16.796875 +GPU:0 | Epoch: 27 | loss=4.1223883628845215 | Batch Time=19.53125 +GPU:0 | Epoch: 27 | loss=4.348942756652832 | Batch Time=15.625 +Model top1 Accuracy: 17.35 +Validation Acc after rounding: 17.35 +AFTER VAL LOOP: GPU: 0 | Epoch 27 | Memory Usage: svmem(total=257568083968, available=9753231360, percent=96.2, used=240056147968, free=5453058048, active=241510174720, inactive=6734835712, buffers=354791424, cached=11704086528, shared=5662941184, slab=905678848) +Rounding model with scheme: naive +Model avg sparsity: 35.10345906328209 +GPU:0 | Epoch: 27 | Acc=20.082 | Epoch Time=23.97570999463399 +Writing results into: results/results_pruning_ImageNet_ResNet50_hc_iter_0_5_5_reg_L2_1e-06_sgd_cosine_lr_0_4_0_1_50_finetune_0_04_MAML_-1_10_fan_False_signed_constant_unif_width_1_0_seed_42_idx_None/acc_and_sparsity.csv +AFTER TRAINING: Epoch 27 | Memory Usage: svmem(total=257568083968, available=9753223168, percent=96.2, used=240055910400, free=5450985472, active=241510678528, inactive=6736384000, buffers=355033088, cached=11706155008, shared=5662941184, slab=905678848) +STARTING TRAINING: Epoch 28 | Memory Usage: svmem(total=257568083968, available=9753223168, percent=96.2, used=240055910400, free=5450985472, active=241510678528, inactive=6736384000, buffers=355033088, cached=11706155008, shared=5662941184, slab=905678848) +BEFORE TRAIN LOOP: Epoch 28 | Memory Usage: svmem(total=257568083968, available=9753223168, percent=96.2, used=240055910400, free=5450985472, active=241510678528, inactive=6736384000, buffers=355033088, cached=11706155008, shared=5662941184, slab=905678848) +(TRAINER)BEFORE TRAIN LOOP: GPU:0 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=9753223168, percent=96.2, used=240055910400, free=5450985472, active=241510678528, inactive=6736384000, buffers=355033088, cached=11706155008, shared=5662941184, slab=905678848) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:1 | Epoch: 28 | loss=9.37633228302002 | Batch Time=16.796875 +GPU:1 | Epoch: 28 | loss=9.390254974365234 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.285062789916992 | Batch Time=21.484375 +GPU:1 | Epoch: 28 | loss=9.285521507263184 | Batch Time=18.75 +GPU:1 | Epoch: 28 | loss=9.255504608154297 | Batch Time=23.4375 +GPU:1 | Epoch: 28 | loss=9.07967472076416 | Batch Time=21.484375 +GPU:1 | Epoch: 28 | loss=9.276369094848633 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.22317123413086 | Batch Time=20.703125 +GPU:1 | Epoch: 28 | loss=9.509305953979492 | Batch Time=14.453125 +GPU:1 | Epoch: 28 | loss=8.979093551635742 | Batch Time=22.65625 +GPU:1 | Epoch: 28 | loss=9.061234474182129 | Batch Time=23.4375 +GPU:1 | Epoch: 28 | loss=9.162039756774902 | Batch Time=24.21875 +GPU:1 | Epoch: 28 | loss=9.129423141479492 | Batch Time=22.265625 +GPU:1 | Epoch: 28 | loss=9.374411582946777 | Batch Time=20.3125 +GPU:1 | Epoch: 28 | loss=8.987927436828613 | Batch Time=23.4375 +GPU:1 | Epoch: 28 | loss=9.399099349975586 | Batch Time=14.84375 +GPU:1 | Epoch: 28 | loss=9.171043395996094 | Batch Time=21.09375 +GPU:1 | Epoch: 28 | loss=9.271373748779297 | Batch Time=19.140625 +GPU:1 | Epoch: 28 | loss=9.266366004943848 | Batch Time=17.578125 +GPU:1 | Epoch: 28 | loss=9.238758087158203 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.033719062805176 | Batch Time=19.921875 +GPU:1 | Epoch: 28 | loss=9.097561836242676 | Batch Time=22.265625 +GPU:1 | Epoch: 28 | loss=9.268423080444336 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.162361145019531 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.065204620361328 | Batch Time=26.953125 +GPU:1 | Epoch: 28 | loss=9.165624618530273 | Batch Time=21.484375 +GPU:1 | Epoch: 28 | loss=9.124959945678711 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.113943099975586 | Batch Time=22.265625 +GPU:1 | Epoch: 28 | loss=9.274537086486816 | Batch Time=21.09375 +GPU:1 | Epoch: 28 | loss=9.136724472045898 | Batch Time=17.96875 +GPU:1 | Epoch: 28 | loss=9.146444320678711 | Batch Time=18.75 +GPU:1 | Epoch: 28 | loss=9.104223251342773 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.02303409576416 | Batch Time=24.609375 +GPU:1 | Epoch: 28 | loss=9.115693092346191 | Batch Time=23.828125 +GPU:1 | Epoch: 28 | loss=9.442475318908691 | Batch Time=14.84375 +GPU:1 | Epoch: 28 | loss=9.248445510864258 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.038764953613281 | Batch Time=20.703125 +GPU:1 | Epoch: 28 | loss=9.155017852783203 | Batch Time=17.96875 +GPU:1 | Epoch: 28 | loss=9.26647663116455 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.245917320251465 | Batch Time=21.09375 +GPU:1 | Epoch: 28 | loss=9.179883003234863 | Batch Time=23.4375 +GPU:1 | Epoch: 28 | loss=9.22554874420166 | Batch Time=19.921875 +GPU:1 | Epoch: 28 | loss=9.262284278869629 | Batch Time=18.75 +GPU:1 | Epoch: 28 | loss=9.27520751953125 | Batch Time=19.140625 +GPU:1 | Epoch: 28 | loss=9.168824195861816 | Batch Time=21.484375 +GPU:1 | Epoch: 28 | loss=9.093426704406738 | Batch Time=24.21875 +GPU:1 | Epoch: 28 | loss=9.09139633178711 | Batch Time=19.140625 +GPU:1 | Epoch: 28 | loss=9.213196754455566 | Batch Time=21.875 +GPU:1 | Epoch: 28 | loss=9.189037322998047 | Batch Time=21.875 +GPU:1 | Epoch: 28 | loss=9.203550338745117 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.141866683959961 | Batch Time=22.65625 +GPU:1 | Epoch: 28 | loss=9.235321044921875 | Batch Time=17.96875 +GPU:1 | Epoch: 28 | loss=9.097541809082031 | Batch Time=19.921875 +GPU:1 | Epoch: 28 | loss=9.372864723205566 | Batch Time=18.75 +GPU:1 | Epoch: 28 | loss=9.28736686706543 | Batch Time=23.4375 +GPU:1 | Epoch: 28 | loss=9.225522994995117 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.323097229003906 | Batch Time=17.96875 +GPU:1 | Epoch: 28 | loss=9.27322769165039 | Batch Time=19.140625 +GPU:1 | Epoch: 28 | loss=9.28319263458252 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.237454414367676 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.444598197937012 | Batch Time=19.921875 +GPU:1 | Epoch: 28 | loss=9.198486328125 | Batch Time=23.046875 +GPU:1 | Epoch: 28 | loss=8.899834632873535 | Batch Time=25.78125 +GPU:1 | Epoch: 28 | loss=9.053791046142578 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.02253532409668 | Batch Time=22.265625 +GPU:1 | Epoch: 28 | loss=8.803079605102539 | Batch Time=28.515625 +GPU:1 | Epoch: 28 | loss=9.169946670532227 | Batch Time=19.921875 +GPU:1 | Epoch: 28 | loss=9.266613006591797 | Batch Time=19.140625 +GPU:1 | Epoch: 28 | loss=9.264800071716309 | Batch Time=23.828125 +GPU:1 | Epoch: 28 | loss=9.230621337890625 | Batch Time=17.578125 +GPU:1 | Epoch: 28 | loss=9.071898460388184 | Batch Time=21.875 +GPU:1 | Epoch: 28 | loss=9.157273292541504 | Batch Time=19.921875 +GPU:1 | Epoch: 28 | loss=9.253320693969727 | Batch Time=17.1875 +GPU:1 | Epoch: 28 | loss=9.20114517211914 | Batch Time=17.578125 +GPU:1 | Epoch: 28 | loss=9.262163162231445 | Batch Time=19.921875 +GPU:1 | Epoch: 28 | loss=9.213590621948242 | Batch Time=22.265625 +GPU:1 | Epoch: 28 | loss=9.128725051879883 | Batch Time=17.96875 +GPU:1 | Epoch: 28 | loss=8.948431015014648 | Batch Time=23.046875 +GPU:1 | Epoch: 28 | loss=9.405448913574219 | Batch Time=14.453125 +GPU:1 | Epoch: 28 | loss=9.017047882080078 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.330831527709961 | Batch Time=21.09375 +GPU:1 | Epoch: 28 | loss=9.111822128295898 | Batch Time=23.046875 +GPU:1 | Epoch: 28 | loss=9.207876205444336 | Batch Time=15.625 +GPU:1 | Epoch: 28 | loss=9.390477180480957 | Batch Time=16.40625 +GPU:1 | Epoch: 28 | loss=9.228073120117188 | Batch Time=22.265625 +GPU:1 | Epoch: 28 | loss=9.01278305053711 | Batch Time=23.046875 +GPU:1 | Epoch: 28 | loss=9.102975845336914 | Batch Time=21.484375 +GPU:1 | Epoch: 28 | loss=9.318907737731934 | Batch Time=19.140625 +GPU:1 | Epoch: 28 | loss=9.260025978088379 | Batch Time=18.75 +GPU:1 | Epoch: 28 | loss=9.16950798034668 | Batch Time=21.484375 +GPU:1 | Epoch: 28 | loss=8.960038185119629 | Batch Time=22.265625 +GPU:1 | Epoch: 28 | loss=9.21550178527832 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.048816680908203 | Batch Time=21.875 +GPU:1 | Epoch: 28 | loss=9.33156681060791 | Batch Time=20.3125 +GPU:1 | Epoch: 28 | loss=9.260717391967773 | Batch Time=17.578125 +GPU:1 | Epoch: 28 | loss=9.158697128295898 | Batch Time=19.140625 +GPU:1 | Epoch: 28 | loss=9.238336563110352 | Batch Time=17.96875 +GPU:1 | Epoch: 28 | loss=9.209089279174805 | Batch Time=19.53125 +GPU:1 | Epoch: 28 | loss=9.059328079223633 | Batch Time=26.953125 +GPU:1 | Epoch: 28 | loss=9.088920593261719 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.07253646850586 | Batch Time=23.828125 +GPU:1 | Epoch: 28 | loss=9.016417503356934 | Batch Time=18.75 +GPU:1 | Epoch: 28 | loss=9.304730415344238 | Batch Time=20.3125 +GPU:1 | Epoch: 28 | loss=8.99719524383545 | Batch Time=22.65625 +GPU:1 | Epoch: 28 | loss=9.022823333740234 | Batch Time=20.3125 +GPU:1 | Epoch: 28 | loss=9.012369155883789 | Batch Time=23.4375 +GPU:1 | Epoch: 28 | loss=9.18919563293457 | Batch Time=18.75 +GPU:1 | Epoch: 28 | loss=9.115127563476562 | Batch Time=21.09375 +GPU:1 | Epoch: 28 | loss=8.909443855285645 | Batch Time=24.609375 +GPU:1 | Epoch: 28 | loss=9.267890930175781 | Batch Time=15.234375 +GPU:1 | Epoch: 28 | loss=9.154071807861328 | Batch Time=18.75 +GPU:1 | Epoch: 28 | loss=9.215564727783203 | Batch Time=22.265625 +GPU:1 | Epoch: 28 | loss=9.14788818359375 | Batch Time=21.09375 +GPU:1 | Epoch: 28 | loss=8.948385238647461 | Batch Time=25.78125 +GPU:1 | Epoch: 28 | loss=9.115457534790039 | Batch Time=21.875 +GPU:1 | Epoch: 28 | loss=9.167200088500977 | Batch Time=21.875 +GPU:1 | Epoch: 28 | loss=9.087018013000488 | Batch Time=20.703125 +GPU:1 | Epoch: 28 | loss=9.100634574890137 | Batch Time=21.875 +GPU:1 | Epoch: 28 | loss=9.0217866897583 | Batch Time=22.65625 +GPU:1 | Epoch: 28 | loss=8.989118576049805 | Batch Time=19.921875 +GPU:1 | Epoch: 28 | loss=8.930525779724121 | Batch Time=24.609375 +GPU:1 | Epoch: 28 | loss=9.215283393859863 | Batch Time=19.140625 +GPU:1 | Epoch: 28 | loss=9.40475845336914 | Batch Time=18.359375 +GPU:1 | Epoch: 28 | loss=9.019304275512695 | Batch Time=20.3125 +GPU:1 | Epoch: 28 | loss=9.087602615356445 | Batch Time=23.046875 +GPU:2 | Epoch: 28 | loss=9.147744178771973 | Batch Time=24.609375 +GPU:2 | Epoch: 28 | loss=9.285869598388672 | Batch Time=21.484375 +GPU:2 | Epoch: 28 | loss=9.26122760772705 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=9.118045806884766 | Batch Time=21.875 +GPU:2 | Epoch: 28 | loss=9.046092987060547 | Batch Time=20.3125 +GPU:2 | Epoch: 28 | loss=9.178083419799805 | Batch Time=22.265625 +GPU:2 | Epoch: 28 | loss=9.137198448181152 | Batch Time=21.875 +GPU:2 | Epoch: 28 | loss=9.300146102905273 | Batch Time=16.015625 +GPU:2 | Epoch: 28 | loss=9.244133949279785 | Batch Time=16.015625 +GPU:2 | Epoch: 28 | loss=9.193458557128906 | Batch Time=18.75 +GPU:2 | Epoch: 28 | loss=9.093032836914062 | Batch Time=23.4375 +GPU:2 | Epoch: 28 | loss=9.314141273498535 | Batch Time=15.234375 +GPU:2 | Epoch: 28 | loss=9.125558853149414 | Batch Time=20.3125 +GPU:2 | Epoch: 28 | loss=9.295829772949219 | Batch Time=19.921875 +GPU:2 | Epoch: 28 | loss=9.082452774047852 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=8.891792297363281 | Batch Time=25.0 +GPU:2 | Epoch: 28 | loss=9.342348098754883 | Batch Time=23.828125 +GPU:2 | Epoch: 28 | loss=8.976436614990234 | Batch Time=21.484375 +GPU:2 | Epoch: 28 | loss=9.236112594604492 | Batch Time=17.578125 +GPU:2 | Epoch: 28 | loss=9.284379959106445 | Batch Time=17.1875 +GPU:2 | Epoch: 28 | loss=9.274215698242188 | Batch Time=21.484375 +GPU:2 | Epoch: 28 | loss=9.305033683776855 | Batch Time=19.140625 +GPU:2 | Epoch: 28 | loss=9.186260223388672 | Batch Time=18.75 +GPU:2 | Epoch: 28 | loss=9.10493278503418 | Batch Time=25.78125 +GPU:2 | Epoch: 28 | loss=9.175664901733398 | Batch Time=16.40625 +GPU:2 | Epoch: 28 | loss=9.18848705291748 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.266342163085938 | Batch Time=19.53125 +GPU:2 | Epoch: 28 | loss=9.433393478393555 | Batch Time=17.578125 +GPU:2 | Epoch: 28 | loss=9.163453102111816 | Batch Time=22.65625 +GPU:2 | Epoch: 28 | loss=9.374004364013672 | Batch Time=19.140625 +GPU:2 | Epoch: 28 | loss=9.28802490234375 | Batch Time=19.140625 +GPU:2 | Epoch: 28 | loss=9.062569618225098 | Batch Time=24.21875 +GPU:2 | Epoch: 28 | loss=9.1170654296875 | Batch Time=23.4375 +GPU:2 | Epoch: 28 | loss=9.0852689743042 | Batch Time=19.140625 +GPU:2 | Epoch: 28 | loss=9.126615524291992 | Batch Time=21.875 +GPU:2 | Epoch: 28 | loss=9.454384803771973 | Batch Time=18.359375 +GPU:2 | Epoch: 28 | loss=9.046233177185059 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.175130844116211 | Batch Time=22.65625 +GPU:2 | Epoch: 28 | loss=9.204912185668945 | Batch Time=16.40625 +GPU:2 | Epoch: 28 | loss=9.20333480834961 | Batch Time=17.1875 +GPU:2 | Epoch: 28 | loss=9.285070419311523 | Batch Time=18.359375 +GPU:2 | Epoch: 28 | loss=9.279611587524414 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.333600044250488 | Batch Time=16.796875 +GPU:2 | Epoch: 28 | loss=9.068708419799805 | Batch Time=19.53125 +GPU:2 | Epoch: 28 | loss=9.319812774658203 | Batch Time=18.75 +GPU:2 | Epoch: 28 | loss=9.189621925354004 | Batch Time=17.96875 +GPU:2 | Epoch: 28 | loss=9.261224746704102 | Batch Time=19.140625 +GPU:2 | Epoch: 28 | loss=8.969376564025879 | Batch Time=23.828125 +GPU:2 | Epoch: 28 | loss=9.330642700195312 | Batch Time=16.015625 +GPU:2 | Epoch: 28 | loss=9.217695236206055 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=8.938371658325195 | Batch Time=22.265625 +GPU:2 | Epoch: 28 | loss=8.938810348510742 | Batch Time=24.21875 +GPU:2 | Epoch: 28 | loss=9.234400749206543 | Batch Time=19.53125 +GPU:2 | Epoch: 28 | loss=9.238496780395508 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.086814880371094 | Batch Time=20.3125 +GPU:2 | Epoch: 28 | loss=9.309683799743652 | Batch Time=17.96875 +GPU:2 | Epoch: 28 | loss=9.1007661819458 | Batch Time=21.875 +GPU:2 | Epoch: 28 | loss=9.305326461791992 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=9.423293113708496 | Batch Time=19.53125 +GPU:2 | Epoch: 28 | loss=9.217906951904297 | Batch Time=21.875 +GPU:2 | Epoch: 28 | loss=8.996012687683105 | Batch Time=27.34375 +GPU:2 | Epoch: 28 | loss=9.389049530029297 | Batch Time=17.96875 +GPU:2 | Epoch: 28 | loss=9.125040054321289 | Batch Time=20.3125 +GPU:2 | Epoch: 28 | loss=9.22065544128418 | Batch Time=21.484375 +GPU:2 | Epoch: 28 | loss=9.199617385864258 | Batch Time=19.140625 +GPU:2 | Epoch: 28 | loss=9.264616012573242 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.140629768371582 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=9.103202819824219 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.066393852233887 | Batch Time=21.875 +GPU:2 | Epoch: 28 | loss=9.320680618286133 | Batch Time=17.1875 +GPU:2 | Epoch: 28 | loss=9.151361465454102 | Batch Time=23.828125 +GPU:2 | Epoch: 28 | loss=9.186007499694824 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.108566284179688 | Batch Time=17.1875 +GPU:2 | Epoch: 28 | loss=9.193465232849121 | Batch Time=19.921875 +GPU:2 | Epoch: 28 | loss=9.087738037109375 | Batch Time=23.046875 +GPU:2 | Epoch: 28 | loss=9.167786598205566 | Batch Time=23.828125 +GPU:2 | Epoch: 28 | loss=9.196283340454102 | Batch Time=17.1875 +GPU:2 | Epoch: 28 | loss=9.086302757263184 | Batch Time=21.484375 +GPU:2 | Epoch: 28 | loss=9.144030570983887 | Batch Time=21.875 +GPU:2 | Epoch: 28 | loss=9.236701011657715 | Batch Time=24.609375 +GPU:2 | Epoch: 28 | loss=9.072513580322266 | Batch Time=16.796875 +GPU:2 | Epoch: 28 | loss=9.234478950500488 | Batch Time=19.53125 +GPU:2 | Epoch: 28 | loss=8.99648666381836 | Batch Time=21.875 +GPU:2 | Epoch: 28 | loss=9.238255500793457 | Batch Time=17.1875 +GPU:2 | Epoch: 28 | loss=9.137680053710938 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=9.257957458496094 | Batch Time=18.359375 +GPU:2 | Epoch: 28 | loss=9.239583969116211 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=9.149946212768555 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.115962982177734 | Batch Time=22.65625 +GPU:2 | Epoch: 28 | loss=9.161226272583008 | Batch Time=23.4375 +GPU:2 | Epoch: 28 | loss=9.184646606445312 | Batch Time=22.65625 +GPU:2 | Epoch: 28 | loss=9.121610641479492 | Batch Time=19.921875 +GPU:2 | Epoch: 28 | loss=8.989480018615723 | Batch Time=25.78125 +GPU:2 | Epoch: 28 | loss=9.070006370544434 | Batch Time=19.53125 +GPU:2 | Epoch: 28 | loss=8.967811584472656 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=9.356589317321777 | Batch Time=17.1875 +GPU:2 | Epoch: 28 | loss=9.14337158203125 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.079456329345703 | Batch Time=24.21875 +GPU:2 | Epoch: 28 | loss=9.27374267578125 | Batch Time=17.96875 +GPU:2 | Epoch: 28 | loss=9.028675079345703 | Batch Time=23.046875 +GPU:2 | Epoch: 28 | loss=9.490835189819336 | Batch Time=16.40625 +GPU:2 | Epoch: 28 | loss=9.183448791503906 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=9.232419967651367 | Batch Time=17.1875 +GPU:2 | Epoch: 28 | loss=9.047469139099121 | Batch Time=23.046875 +GPU:2 | Epoch: 28 | loss=8.926926612854004 | Batch Time=23.046875 +GPU:2 | Epoch: 28 | loss=9.057939529418945 | Batch Time=24.21875 +GPU:2 | Epoch: 28 | loss=9.23054027557373 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.067344665527344 | Batch Time=20.703125 +GPU:2 | Epoch: 28 | loss=9.128461837768555 | Batch Time=17.578125 +GPU:2 | Epoch: 28 | loss=9.08145523071289 | Batch Time=23.046875 +GPU:2 | Epoch: 28 | loss=9.13315486907959 | Batch Time=22.265625 +GPU:2 | Epoch: 28 | loss=9.16619873046875 | Batch Time=22.65625 +GPU:2 | Epoch: 28 | loss=9.205490112304688 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.272748947143555 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.024181365966797 | Batch Time=21.09375 +GPU:2 | Epoch: 28 | loss=9.116100311279297 | Batch Time=19.140625 +GPU:2 | Epoch: 28 | loss=9.0762939453125 | Batch Time=21.484375 +GPU:2 | Epoch: 28 | loss=9.370264053344727 | Batch Time=19.140625 +GPU:2 | Epoch: 28 | loss=9.237607955932617 | Batch Time=20.3125 +GPU:2 | Epoch: 28 | loss=9.187817573547363 | Batch Time=21.484375 +GPU:2 | Epoch: 28 | loss=8.873053550720215 | Batch Time=26.953125 +GPU:2 | Epoch: 28 | loss=8.998235702514648 | Batch Time=24.609375 +GPU:2 | Epoch: 28 | loss=9.083850860595703 | Batch Time=21.484375 +GPU:2 | Epoch: 28 | loss=8.955276489257812 | Batch Time=20.3125 +GPU:2 | Epoch: 28 | loss=9.219610214233398 | Batch Time=17.96875 +GPU:3 | Epoch: 28 | loss=9.190523147583008 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.271642684936523 | Batch Time=20.3125 +GPU:3 | Epoch: 28 | loss=9.147735595703125 | Batch Time=24.609375 +GPU:3 | Epoch: 28 | loss=9.228687286376953 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=9.084634780883789 | Batch Time=22.265625 +GPU:3 | Epoch: 28 | loss=9.182462692260742 | Batch Time=17.96875 +GPU:3 | Epoch: 28 | loss=9.147550582885742 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.260735511779785 | Batch Time=19.921875 +GPU:3 | Epoch: 28 | loss=9.133325576782227 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=9.264276504516602 | Batch Time=17.1875 +GPU:3 | Epoch: 28 | loss=9.079405784606934 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.229232788085938 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.137489318847656 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=9.298489570617676 | Batch Time=18.75 +GPU:3 | Epoch: 28 | loss=9.108686447143555 | Batch Time=23.046875 +GPU:3 | Epoch: 28 | loss=9.28093433380127 | Batch Time=16.796875 +GPU:3 | Epoch: 28 | loss=9.238625526428223 | Batch Time=21.484375 +GPU:3 | Epoch: 28 | loss=9.133039474487305 | Batch Time=20.3125 +GPU:3 | Epoch: 28 | loss=9.1605863571167 | Batch Time=21.09375 +GPU:3 | Epoch: 28 | loss=9.119585990905762 | Batch Time=23.828125 +GPU:3 | Epoch: 28 | loss=9.395082473754883 | Batch Time=20.3125 +GPU:3 | Epoch: 28 | loss=9.206831932067871 | Batch Time=21.09375 +GPU:3 | Epoch: 28 | loss=9.170665740966797 | Batch Time=22.65625 +GPU:3 | Epoch: 28 | loss=9.35687255859375 | Batch Time=20.3125 +GPU:3 | Epoch: 28 | loss=9.528422355651855 | Batch Time=14.0625 +GPU:3 | Epoch: 28 | loss=9.174247741699219 | Batch Time=22.265625 +GPU:3 | Epoch: 28 | loss=8.893487930297852 | Batch Time=23.046875 +GPU:3 | Epoch: 28 | loss=9.178220748901367 | Batch Time=22.265625 +GPU:3 | Epoch: 28 | loss=9.18272876739502 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.33215045928955 | Batch Time=18.359375 +GPU:3 | Epoch: 28 | loss=9.032854080200195 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=9.315449714660645 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=9.201860427856445 | Batch Time=21.484375 +GPU:3 | Epoch: 28 | loss=9.201750755310059 | Batch Time=23.046875 +GPU:3 | Epoch: 28 | loss=9.268942832946777 | Batch Time=16.40625 +GPU:3 | Epoch: 28 | loss=9.178123474121094 | Batch Time=23.4375 +GPU:3 | Epoch: 28 | loss=9.204339981079102 | Batch Time=22.265625 +GPU:3 | Epoch: 28 | loss=9.277341842651367 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.271875381469727 | Batch Time=21.09375 +GPU:3 | Epoch: 28 | loss=9.130247116088867 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.355772018432617 | Batch Time=21.09375 +GPU:3 | Epoch: 28 | loss=9.074278831481934 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=9.388090133666992 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=8.957778930664062 | Batch Time=25.390625 +GPU:3 | Epoch: 28 | loss=9.277816772460938 | Batch Time=15.625 +GPU:3 | Epoch: 28 | loss=9.29963493347168 | Batch Time=20.3125 +GPU:3 | Epoch: 28 | loss=9.32000732421875 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.158611297607422 | Batch Time=23.828125 +GPU:3 | Epoch: 28 | loss=9.228435516357422 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.019198417663574 | Batch Time=17.578125 +GPU:3 | Epoch: 28 | loss=9.030969619750977 | Batch Time=23.828125 +GPU:3 | Epoch: 28 | loss=9.136844635009766 | Batch Time=19.921875 +GPU:3 | Epoch: 28 | loss=9.22453498840332 | Batch Time=18.359375 +GPU:3 | Epoch: 28 | loss=9.216278076171875 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=9.26456069946289 | Batch Time=15.625 +GPU:3 | Epoch: 28 | loss=9.210935592651367 | Batch Time=22.265625 +GPU:3 | Epoch: 28 | loss=9.27011775970459 | Batch Time=17.1875 +GPU:3 | Epoch: 28 | loss=9.366989135742188 | Batch Time=16.796875 +GPU:3 | Epoch: 28 | loss=9.160726547241211 | Batch Time=21.484375 +GPU:3 | Epoch: 28 | loss=9.265296936035156 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=8.902840614318848 | Batch Time=26.5625 +GPU:3 | Epoch: 28 | loss=9.117298126220703 | Batch Time=21.875 +GPU:3 | Epoch: 28 | loss=9.104923248291016 | Batch Time=22.65625 +GPU:3 | Epoch: 28 | loss=9.033175468444824 | Batch Time=21.875 +GPU:3 | Epoch: 28 | loss=9.2421875 | Batch Time=17.578125 +GPU:3 | Epoch: 28 | loss=9.207619667053223 | Batch Time=21.875 +GPU:3 | Epoch: 28 | loss=9.172235488891602 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.265920639038086 | Batch Time=20.3125 +GPU:3 | Epoch: 28 | loss=9.277101516723633 | Batch Time=20.3125 +GPU:3 | Epoch: 28 | loss=9.054389953613281 | Batch Time=23.828125 +GPU:3 | Epoch: 28 | loss=9.15243911743164 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.175451278686523 | Batch Time=24.21875 +GPU:3 | Epoch: 28 | loss=9.162943840026855 | Batch Time=20.3125 +GPU:3 | Epoch: 28 | loss=9.245428085327148 | Batch Time=19.140625 +GPU:3 | Epoch: 28 | loss=9.15813159942627 | Batch Time=22.65625 +GPU:3 | Epoch: 28 | loss=9.170650482177734 | Batch Time=21.484375 +GPU:3 | Epoch: 28 | loss=9.151480674743652 | Batch Time=21.875 +GPU:3 | Epoch: 28 | loss=9.156405448913574 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.16370964050293 | Batch Time=19.921875 +GPU:3 | Epoch: 28 | loss=9.287245750427246 | Batch Time=19.140625 +GPU:3 | Epoch: 28 | loss=9.016471862792969 | Batch Time=22.265625 +GPU:3 | Epoch: 28 | loss=9.141653060913086 | Batch Time=22.65625 +GPU:3 | Epoch: 28 | loss=9.142206192016602 | Batch Time=24.609375 +GPU:3 | Epoch: 28 | loss=9.212782859802246 | Batch Time=19.921875 +GPU:3 | Epoch: 28 | loss=9.081097602844238 | Batch Time=23.828125 +GPU:3 | Epoch: 28 | loss=9.269633293151855 | Batch Time=18.75 +GPU:3 | Epoch: 28 | loss=9.123937606811523 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.046165466308594 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.104212760925293 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.045976638793945 | Batch Time=21.09375 +GPU:3 | Epoch: 28 | loss=8.991999626159668 | Batch Time=23.4375 +GPU:3 | Epoch: 28 | loss=9.19980239868164 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=9.135825157165527 | Batch Time=19.140625 +GPU:3 | Epoch: 28 | loss=8.883064270019531 | Batch Time=25.78125 +GPU:3 | Epoch: 28 | loss=9.29869270324707 | Batch Time=16.015625 +GPU:3 | Epoch: 28 | loss=9.195837020874023 | Batch Time=22.65625 +GPU:3 | Epoch: 28 | loss=9.017155647277832 | Batch Time=23.828125 +GPU:3 | Epoch: 28 | loss=9.206886291503906 | Batch Time=19.921875 +GPU:3 | Epoch: 28 | loss=9.087932586669922 | Batch Time=22.265625 +GPU:3 | Epoch: 28 | loss=9.384754180908203 | Batch Time=17.578125 +GPU:3 | Epoch: 28 | loss=9.164653778076172 | Batch Time=16.40625 +GPU:3 | Epoch: 28 | loss=8.908349990844727 | Batch Time=26.171875 +GPU:3 | Epoch: 28 | loss=9.218460083007812 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.007308959960938 | Batch Time=25.78125 +GPU:3 | Epoch: 28 | loss=9.081144332885742 | Batch Time=21.875 +GPU:3 | Epoch: 28 | loss=9.119306564331055 | Batch Time=22.65625 +GPU:3 | Epoch: 28 | loss=9.205153465270996 | Batch Time=19.140625 +GPU:3 | Epoch: 28 | loss=9.117805480957031 | Batch Time=17.96875 +GPU:3 | Epoch: 28 | loss=8.993517875671387 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=9.330272674560547 | Batch Time=19.921875 +GPU:3 | Epoch: 28 | loss=9.296442985534668 | Batch Time=17.96875 +GPU:3 | Epoch: 28 | loss=9.039752960205078 | Batch Time=19.140625 +GPU:3 | Epoch: 28 | loss=9.054854393005371 | Batch Time=19.53125 +GPU:3 | Epoch: 28 | loss=8.918821334838867 | Batch Time=24.21875 +GPU:3 | Epoch: 28 | loss=9.089433670043945 | Batch Time=21.484375 +GPU:3 | Epoch: 28 | loss=9.19744873046875 | Batch Time=19.140625 +GPU:3 | Epoch: 28 | loss=9.2533540725708 | Batch Time=19.140625 +GPU:3 | Epoch: 28 | loss=9.172658920288086 | Batch Time=17.1875 +GPU:3 | Epoch: 28 | loss=9.248418807983398 | Batch Time=21.09375 +GPU:3 | Epoch: 28 | loss=9.259355545043945 | Batch Time=21.484375 +GPU:3 | Epoch: 28 | loss=9.304935455322266 | Batch Time=16.40625 +GPU:3 | Epoch: 28 | loss=9.050966262817383 | Batch Time=21.875 +GPU:3 | Epoch: 28 | loss=9.08736515045166 | Batch Time=21.875 +GPU:3 | Epoch: 28 | loss=9.272835731506348 | Batch Time=20.703125 +GPU:3 | Epoch: 28 | loss=8.821601867675781 | Batch Time=22.65625 +GPU:0 | Epoch: 28 | loss=9.289709091186523 | Batch Time=14.453125 +GPU:0 | Epoch: 28 | loss=9.250465393066406 | Batch Time=17.1875 +GPU:0 | Epoch: 28 | loss=9.193807601928711 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=8.842240333557129 | Batch Time=24.21875 +GPU:0 | Epoch: 28 | loss=9.354107856750488 | Batch Time=18.359375 +GPU:0 | Epoch: 28 | loss=9.069229125976562 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=9.157108306884766 | Batch Time=18.75 +GPU:0 | Epoch: 28 | loss=9.200332641601562 | Batch Time=21.875 +GPU:0 | Epoch: 28 | loss=9.257980346679688 | Batch Time=22.65625 +GPU:0 | Epoch: 28 | loss=9.202792167663574 | Batch Time=23.828125 +GPU:0 | Epoch: 28 | loss=9.17175006866455 | Batch Time=22.65625 +GPU:0 | Epoch: 28 | loss=9.183757781982422 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.007457733154297 | Batch Time=21.09375 +GPU:0 | Epoch: 28 | loss=9.290067672729492 | Batch Time=19.921875 +GPU:0 | Epoch: 28 | loss=9.249434471130371 | Batch Time=18.359375 +GPU:0 | Epoch: 28 | loss=9.373998641967773 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.237667083740234 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.3345947265625 | Batch Time=18.75 +GPU:0 | Epoch: 28 | loss=9.148309707641602 | Batch Time=21.484375 +GPU:0 | Epoch: 28 | loss=9.180072784423828 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.27470874786377 | Batch Time=14.84375 +GPU:0 | Epoch: 28 | loss=9.244601249694824 | Batch Time=24.21875 +GPU:0 | Epoch: 28 | loss=9.075529098510742 | Batch Time=24.609375 +GPU:0 | Epoch: 28 | loss=9.241029739379883 | Batch Time=17.578125 +GPU:0 | Epoch: 28 | loss=9.028735160827637 | Batch Time=21.875 +GPU:0 | Epoch: 28 | loss=9.119436264038086 | Batch Time=21.09375 +GPU:0 | Epoch: 28 | loss=9.188252449035645 | Batch Time=21.484375 +GPU:0 | Epoch: 28 | loss=9.158426284790039 | Batch Time=23.046875 +GPU:0 | Epoch: 28 | loss=9.082698822021484 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=9.31283950805664 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.227163314819336 | Batch Time=24.21875 +GPU:0 | Epoch: 28 | loss=9.196569442749023 | Batch Time=19.140625 +GPU:0 | Epoch: 28 | loss=9.201200485229492 | Batch Time=18.359375 +GPU:0 | Epoch: 28 | loss=9.357406616210938 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.477828979492188 | Batch Time=16.796875 +GPU:0 | Epoch: 28 | loss=9.237580299377441 | Batch Time=20.3125 +GPU:0 | Epoch: 28 | loss=9.266448974609375 | Batch Time=17.96875 +GPU:0 | Epoch: 28 | loss=9.000580787658691 | Batch Time=19.921875 +GPU:0 | Epoch: 28 | loss=9.137697219848633 | Batch Time=24.21875 +GPU:0 | Epoch: 28 | loss=9.327716827392578 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.306013107299805 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.390543937683105 | Batch Time=13.28125 +GPU:0 | Epoch: 28 | loss=9.090742111206055 | Batch Time=24.21875 +GPU:0 | Epoch: 28 | loss=9.186250686645508 | Batch Time=16.796875 +GPU:0 | Epoch: 28 | loss=9.230843544006348 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.2767333984375 | Batch Time=19.140625 +GPU:0 | Epoch: 28 | loss=9.115174293518066 | Batch Time=23.4375 +GPU:0 | Epoch: 28 | loss=9.043073654174805 | Batch Time=19.921875 +GPU:0 | Epoch: 28 | loss=9.144147872924805 | Batch Time=21.484375 +GPU:0 | Epoch: 28 | loss=9.492334365844727 | Batch Time=20.3125 +GPU:0 | Epoch: 28 | loss=9.16906452178955 | Batch Time=24.609375 +GPU:0 | Epoch: 28 | loss=9.062295913696289 | Batch Time=27.34375 +GPU:0 | Epoch: 28 | loss=9.106315612792969 | Batch Time=21.875 +GPU:0 | Epoch: 28 | loss=9.149520874023438 | Batch Time=17.578125 +GPU:0 | Epoch: 28 | loss=9.04731559753418 | Batch Time=21.875 +GPU:0 | Epoch: 28 | loss=9.236580848693848 | Batch Time=20.3125 +GPU:0 | Epoch: 28 | loss=9.23532485961914 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.166452407836914 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.164937973022461 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=9.02197265625 | Batch Time=22.65625 +GPU:0 | Epoch: 28 | loss=9.226615905761719 | Batch Time=18.75 +GPU:0 | Epoch: 28 | loss=9.164223670959473 | Batch Time=17.1875 +GPU:0 | Epoch: 28 | loss=8.948347091674805 | Batch Time=21.484375 +GPU:0 | Epoch: 28 | loss=9.179603576660156 | Batch Time=19.921875 +GPU:0 | Epoch: 28 | loss=9.210105895996094 | Batch Time=14.453125 +GPU:0 | Epoch: 28 | loss=9.0162353515625 | Batch Time=21.09375 +GPU:0 | Epoch: 28 | loss=9.208207130432129 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.187786102294922 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.109559059143066 | Batch Time=21.484375 +GPU:0 | Epoch: 28 | loss=9.060790061950684 | Batch Time=19.921875 +GPU:0 | Epoch: 28 | loss=9.370309829711914 | Batch Time=21.09375 +GPU:0 | Epoch: 28 | loss=9.228565216064453 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.110580444335938 | Batch Time=25.78125 +GPU:0 | Epoch: 28 | loss=9.268436431884766 | Batch Time=17.578125 +GPU:0 | Epoch: 28 | loss=9.255403518676758 | Batch Time=18.75 +GPU:0 | Epoch: 28 | loss=9.056990623474121 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=9.051898002624512 | Batch Time=23.046875 +GPU:0 | Epoch: 28 | loss=9.143021583557129 | Batch Time=21.09375 +GPU:0 | Epoch: 28 | loss=9.29302978515625 | Batch Time=21.875 +GPU:0 | Epoch: 28 | loss=9.123885154724121 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=9.298792839050293 | Batch Time=16.40625 +GPU:0 | Epoch: 28 | loss=9.140509605407715 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.037282943725586 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.085597038269043 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=9.432820320129395 | Batch Time=18.75 +GPU:0 | Epoch: 28 | loss=9.407482147216797 | Batch Time=19.140625 +GPU:0 | Epoch: 28 | loss=9.192465782165527 | Batch Time=20.3125 +GPU:0 | Epoch: 28 | loss=9.168708801269531 | Batch Time=21.484375 +GPU:0 | Epoch: 28 | loss=9.178200721740723 | Batch Time=22.65625 +GPU:0 | Epoch: 28 | loss=9.183849334716797 | Batch Time=18.75 +GPU:0 | Epoch: 28 | loss=9.005186080932617 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=9.039852142333984 | Batch Time=22.65625 +GPU:0 | Epoch: 28 | loss=9.31192684173584 | Batch Time=17.578125 +GPU:0 | Epoch: 28 | loss=9.156669616699219 | Batch Time=18.359375 +GPU:0 | Epoch: 28 | loss=9.130911827087402 | Batch Time=19.140625 +GPU:0 | Epoch: 28 | loss=9.138467788696289 | Batch Time=20.3125 +GPU:0 | Epoch: 28 | loss=9.069173812866211 | Batch Time=19.921875 +GPU:0 | Epoch: 28 | loss=9.057395935058594 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=9.00066089630127 | Batch Time=22.65625 +GPU:0 | Epoch: 28 | loss=9.21182632446289 | Batch Time=23.046875 +GPU:0 | Epoch: 28 | loss=9.10411262512207 | Batch Time=19.53125 +GPU:0 | Epoch: 28 | loss=9.104040145874023 | Batch Time=21.09375 +GPU:0 | Epoch: 28 | loss=9.042216300964355 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=8.988121032714844 | Batch Time=25.390625 +GPU:0 | Epoch: 28 | loss=8.968843460083008 | Batch Time=23.046875 +GPU:0 | Epoch: 28 | loss=9.040127754211426 | Batch Time=20.3125 +GPU:0 | Epoch: 28 | loss=9.111089706420898 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=8.935850143432617 | Batch Time=26.5625 +GPU:0 | Epoch: 28 | loss=9.137140274047852 | Batch Time=17.1875 +GPU:0 | Epoch: 28 | loss=9.049195289611816 | Batch Time=24.21875 +GPU:0 | Epoch: 28 | loss=9.236969947814941 | Batch Time=19.140625 +GPU:0 | Epoch: 28 | loss=9.202422142028809 | Batch Time=19.921875 +GPU:0 | Epoch: 28 | loss=9.14420223236084 | Batch Time=14.84375 +GPU:0 | Epoch: 28 | loss=9.02148723602295 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.103109359741211 | Batch Time=17.96875 +GPU:0 | Epoch: 28 | loss=9.030058860778809 | Batch Time=19.140625 +GPU:0 | Epoch: 28 | loss=9.166451454162598 | Batch Time=21.875 +GPU:0 | Epoch: 28 | loss=9.067804336547852 | Batch Time=21.875 +GPU:0 | Epoch: 28 | loss=9.188226699829102 | Batch Time=16.015625 +GPU:0 | Epoch: 28 | loss=9.189104080200195 | Batch Time=16.40625 +GPU:0 | Epoch: 28 | loss=9.086889266967773 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=8.9110107421875 | Batch Time=25.78125 +GPU:0 | Epoch: 28 | loss=8.927143096923828 | Batch Time=22.65625 +GPU:0 | Epoch: 28 | loss=9.201814651489258 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=9.218262672424316 | Batch Time=18.75 +(TRAINER)AFTER TRAIN LOOP: GPU:3 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +AFTER TRAIN LOOP: Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +STARTING TRAINING: Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +BEFORE TRAIN LOOP: Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +(TRAINER)BEFORE TRAIN LOOP: GPU:3 | Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +(TRAINER)AFTER TRAIN LOOP: GPU:0 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +AFTER TRAIN LOOP: Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +BEFORE VAL LOOP: GPU: 0 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +(TRAINER)AFTER TRAIN LOOP: GPU:1 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +AFTER TRAIN LOOP: Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +STARTING TRAINING: Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +BEFORE TRAIN LOOP: Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +(TRAINER)BEFORE TRAIN LOOP: GPU:1 | Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +(TRAINER)AFTER TRAIN LOOP: GPU:2 | Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +AFTER TRAIN LOOP: Epoch 28 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +STARTING TRAINING: Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +BEFORE TRAIN LOOP: Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +(TRAINER)BEFORE TRAIN LOOP: GPU:2 | Epoch 29 | Memory Usage: svmem(total=257568083968, available=20050513920, percent=92.2, used=229793738752, free=16865992704, active=226856173568, inactive=10075824128, buffers=376020992, cached=10532331520, shared=5662908416, slab=835997696) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 28 | loss=3.5933961868286133 | Batch Time=30.078125 +GPU:0 | Epoch: 28 | loss=4.248144149780273 | Batch Time=13.28125 +GPU:0 | Epoch: 28 | loss=3.8941657543182373 | Batch Time=27.34375 +GPU:0 | Epoch: 28 | loss=2.5694305896759033 | Batch Time=35.9375 +GPU:0 | Epoch: 28 | loss=4.103115081787109 | Batch Time=10.15625 +GPU:0 | Epoch: 28 | loss=3.4555578231811523 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=4.662283420562744 | Batch Time=9.765625 +GPU:0 | Epoch: 28 | loss=4.069156646728516 | Batch Time=18.75 +GPU:0 | Epoch: 28 | loss=4.410639762878418 | Batch Time=12.5 +GPU:0 | Epoch: 28 | loss=5.177069664001465 | Batch Time=5.859375 +GPU:0 | Epoch: 28 | loss=5.10302734375 | Batch Time=5.859375 +GPU:0 | Epoch: 28 | loss=3.917306661605835 | Batch Time=32.421875 +GPU:0 | Epoch: 28 | loss=4.925539493560791 | Batch Time=13.671875 +GPU:0 | Epoch: 28 | loss=4.067954063415527 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=4.42090368270874 | Batch Time=13.28125 +GPU:0 | Epoch: 28 | loss=4.606112480163574 | Batch Time=13.28125 +GPU:0 | Epoch: 28 | loss=4.388478755950928 | Batch Time=14.0625 +GPU:0 | Epoch: 28 | loss=3.49013614654541 | Batch Time=26.953125 +GPU:0 | Epoch: 28 | loss=3.4802422523498535 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=3.3919878005981445 | Batch Time=28.125 +Model top1 Accuracy: 18.87 +Acc before rounding: 18.87 +Rounding model with scheme: naive +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +GPU:0 | Epoch: 28 | loss=3.5933961868286133 | Batch Time=30.078125 +GPU:0 | Epoch: 28 | loss=4.248144149780273 | Batch Time=13.28125 +GPU:0 | Epoch: 28 | loss=3.8941657543182373 | Batch Time=27.34375 +GPU:0 | Epoch: 28 | loss=2.5694305896759033 | Batch Time=35.9375 +GPU:0 | Epoch: 28 | loss=4.103115081787109 | Batch Time=10.15625 +GPU:0 | Epoch: 28 | loss=3.4555578231811523 | Batch Time=22.265625 +GPU:0 | Epoch: 28 | loss=4.662283420562744 | Batch Time=9.765625 +GPU:0 | Epoch: 28 | loss=4.069156646728516 | Batch Time=18.75 +GPU:0 | Epoch: 28 | loss=4.410639762878418 | Batch Time=12.5 +GPU:0 | Epoch: 28 | loss=5.177069664001465 | Batch Time=5.859375 +GPU:0 | Epoch: 28 | loss=5.10302734375 | Batch Time=5.859375 +GPU:0 | Epoch: 28 | loss=3.917306661605835 | Batch Time=32.421875 +GPU:0 | Epoch: 28 | loss=4.925539493560791 | Batch Time=13.671875 +GPU:0 | Epoch: 28 | loss=4.067954063415527 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=4.42090368270874 | Batch Time=13.28125 +GPU:0 | Epoch: 28 | loss=4.606112480163574 | Batch Time=13.28125 +GPU:0 | Epoch: 28 | loss=4.388478755950928 | Batch Time=14.0625 +GPU:0 | Epoch: 28 | loss=3.49013614654541 | Batch Time=26.953125 +GPU:0 | Epoch: 28 | loss=3.4802422523498535 | Batch Time=20.703125 +GPU:0 | Epoch: 28 | loss=3.3919878005981445 | Batch Time=28.125 +Model top1 Accuracy: 18.87 +Acc after rounding: 18.87 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Traceback (most recent call last): + File "main.py", line 343, in + main() + File "main.py", line 17, in main + mp.spawn(main_worker, args=(ngpus_per_node,), nprocs=ngpus_per_node, join=True) + File "/home/ubuntu/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 230, in spawn + return start_processes(fn, args, nprocs, join, daemon, start_method='spawn') + File "/home/ubuntu/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 188, in start_processes + while not context.join(): + File "/home/ubuntu/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 130, in join + raise ProcessExitedException( +torch.multiprocessing.spawn.ProcessExitedException: process 3 terminated with signal SIGKILL +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +/home/ubuntu/anaconda3/envs/pytorch_p38/lib/python3.8/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 80 leaked semaphore objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' From 90a827b7a6da70a5d7977e74c2d3274358cd4648 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 9 May 2022 05:25:34 +0000 Subject: [PATCH 040/113] will reset this commit. but keeping it here for debug --- data/imagenet.py | 10 +++++----- imagenet_exec.sh | 2 +- main.py | 25 +++++++++++++------------ main_utils.py | 8 ++++---- trainers/default.py | 21 +++++++++++++-------- 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/data/imagenet.py b/data/imagenet.py index 5e978d27..ca7ecb4c 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -56,8 +56,8 @@ def __init__(self, args): # use_full_data => we are not tuning hyperparameters validation_dataset = test_dataset else: - val_size = 10000 - train_size = len(dataset) - val_size + train_size = 1000 + val_size = len(dataset) - train_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) if parser_args.multiprocessing_distributed: @@ -75,15 +75,15 @@ def __init__(self, args): ) self.val_loader = torch.utils.data.DataLoader( - test_dataset, + validation_dataset, batch_size=parser_args.batch_size, - shuffle=False, + shuffle=True, **kwargs ) self.actual_val_loader = torch.utils.data.DataLoader( validation_dataset, batch_size=parser_args.batch_size, - shuffle=False, + shuffle=True, **kwargs ) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index 3e992ff9..baf2d31b 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -25,5 +25,5 @@ BLOCK conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" log_root="resnet50_sp5" log_end="_log" -python main.py \ +CUDA_VISIBLE_DEVICES=0,1,2,3 python main.py \ --config "$conf_file" > "$log_root$log_end" 2>&1 & diff --git a/main.py b/main.py index af1a83be..b3e3898a 100644 --- a/main.py +++ b/main.py @@ -13,7 +13,7 @@ def main(): ngpus_per_node = torch.cuda.device_count() if parser_args.multiprocessing_distributed: - assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" + # assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" mp.spawn(main_worker, args=(ngpus_per_node,), nprocs=ngpus_per_node, join=True) else: # Simply call main_worker function @@ -151,18 +151,19 @@ def main_worker(gpu, ngpus_per_node): modifier(parser_args, epoch, model) cur_lr = get_lr(optimizer) - # print("Skipping training, just gonna round") - # print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + print("Skipping training, just gonna round") + print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, - # ratio=parser_args.noise_ratio, rank=parser_args.gpu) - # if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - # acc1, acc5, acc10 = validate(data.val_loader, cp_model, criterion, parser_args, writer, epoch) - # else: - # acc1 = -1 - # print("GPU: {} | acc1={}".format(parser_args.gpu, acc1)) - # print("After Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) - # dist.barrier() - # continue + #ratio=parser_args.noise_ratio, rank=parser_args.gpu) + if True:#(parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + validate(data.train_loader, model, criterion, parser_args, writer, epoch) + acc1 = -1 + else: + acc1 = -1 + print("GPU: {} | acc1={}".format(parser_args.gpu, acc1)) + print("After Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + dist.barrier() + continue # save the score at the beginning of training epoch, so if we set parser.args.rewind_to_epoch to 0 diff --git a/main_utils.py b/main_utils.py index 07251bd0..1cf44d9f 100644 --- a/main_utils.py +++ b/main_utils.py @@ -1107,17 +1107,17 @@ def get_directories(parser_args): log_base_dir = run_base_dir / "logs" ckpt_base_dir = run_base_dir / "checkpoints" - if not run_base_dir.exists(): - os.makedirs(run_base_dir) + #if not run_base_dir.exists(): + # os.makedirs(run_base_dir) - (run_base_dir / "settings.txt").write_text(str(parser_args)) + #(run_base_dir / "settings.txt").write_text(str(parser_args)) return run_base_dir, ckpt_base_dir, log_base_dir def write_result_to_csv(**kwargs): results = pathlib.Path("runs") / "results.csv" - + return -1 if not results.exists(): results.write_text( "Date Finished, " diff --git a/trainers/default.py b/trainers/default.py index 774cc9f4..d7b8ae42 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -155,7 +155,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): top1 = 0 top5 = 0 top10 = 0 - num_images = 0 + num_images = 1 # switch to evaluate mode model.eval() @@ -165,28 +165,33 @@ def validate(val_loader, model, criterion, args, writer, epoch): # for i, (images, target) in tqdm.tqdm( # enumerate(val_loader), ascii=True, total=len(val_loader) # ): + # time.sleep(0.1) + # return -1, -1, -1 for i, (images, target) in enumerate(val_loader): + continue images = images.to(args.gpu) target = target.to(args.gpu) #print(images.shape, target.shape) # compute output - output = model(images) + # output = model(images) - loss = criterion(output, target) + # loss = criterion(output, target) + loss = torch.Tensor([0]) # measure accuracy and record loss - acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) + # acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) + acc1, acc5, acc10 = torch.Tensor([5]), torch.Tensor([5]), torch.Tensor([5]) # losses.update(loss.item(), images.size(0)) # top1.update(acc1.item(), images.size(0)) # top5.update(acc5.item(), images.size(0)) # top10.update(acc10.item(), images.size(0)) # compute weighted sum for each accuracy so we can average it later - top1 += acc1.item() * images.size(0) - top5 += acc5.item() * images.size(0) - top10 += acc10.item() * images.size(0) - num_images += images.size(0) + top1 += acc1.item()# * images.size(0) + top5 += acc5.item()# * images.size(0) + top10 += acc10.item()# * images.size(0) + num_images += 1#images.size(0) # measure elapsed time # batch_time.update(time.time() - end) From dfdd3d83713c8e5bec5c9bb237d50de06cdcb66a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 9 May 2022 05:26:11 +0000 Subject: [PATCH 041/113] pushing ddp debug logt --- ddp_debug_just_val_log | 867 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 867 insertions(+) create mode 100644 ddp_debug_just_val_log diff --git a/ddp_debug_just_val_log b/ddp_debug_just_val_log new file mode 100644 index 00000000..2cf07b43 --- /dev/null +++ b/ddp_debug_just_val_log @@ -0,0 +1,867 @@ +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Namespace(algo='hc_iter', alpha=1.0, alpha_prime=1.0, arch='ResNet50', batch_size=1024, bias=False, bn_type='NonAffineBatchNorm', bottom_k_on_forward=False, checkpoint_at_prune=False, chg_mask=False, chg_weight=False, ckpt_interval=-1, config='configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml', conv_type='SubnetConv', data='/home/ubuntu/ILSVRC2012/', data_dir='../data', dataset='ImageNet', dense_training=False, differentiate_clamp=False, dist_backend='nccl', epochs=88, evaluate=False, evaluate_only=False, fine_tune_lr=0.04, fine_tune_lr_policy='multistep_lr', fine_tune_optimizer='sgd', fine_tune_wd=0.0001, first_layer_dense=False, first_layer_type=None, fixed_init=False, flips=None, freeze_weights=True, gamma=0.1, gpu=0, hc_period=1, hc_quantized=True, hc_warmup=9999, hidden_size=500, how_to_connect='prob', how_to_prune='random', imp_no_rewind=False, imp_resume_epoch=-1, imp_resume_iter=-1, imp_resume_round=-1, imp_rewind_iter=1000, imp_rewind_model='short_imp/Liu_checkpoint_model_correct.pth', imp_rounds=-1, init='signed_constant', interpolate='prob', invert_sanity_check=False, iter_period=5, iter_start=0, label_smoothing=None, lam_finetune_loss=-1, last_layer_dense=False, lmbda=1e-06, load_ckpt=None, log_dir=None, log_interval=2, loss='cross-entropy-loss', lr=0.4, lr_adjust=50, lr_gamma=0.1, lr_policy='cosine_lr', max_iter=100000, metric='loss', milestones=[50, 100, 150, 200], mixed_precision=True, mode='fan_in', mode_connect=False, mode_connect_filename=None, momentum=0.9, multiprocessing_distributed=True, name='resnet50_imagenet', nesterov=False, no_bn_decay=False, no_cuda=False, noise=True, noise_ratio=0, nonlinearity='relu', num_round=1, num_step_finetune=10, num_test=1, num_trial=1, num_workers=4, only_sanity=False, optimizer='sgd', override_prune_rate=False, plot_hc_convergence=False, port=29500, pretrained=None, pretrained2=None, print_freq=10, project_freq=1, prune_rate=0.5, prune_type='BottomK', pruning_strategy=None, quantize_threshold=0.5, random_subnet=False, rank=-1, regularization='L2', reinit=False, results_filename=None, resume=None, rewind_score=False, rewind_to_epoch=-1, round='naive', run_idx=None, sanity_folder=None, save_every=-1, save_model=False, save_plot_data=False, scale_fan=False, score_init='unif', score_init_constant=None, seed=42, seed_fixed_init=24, shift=0.0, shuffle=False, skip_fine_tune=False, skip_sanity_checks=True, smart_ratio=-1, sr_version=1, start_epoch=None, start_from_nothing=False, subfolder=None, submask_size=1, target_sparsity=5, td=0.99, temp=100000, trainer='default', transformer_bptt=35, transformer_clip=0.25, transformer_dropout=0.2, transformer_emsize=200, transformer_nhead=2, transformer_nhid=200, transformer_nlayers=2, trial_num=1, unflag_before_finetune=False, use_full_data=False, warmup_length=0, wd=0.0, weight_training=False, width=1.0, width_mult=1.0, workers=12, world_size=-1, **{'compare-rounding': False}) + + +Beginning of process. + + +-------------------------------------- +TIME: The current time is: Mon May 9 05:17:43 2022 +TIME: The current time in seconds is: 1652073463.3837602 +-------------------------------------- + + +Seeded everything: 42 +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Use GPU: 3 for training +=> Using trainer from trainers.default +=> Creating model 'ResNet50' +==> Conv Type: SubnetConv +==> BN Type: NonAffineBatchNorm +==> Building first layer +==> Setting prune rate of network to 0.5 +=> Rough estimate model params 25502912 +=> Freezing model weights +=> Getting ImageNet dataset +scheduler: use cosine learning rate decay, with max epochs 88 +Computing prune_rate for target_sparsity 5 with iter_period 5 +Setting prune_rate to 0.16156611126074005 +STARTING TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241866600448, percent=6.1, used=13576986624, free=237415022592, active=14918508544, inactive=3857522688, buffers=452333568, cached=6123741184, shared=96784384, slab=553857024) +Skipping training, just gonna round +Before Round: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241866600448, percent=6.1, used=13576986624, free=237415022592, active=14918508544, inactive=3857522688, buffers=452333568, cached=6123741184, shared=96784384, slab=553857024) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Use GPU: 0 for training +=> Using trainer from trainers.default +=> Creating model 'ResNet50' +==> Conv Type: SubnetConv +==> BN Type: NonAffineBatchNorm +==> Building first layer +==> Setting prune rate of network to 0.5 +=> Rough estimate model params 25502912 +=> Freezing model weights +conv1.scores 9408 +layer1.0.conv1.scores 4096 +layer1.0.conv2.scores 36864 +layer1.0.conv3.scores 16384 +layer1.0.downsample.0.scores 16384 +layer1.1.conv1.scores 16384 +layer1.1.conv2.scores 36864 +layer1.1.conv3.scores 16384 +layer1.2.conv1.scores 16384 +layer1.2.conv2.scores 36864 +layer1.2.conv3.scores 16384 +layer2.0.conv1.scores 32768 +layer2.0.conv2.scores 147456 +layer2.0.conv3.scores 65536 +layer2.0.downsample.0.scores 131072 +layer2.1.conv1.scores 65536 +layer2.1.conv2.scores 147456 +layer2.1.conv3.scores 65536 +layer2.2.conv1.scores 65536 +layer2.2.conv2.scores 147456 +layer2.2.conv3.scores 65536 +layer2.3.conv1.scores 65536 +layer2.3.conv2.scores 147456 +layer2.3.conv3.scores 65536 +layer3.0.conv1.scores 131072 +layer3.0.conv2.scores 589824 +layer3.0.conv3.scores 262144 +layer3.0.downsample.0.scores 524288 +layer3.1.conv1.scores 262144 +layer3.1.conv2.scores 589824 +layer3.1.conv3.scores 262144 +layer3.2.conv1.scores 262144 +layer3.2.conv2.scores 589824 +layer3.2.conv3.scores 262144 +layer3.3.conv1.scores 262144 +layer3.3.conv2.scores 589824 +layer3.3.conv3.scores 262144 +layer3.4.conv1.scores 262144 +layer3.4.conv2.scores 589824 +layer3.4.conv3.scores 262144 +layer3.5.conv1.scores 262144 +layer3.5.conv2.scores 589824 +layer3.5.conv3.scores 262144 +layer4.0.conv1.scores 524288 +layer4.0.conv2.scores 2359296 +layer4.0.conv3.scores 1048576 +layer4.0.downsample.0.scores 2097152 +layer4.1.conv1.scores 1048576 +layer4.1.conv2.scores 2359296 +layer4.1.conv3.scores 1048576 +layer4.2.conv1.scores 1048576 +layer4.2.conv2.scores 2359296 +layer4.2.conv3.scores 1048576 +fc.scores 2048000 +total num_params: 25502912 +=> Getting ImageNet dataset +scheduler: use cosine learning rate decay, with max epochs 88 +Computing prune_rate for target_sparsity 5 with iter_period 5 +Setting prune_rate to 0.16156611126074005 +1 SubnetConv(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) +2 SubnetConv(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) +3 SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +4 SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +5 SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) +6 SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +7 SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +8 SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) +9 SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +10 SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +11 SubnetConv(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) +12 SubnetConv(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) +13 SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +14 SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) +15 SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +16 SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +17 SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) +18 SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +19 SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +20 SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) +21 SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +22 SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +23 SubnetConv(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +24 SubnetConv(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) +25 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +26 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +27 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +28 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +29 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +30 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +31 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +32 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +33 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +34 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +35 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +36 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +37 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +38 SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) +39 SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +40 SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) +41 SubnetConv(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +42 SubnetConv(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) +43 SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) +44 SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +45 SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +46 SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) +47 SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) +48 SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) +49 SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) +50 SubnetConv(2048, 1000, kernel_size=(1, 1), stride=(1, 1), bias=False) +STARTING TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241830998016, percent=6.1, used=13612589056, free=237379420160, active=14953283584, inactive=3857522688, buffers=452333568, cached=6123741184, shared=96784384, slab=553857024) +Skipping training, just gonna round +Before Round: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241830998016, percent=6.1, used=13612589056, free=237379420160, active=14953283584, inactive=3857522688, buffers=452333568, cached=6123741184, shared=96784384, slab=553857024) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Use GPU: 2 for training +=> Using trainer from trainers.default +=> Creating model 'ResNet50' +==> Conv Type: SubnetConv +==> BN Type: NonAffineBatchNorm +==> Building first layer +==> Setting prune rate of network to 0.5 +=> Rough estimate model params 25502912 +=> Freezing model weights +=> Getting ImageNet dataset +scheduler: use cosine learning rate decay, with max epochs 88 +Computing prune_rate for target_sparsity 5 with iter_period 5 +Setting prune_rate to 0.16156611126074005 +STARTING TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241832804352, percent=6.1, used=13610729472, free=237381226496, active=14950014976, inactive=3857522688, buffers=452333568, cached=6123794432, shared=96833536, slab=553857024) +Skipping training, just gonna round +Before Round: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241832804352, percent=6.1, used=13610729472, free=237381226496, active=14950014976, inactive=3857522688, buffers=452333568, cached=6123794432, shared=96833536, slab=553857024) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Use GPU: 1 for training +=> Using trainer from trainers.default +=> Creating model 'ResNet50' +==> Conv Type: SubnetConv +==> BN Type: NonAffineBatchNorm +==> Building first layer +==> Setting prune rate of network to 0.5 +=> Rough estimate model params 25502912 +=> Freezing model weights +=> Getting ImageNet dataset +scheduler: use cosine learning rate decay, with max epochs 88 +Computing prune_rate for target_sparsity 5 with iter_period 5 +Setting prune_rate to 0.16156611126074005 +STARTING TRAINING: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241675661312, percent=6.2, used=13767872512, free=237224083456, active=15108395008, inactive=3857522688, buffers=452333568, cached=6123794432, shared=96833536, slab=553857024) +Skipping training, just gonna round +Before Round: Epoch 0 | Memory Usage: svmem(total=257568083968, available=241675661312, percent=6.2, used=13767872512, free=237224083456, active=15108395008, inactive=3857522688, buffers=452333568, cached=6123794432, shared=96833536, slab=553857024) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 0 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +STARTING TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +Skipping training, just gonna round +Before Round: Epoch 1 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 0 | Memory Usage: svmem(total=257568083968, available=239854374912, percent=6.9, used=14984900608, free=235292409856, active=16281235456, inactive=4562644992, buffers=452378624, cached=6838394880, shared=699240448, slab=564129792) +STARTING TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +Skipping training, just gonna round +Before Round: Epoch 1 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 0 | Memory Usage: svmem(total=257568083968, available=239868772352, percent=6.9, used=14970503168, free=235306807296, active=16267505664, inactive=4562644992, buffers=452378624, cached=6838394880, shared=699240448, slab=564097024) +STARTING TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +Skipping training, just gonna round +Before Round: Epoch 1 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 0 | Memory Usage: svmem(total=257568083968, available=239868227584, percent=6.9, used=14971047936, free=235306262528, active=16267505664, inactive=4562644992, buffers=452378624, cached=6838394880, shared=699240448, slab=564097024) +STARTING TRAINING: Epoch 1 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +Skipping training, just gonna round +Before Round: Epoch 1 | Memory Usage: svmem(total=257568083968, available=239869431808, percent=6.9, used=14969909248, free=235307470848, active=16267698176, inactive=4562612224, buffers=452399104, cached=6838304768, shared=699158528, slab=563982336) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 1 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +STARTING TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +Skipping training, just gonna round +Before Round: Epoch 2 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 1 | Memory Usage: svmem(total=257568083968, available=238790676480, percent=7.3, used=16047767552, free=234149498880, active=17369243648, inactive=4606627840, buffers=452415488, cached=6918402048, shared=699174912, slab=568074240) +STARTING TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +Skipping training, just gonna round +Before Round: Epoch 2 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 1 | Memory Usage: svmem(total=257568083968, available=235897942016, percent=8.4, used=18940383232, free=231256784896, active=20249591808, inactive=4606734336, buffers=452415488, cached=6918500352, shared=699273216, slab=568410112) +STARTING TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +Skipping training, just gonna round +Before Round: Epoch 2 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 1 | Memory Usage: svmem(total=257568083968, available=238790983680, percent=7.3, used=16047542272, free=234149806080, active=17369198592, inactive=4606595072, buffers=452415488, cached=6918320128, shared=699092992, slab=568033280) +STARTING TRAINING: Epoch 2 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +Skipping training, just gonna round +Before Round: Epoch 2 | Memory Usage: svmem(total=257568083968, available=238790684672, percent=7.3, used=16047853568, free=234149507072, active=17369210880, inactive=4606590976, buffers=452415488, cached=6918307840, shared=699080704, slab=568066048) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 2 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +STARTING TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +Skipping training, just gonna round +Before Round: Epoch 3 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 2 | Memory Usage: svmem(total=257568083968, available=237710925824, percent=7.7, used=17126723584, free=233007710208, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +STARTING TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +Skipping training, just gonna round +Before Round: Epoch 3 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 2 | Memory Usage: svmem(total=257568083968, available=237711011840, percent=7.7, used=17126473728, free=233007796224, active=18478981120, inactive=4624818176, buffers=452456448, cached=6981357568, shared=699273216, slab=570617856) +STARTING TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +Skipping training, just gonna round +Before Round: Epoch 3 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 2 | Memory Usage: svmem(total=257568083968, available=237711093760, percent=7.7, used=17126391808, free=233007878144, active=18479149056, inactive=4624818176, buffers=452456448, cached=6981357568, shared=699273216, slab=570617856) +STARTING TRAINING: Epoch 3 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +Skipping training, just gonna round +Before Round: Epoch 3 | Memory Usage: svmem(total=257568083968, available=237710934016, percent=7.7, used=17126715392, free=233007718400, active=18478784512, inactive=4624752640, buffers=452456448, cached=6981193728, shared=699109376, slab=570544128) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 3 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +STARTING TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +Skipping training, just gonna round +Before Round: Epoch 4 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 3 | Memory Usage: svmem(total=257568083968, available=236631998464, percent=8.1, used=18204352512, free=231859339264, active=19605348352, inactive=4633477120, buffers=458727424, cached=7045664768, shared=699289600, slab=574271488) +STARTING TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +Skipping training, just gonna round +Before Round: Epoch 4 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 3 | Memory Usage: svmem(total=257568083968, available=236632104960, percent=8.1, used=18204409856, free=231859445760, active=19605250048, inactive=4633411584, buffers=458727424, cached=7045500928, shared=699125760, slab=574185472) +STARTING TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +Skipping training, just gonna round +Before Round: Epoch 4 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 3 | Memory Usage: svmem(total=257568083968, available=236631998464, percent=8.1, used=18204352512, free=231859339264, active=19605348352, inactive=4633477120, buffers=458727424, cached=7045664768, shared=699289600, slab=574271488) +STARTING TRAINING: Epoch 4 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +Skipping training, just gonna round +Before Round: Epoch 4 | Memory Usage: svmem(total=257568083968, available=236632559616, percent=8.1, used=18204184576, free=231860031488, active=19605176320, inactive=4633370624, buffers=458727424, cached=7045140480, shared=699027456, slab=573870080) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 4 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +STARTING TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +Skipping training, just gonna round +Before Round: Epoch 5 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 4 | Memory Usage: svmem(total=257568083968, available=235554959360, percent=8.5, used=19281162240, free=230745763840, active=20719177728, inactive=4622475264, buffers=458739712, cached=7082418176, shared=699224064, slab=574783488) +STARTING TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +Skipping training, just gonna round +Before Round: Epoch 5 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 4 | Memory Usage: svmem(total=257568083968, available=235554226176, percent=8.5, used=19281813504, free=230745030656, active=20719792128, inactive=4622508032, buffers=458739712, cached=7082500096, shared=699305984, slab=574799872) +STARTING TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +Skipping training, just gonna round +Before Round: Epoch 5 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 4 | Memory Usage: svmem(total=257568083968, available=235554951168, percent=8.5, used=19281170432, free=230745755648, active=20719177728, inactive=4622475264, buffers=458739712, cached=7082418176, shared=699224064, slab=574783488) +STARTING TRAINING: Epoch 5 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +Skipping training, just gonna round +Before Round: Epoch 5 | Memory Usage: svmem(total=257568083968, available=235555250176, percent=8.5, used=19281035264, free=230746054656, active=20719042560, inactive=4622409728, buffers=458739712, cached=7082254336, shared=699060224, slab=574767104) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 5 | Memory Usage: svmem(total=257568083968, available=234476613632, percent=9.0, used=20359675904, free=229640441856, active=21827698688, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +STARTING TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +Skipping training, just gonna round +Before Round: Epoch 6 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 5 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +STARTING TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +Skipping training, just gonna round +Before Round: Epoch 6 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 5 | Memory Usage: svmem(total=257568083968, available=234476482560, percent=9.0, used=20359667712, free=229640310784, active=21827551232, inactive=4607844352, buffers=458752000, cached=7109353472, shared=699240448, slab=573894656) +STARTING TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +Skipping training, just gonna round +Before Round: Epoch 6 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 5 | Memory Usage: svmem(total=257568083968, available=234475950080, percent=9.0, used=20360146944, free=229639778304, active=21827592192, inactive=4607860736, buffers=458752000, cached=7109406720, shared=699293696, slab=573911040) +STARTING TRAINING: Epoch 6 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +Skipping training, just gonna round +Before Round: Epoch 6 | Memory Usage: svmem(total=257568083968, available=234476621824, percent=9.0, used=20359667712, free=229640450048, active=21827751936, inactive=4607787008, buffers=458752000, cached=7109214208, shared=699101184, slab=573861888) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 6 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +STARTING TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +Skipping training, just gonna round +Before Round: Epoch 7 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 6 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +STARTING TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +Skipping training, just gonna round +Before Round: Epoch 7 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 6 | Memory Usage: svmem(total=257568083968, available=233382707200, percent=9.4, used=21452374016, free=228523671552, active=22946299904, inactive=4590821376, buffers=458764288, cached=7133274112, shared=699256832, slab=578170880) +STARTING TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +Skipping training, just gonna round +Before Round: Epoch 7 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 6 | Memory Usage: svmem(total=257568083968, available=233395535872, percent=9.4, used=21439463424, free=228536500224, active=22933372928, inactive=4590854144, buffers=458764288, cached=7133356032, shared=699338752, slab=578285568) +STARTING TRAINING: Epoch 7 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +Skipping training, just gonna round +Before Round: Epoch 7 | Memory Usage: svmem(total=257568083968, available=233396252672, percent=9.4, used=21438910464, free=228537217024, active=22933069824, inactive=4590788608, buffers=458764288, cached=7133192192, shared=699174912, slab=578220032) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 7 | Memory Usage: svmem(total=257568083968, available=232318562304, percent=9.8, used=22516482048, free=227448471552, active=24030625792, inactive=4571344896, buffers=458772480, cached=7144357888, shared=699191296, slab=579198976) +STARTING TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=232318836736, percent=9.8, used=22516281344, free=227448745984, active=24030580736, inactive=4571316224, buffers=458772480, cached=7144284160, shared=699117568, slab=579190784) +Skipping training, just gonna round +Before Round: Epoch 8 | Memory Usage: svmem(total=257568083968, available=232318836736, percent=9.8, used=22516281344, free=227448745984, active=24030580736, inactive=4571316224, buffers=458772480, cached=7144284160, shared=699117568, slab=579190784) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 7 | Memory Usage: svmem(total=257568083968, available=232318554112, percent=9.8, used=22516490240, free=227448463360, active=24030625792, inactive=4571344896, buffers=458772480, cached=7144357888, shared=699191296, slab=579198976) +STARTING TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=232318562304, percent=9.8, used=22516482048, free=227448471552, active=24030625792, inactive=4571344896, buffers=458772480, cached=7144357888, shared=699191296, slab=579198976) +Skipping training, just gonna round +Before Round: Epoch 8 | Memory Usage: svmem(total=257568083968, available=232318836736, percent=9.8, used=22516281344, free=227448745984, active=24030580736, inactive=4571316224, buffers=458772480, cached=7144284160, shared=699117568, slab=579190784) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 7 | Memory Usage: svmem(total=257568083968, available=232317935616, percent=9.8, used=22517026816, free=227447844864, active=24030711808, inactive=4571377664, buffers=458772480, cached=7144439808, shared=699273216, slab=579166208) +STARTING TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=232318562304, percent=9.8, used=22516482048, free=227448471552, active=24030625792, inactive=4571344896, buffers=458772480, cached=7144357888, shared=699191296, slab=579198976) +Skipping training, just gonna round +Before Round: Epoch 8 | Memory Usage: svmem(total=257568083968, available=232318836736, percent=9.8, used=22516281344, free=227448745984, active=24030580736, inactive=4571316224, buffers=458772480, cached=7144284160, shared=699117568, slab=579190784) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 7 | Memory Usage: svmem(total=257568083968, available=232317820928, percent=9.8, used=22517059584, free=227447730176, active=24030576640, inactive=4571410432, buffers=458772480, cached=7144521728, shared=699355136, slab=579174400) +STARTING TRAINING: Epoch 8 | Memory Usage: svmem(total=257568083968, available=232318836736, percent=9.8, used=22516281344, free=227448745984, active=24030580736, inactive=4571316224, buffers=458772480, cached=7144284160, shared=699117568, slab=579190784) +Skipping training, just gonna round +Before Round: Epoch 8 | Memory Usage: svmem(total=257568083968, available=232318836736, percent=9.8, used=22516281344, free=227448745984, active=24030580736, inactive=4571316224, buffers=458772480, cached=7144284160, shared=699117568, slab=579190784) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 8 | Memory Usage: svmem(total=257568083968, available=231240687616, percent=10.2, used=23593824256, free=226358386688, active=25124098048, inactive=4556869632, buffers=458780672, cached=7157092352, shared=699371520, slab=580452352) +STARTING TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +Skipping training, just gonna round +Before Round: Epoch 9 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 8 | Memory Usage: svmem(total=257568083968, available=231240986624, percent=10.2, used=23593607168, free=226358685696, active=25124143104, inactive=4556836864, buffers=458780672, cached=7157010432, shared=699289600, slab=580427776) +STARTING TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +Skipping training, just gonna round +Before Round: Epoch 9 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 8 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +STARTING TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +Skipping training, just gonna round +Before Round: Epoch 9 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 8 | Memory Usage: svmem(total=257568083968, available=231240695808, percent=10.2, used=23593816064, free=226358394880, active=25124139008, inactive=4556869632, buffers=458780672, cached=7157092352, shared=699371520, slab=580452352) +STARTING TRAINING: Epoch 9 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +Skipping training, just gonna round +Before Round: Epoch 9 | Memory Usage: svmem(total=257568083968, available=231241297920, percent=10.2, used=23593623552, free=226359160832, active=25124016128, inactive=4556771328, buffers=458780672, cached=7156518912, shared=699125760, slab=580009984) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 9 | Memory Usage: svmem(total=257568083968, available=230163259392, percent=10.6, used=24671035392, free=225271758848, active=26218409984, inactive=4538830848, buffers=458780672, cached=7166509056, shared=699359232, slab=581611520) +STARTING TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +Skipping training, just gonna round +Before Round: Epoch 10 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 9 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +STARTING TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +Skipping training, just gonna round +Before Round: Epoch 10 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 9 | Memory Usage: svmem(total=257568083968, available=230151180288, percent=10.6, used=24683073536, free=225259679744, active=26230935552, inactive=4538851328, buffers=458780672, cached=7166550016, shared=699400192, slab=581545984) +STARTING TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +Skipping training, just gonna round +Before Round: Epoch 10 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 9 | Memory Usage: svmem(total=257568083968, available=230163075072, percent=10.6, used=24671178752, free=225271574528, active=26218442752, inactive=4538851328, buffers=458780672, cached=7166550016, shared=699400192, slab=581545984) +STARTING TRAINING: Epoch 10 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +Skipping training, just gonna round +Before Round: Epoch 10 | Memory Usage: svmem(total=257568083968, available=230164344832, percent=10.6, used=24670126080, free=225272844288, active=26217717760, inactive=4538720256, buffers=458780672, cached=7166332928, shared=699183104, slab=581308416) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 10 | Memory Usage: svmem(total=257568083968, available=229084164096, percent=11.1, used=25749921792, free=224185765888, active=27302985728, inactive=4527165440, buffers=458788864, cached=7173607424, shared=699396096, slab=584314880) +STARTING TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +Skipping training, just gonna round +Before Round: Epoch 11 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 10 | Memory Usage: svmem(total=257568083968, available=229084155904, percent=11.1, used=25749929984, free=224185757696, active=27302944768, inactive=4527165440, buffers=458788864, cached=7173607424, shared=699396096, slab=584314880) +STARTING TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +Skipping training, just gonna round +Before Round: Epoch 11 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 10 | Memory Usage: svmem(total=257568083968, available=229084147712, percent=11.1, used=25749938176, free=224185749504, active=27302944768, inactive=4527165440, buffers=458788864, cached=7173607424, shared=699396096, slab=584314880) +STARTING TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +Skipping training, just gonna round +Before Round: Epoch 11 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 10 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +STARTING TRAINING: Epoch 11 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +Skipping training, just gonna round +Before Round: Epoch 11 | Memory Usage: svmem(total=257568083968, available=229084471296, percent=11.1, used=25749860352, free=224186073088, active=27302887424, inactive=4527067136, buffers=458788864, cached=7173361664, shared=699150336, slab=584282112) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 11 | Memory Usage: svmem(total=257568083968, available=228007362560, percent=11.5, used=26826797056, free=223103909888, active=28385570816, inactive=4516302848, buffers=458797056, cached=7178579968, shared=699396096, slab=583700480) +STARTING TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +Skipping training, just gonna round +Before Round: Epoch 12 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 11 | Memory Usage: svmem(total=257568083968, available=228006748160, percent=11.5, used=26827325440, free=223103201280, active=28385570816, inactive=4516302848, buffers=458788864, cached=7178768384, shared=699396096, slab=583888896) +STARTING TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +Skipping training, just gonna round +Before Round: Epoch 12 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 11 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +STARTING TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +Skipping training, just gonna round +Before Round: Epoch 12 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 11 | Memory Usage: svmem(total=257568083968, available=228006739968, percent=11.5, used=26827333632, free=223103193088, active=28385570816, inactive=4516302848, buffers=458788864, cached=7178768384, shared=699396096, slab=583888896) +STARTING TRAINING: Epoch 12 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +Skipping training, just gonna round +Before Round: Epoch 12 | Memory Usage: svmem(total=257568083968, available=228007903232, percent=11.5, used=26826346496, free=223104442368, active=28385525760, inactive=4516270080, buffers=458797056, cached=7178498048, shared=699314176, slab=583684096) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 12 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +STARTING TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +Skipping training, just gonna round +Before Round: Epoch 13 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 12 | Memory Usage: svmem(total=257568083968, available=226926505984, percent=11.9, used=27907551232, free=222011092992, active=29474377728, inactive=4507623424, buffers=460058624, cached=7189381120, shared=699412480, slab=585732096) +STARTING TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +Skipping training, just gonna round +Before Round: Epoch 13 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 12 | Memory Usage: svmem(total=257568083968, available=226926522368, percent=11.9, used=27907534848, free=222011109376, active=29474385920, inactive=4507623424, buffers=460058624, cached=7189381120, shared=699412480, slab=585732096) +STARTING TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +Skipping training, just gonna round +Before Round: Epoch 13 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 12 | Memory Usage: svmem(total=257568083968, available=226926563328, percent=11.9, used=27907493888, free=222011150336, active=29474385920, inactive=4507623424, buffers=460058624, cached=7189381120, shared=699412480, slab=585699328) +STARTING TRAINING: Epoch 13 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +Skipping training, just gonna round +Before Round: Epoch 13 | Memory Usage: svmem(total=257568083968, available=226926379008, percent=11.9, used=27907760128, free=222010966016, active=29474430976, inactive=4507590656, buffers=460058624, cached=7189299200, shared=699330560, slab=585682944) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 13 | Memory Usage: svmem(total=257568083968, available=225844846592, percent=12.3, used=28988944384, free=220925771776, active=30554353664, inactive=4498653184, buffers=460075008, cached=7193292800, shared=699428864, slab=587362304) +STARTING TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +Skipping training, just gonna round +Before Round: Epoch 14 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 13 | Memory Usage: svmem(total=257568083968, available=225844846592, percent=12.3, used=28988944384, free=220925771776, active=30554353664, inactive=4498653184, buffers=460075008, cached=7193292800, shared=699428864, slab=587362304) +STARTING TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +Skipping training, just gonna round +Before Round: Epoch 14 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 13 | Memory Usage: svmem(total=257568083968, available=225844830208, percent=12.3, used=28988960768, free=220925755392, active=30554353664, inactive=4498653184, buffers=460075008, cached=7193292800, shared=699428864, slab=587362304) +STARTING TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +Skipping training, just gonna round +Before Round: Epoch 14 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 13 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +STARTING TRAINING: Epoch 14 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +Skipping training, just gonna round +Before Round: Epoch 14 | Memory Usage: svmem(total=257568083968, available=225845911552, percent=12.3, used=28988043264, free=220926836736, active=30554341376, inactive=4498587648, buffers=460075008, cached=7193128960, shared=699265024, slab=587337728) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 14 | Memory Usage: svmem(total=257568083968, available=224768364544, percent=12.7, used=30065111040, free=219846156288, active=31638274048, inactive=4487221248, buffers=460095488, cached=7196721152, shared=699445248, slab=587350016) +STARTING TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +Skipping training, just gonna round +Before Round: Epoch 15 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 14 | Memory Usage: svmem(total=257568083968, available=224768348160, percent=12.7, used=30065127424, free=219846139904, active=31638220800, inactive=4487221248, buffers=460095488, cached=7196721152, shared=699445248, slab=587350016) +STARTING TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +Skipping training, just gonna round +Before Round: Epoch 15 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 14 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +STARTING TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +Skipping training, just gonna round +Before Round: Epoch 15 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 14 | Memory Usage: svmem(total=257568083968, available=224768364544, percent=12.7, used=30065111040, free=219846156288, active=31638274048, inactive=4487221248, buffers=460095488, cached=7196721152, shared=699445248, slab=587350016) +STARTING TRAINING: Epoch 15 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +Skipping training, just gonna round +Before Round: Epoch 15 | Memory Usage: svmem(total=257568083968, available=224769335296, percent=12.7, used=30064316416, free=219847131136, active=31638179840, inactive=4487151616, buffers=460095488, cached=7196540928, shared=699273216, slab=587251712) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 15 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +STARTING TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +Skipping training, just gonna round +Before Round: Epoch 16 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 15 | Memory Usage: svmem(total=257568083968, available=223691268096, percent=13.2, used=31141892096, free=218766520320, active=32714416128, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +STARTING TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +Skipping training, just gonna round +Before Round: Epoch 16 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 15 | Memory Usage: svmem(total=257568083968, available=223690809344, percent=13.2, used=31142268928, free=218766061568, active=32714752000, inactive=4476952576, buffers=460107776, cached=7199645696, shared=699461632, slab=589012992) +STARTING TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +Skipping training, just gonna round +Before Round: Epoch 16 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 15 | Memory Usage: svmem(total=257568083968, available=223691268096, percent=13.2, used=31141892096, free=218766520320, active=32714416128, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +STARTING TRAINING: Epoch 16 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +Skipping training, just gonna round +Before Round: Epoch 16 | Memory Usage: svmem(total=257568083968, available=223678771200, percent=13.2, used=31154388992, free=218754023424, active=32727724032, inactive=4476911616, buffers=460107776, cached=7199563776, shared=699379712, slab=588996608) +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +=> Reading YAML config from configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +Model top1 Accuracy: 0.0 +GPU: 0 | acc1=-1 +After Round: Epoch 16 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +STARTING TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +Skipping training, just gonna round +Before Round: Epoch 17 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +Model top1 Accuracy: 0.0 +GPU: 2 | acc1=-1 +After Round: Epoch 16 | Memory Usage: svmem(total=257568083968, available=222613385216, percent=13.6, used=32219484160, free=217685495808, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +STARTING TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +Skipping training, just gonna round +Before Round: Epoch 17 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +Model top1 Accuracy: 0.0 +GPU: 3 | acc1=-1 +After Round: Epoch 16 | Memory Usage: svmem(total=257568083968, available=222600069120, percent=13.6, used=32232714240, free=217672183808, active=33803800576, inactive=4470636544, buffers=460132352, cached=7203053568, shared=699478016, slab=590229504) +STARTING TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +Skipping training, just gonna round +Before Round: Epoch 17 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +Model top1 Accuracy: 0.0 +GPU: 1 | acc1=-1 +After Round: Epoch 16 | Memory Usage: svmem(total=257568083968, available=222613098496, percent=13.6, used=32219684864, free=217685213184, active=33790713856, inactive=4470636544, buffers=460132352, cached=7203053568, shared=699478016, slab=590229504) +STARTING TRAINING: Epoch 17 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) +Skipping training, just gonna round +Before Round: Epoch 17 | Memory Usage: svmem(total=257568083968, available=222613651456, percent=13.6, used=32219217920, free=217685762048, active=33790447616, inactive=4470607872, buffers=460132352, cached=7202971648, shared=699396096, slab=590221312) From f8177922db80a55a6d2d31bf28c25d010578978d Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 9 May 2022 13:48:45 -0500 Subject: [PATCH 042/113] modifying data loader to look more like pytorch example --- data/imagenet.py | 82 ++++++++++++++++++------------------------------ 1 file changed, 31 insertions(+), 51 deletions(-) diff --git a/data/imagenet.py b/data/imagenet.py index ca7ecb4c..ad56d129 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -14,76 +14,56 @@ def __init__(self, args): data_root = parser_args.data - use_cuda = torch.cuda.is_available() - - # Data loading code - kwargs = {"num_workers": parser_args.num_workers, "pin_memory": True} if use_cuda else {} - - # Data loading code - traindir = os.path.join(data_root, "train") - valdir = os.path.join(data_root, "val") - - normalize = transforms.Normalize( - mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] - ) + traindir = os.path.join(data_root, 'train') + valdir = os.path.join(data_root, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) dataset = datasets.ImageFolder( - traindir, - transforms.Compose( - [ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - normalize, - ] - ), - ) - - test_dataset = datasets.ImageFolder( - valdir, - transforms.Compose( - [ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - normalize, - ] - ), - ) + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) if parser_args.use_full_data: train_dataset = dataset # use_full_data => we are not tuning hyperparameters validation_dataset = test_dataset else: - train_size = 1000 - val_size = len(dataset) - train_size + # train_size = 1000 + # val_size = len(dataset) - train_size + val_size = 10000 + train_size = len(dataset) - val_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) - if parser_args.multiprocessing_distributed: + if parser_args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None + val_dataset = datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])) self.train_loader = torch.utils.data.DataLoader( - train_dataset, - batch_size=parser_args.batch_size, - shuffle=(train_sampler is None), - sampler=train_sampler, - **kwargs - ) + train_dataset, batch_size=parser_args.batch_size, + shuffle=(train_sampler is None), + num_workers=parser_args.num_workers, + pin_memory=True, sampler=train_sampler) self.val_loader = torch.utils.data.DataLoader( - validation_dataset, - batch_size=parser_args.batch_size, - shuffle=True, - **kwargs - ) + val_dataset, + batch_size=parser_args.batch_size, shuffle=False, + num_workers=parser_args.num_workers, pin_memory=True) self.actual_val_loader = torch.utils.data.DataLoader( validation_dataset, - batch_size=parser_args.batch_size, - shuffle=True, - **kwargs + batch_size=parser_args.batch_size, shuffle=False, + num_workers=parser_args.num_workers, pin_memory=True ) From 9c5dad2361fb48d7f173157573389a0d57a957b6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 9 May 2022 19:06:55 +0000 Subject: [PATCH 043/113] trying more things with dataloader --- data/imagenet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/imagenet.py b/data/imagenet.py index ad56d129..65f7d934 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -35,11 +35,11 @@ def __init__(self, args): else: # train_size = 1000 # val_size = len(dataset) - train_size - val_size = 10000 + val_size = 1000 train_size = len(dataset) - val_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) - if parser_args.distributed: + if parser_args.multiprocessing_distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None From 0d8fe44412b57eab1ce6a1632805cb14adcf17f2 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 9 May 2022 14:09:28 -0500 Subject: [PATCH 044/113] trying to avoid using data loader object --- main.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index b3e3898a..b1c068c7 100644 --- a/main.py +++ b/main.py @@ -142,6 +142,69 @@ def main_worker(gpu, ngpus_per_node): return + #########################DATA LOADING CODE######################### + from torchvision import datasets, transforms + import torch.multiprocessing + from torch.utils.data import random_split + torch.multiprocessing.set_sharing_strategy("file_system") + data_root = parser_args.data + + traindir = os.path.join(data_root, 'train') + valdir = os.path.join(data_root, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if parser_args.use_full_data: + train_dataset = dataset + # use_full_data => we are not tuning hyperparameters + validation_dataset = test_dataset + else: + # train_size = 1000 + # val_size = len(dataset) - train_size + val_size = 10000 + train_size = len(dataset) - val_size + train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) + + if parser_args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + val_dataset = datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])) + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=parser_args.batch_size, + shuffle=(train_sampler is None), + num_workers=parser_args.num_workers, + pin_memory=True, sampler=train_sampler) + + val_loader = torch.utils.data.DataLoader( + val_dataset, + batch_size=parser_args.batch_size, shuffle=False, + num_workers=parser_args.num_workers, pin_memory=True) + + actual_val_loader = torch.utils.data.DataLoader( + validation_dataset, + batch_size=parser_args.batch_size, shuffle=False, + num_workers=parser_args.num_workers, pin_memory=True + ) + + #########################DATA LOADING CODE######################### + # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): print("STARTING TRAINING: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) @@ -156,7 +219,7 @@ def main_worker(gpu, ngpus_per_node): # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, #ratio=parser_args.noise_ratio, rank=parser_args.gpu) if True:#(parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - validate(data.train_loader, model, criterion, parser_args, writer, epoch) + validate(actual_val_loader, model, criterion, parser_args, writer, epoch) acc1 = -1 else: acc1 = -1 From 8e3503bd2a95c6e268d4d7eccbd848ad104f3e46 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 9 May 2022 14:19:58 -0500 Subject: [PATCH 045/113] removing trainer --- main.py | 197 ++++++++++++++++++++++++++++++++++++++++++++++++++ main_utils.py | 15 +++- 2 files changed, 211 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index b1c068c7..518e79f1 100644 --- a/main.py +++ b/main.py @@ -417,5 +417,202 @@ def main_worker(gpu, ngpus_per_node): cleanup_distributed() +def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler=None): + # batch_time = AverageMeter("Time", ":6.3f") + # data_time = AverageMeter("Data", ":6.3f") + # losses = AverageMeter("Loss", ":.3f") + # top1 = AverageMeter("Acc@1", ":6.2f") + # top5 = AverageMeter("Acc@5", ":6.2f") + # top10 = AverageMeter("Acc@10", ":6.2f") + # progress = ProgressMeter( + # len(train_loader), + # [batch_time, data_time, losses, top1, top5], + # prefix=f"GPU:[{args.gpu}] | Epoch: [{epoch}]", + # ) + top1 = 0 + top5 = 0 + top10 = 0 + num_images = 0 + + # switch to train mode + model.train() + + batch_size = train_loader.batch_size + num_batches = len(train_loader) + end = time.time() + # for i, (images, target) in tqdm.tqdm( + # enumerate(train_loader), ascii=True, total=len(train_loader) + # ): + print("(TRAINER)BEFORE TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + for i, (images, target) in enumerate(train_loader): + # print("(TRAINER)AFTER LOADING IMAGES: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # print("Is it just the image?") + # print("(TRAINER): Size of images: {}".format(sys.getsizeof(images))) + # measure data loading time + data_time = time.time() - end + # print("Data Time: {}".format(data_time)) + # print(images.shape, target.shape) + + images = images.to(args.gpu) + target = target.to(args.gpu) + + # update score thresholds for global ep + if args.algo in ['global_ep', 'global_ep_iter']: + prune(model, update_thresholds_only=True) + + # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + if args.algo in ['hc', 'hc_iter', 'pt'] and i % args.project_freq == 0 and not args.differentiate_clamp: + for name, params in model.named_parameters(): + if "score" in name: + scores = params + with torch.no_grad(): + scores.data = torch.clamp(scores.data, 0.0, 1.0) + # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + + # compute output + if scaler is None: + output = model(images) + loss = criterion(output, target) + else: + with torch.cuda.amp.autocast(enabled=True): # mixed precision + output = model(images) + loss = criterion(output, target) + + if args.lam_finetune_loss > 0: + raise NotImplementedError # please check finetune_loss repo + + regularization_loss = torch.tensor(0) + # print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + if args.regularization: + regularization_loss =\ + get_regularization_loss(model, regularizer=args.regularization, + lmbda=args.lmbda, alpha=args.alpha, + alpha_prime=args.alpha_prime) + # print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + #print('regularization_loss: ', regularization_loss) + loss += regularization_loss + # print("(TRAINER)BEFORE ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # measure accuracy and record loss + acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) + # losses.update(loss.item(), images.size(0)) + # top1.update(acc1.item(), images.size(0)) + # top5.update(acc5.item(), images.size(0)) + # top10.update(acc10.item(), images.size(0)) + # compute weighted sum for each accuracy so we can average it later + top1 += acc1.item() * images.size(0) + top5 += acc5.item() * images.size(0) + top10 += acc10.item() * images.size(0) + num_images += images.size(0) + + # print("(TRAINER)AFTER ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + + # compute gradient and do SGD step + optimizer.zero_grad() + if scaler is None: + loss.backward() + optimizer.step() + else: + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() + + # measure elapsed time + # batch_time.update(time.time() - end) + batch_time = time.time() - end + end = time.time() + + if i % args.print_freq == 0: + t = (num_batches * epoch + i) * batch_size + # progress.display(i) + # progress.write_to_tensorboard( + # writer, prefix="train", global_step=t) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) + + print("(TRAINER)AFTER TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + # before completing training, clean up model based on latest scores + # update score thresholds for global ep + if args.algo in ['global_ep', 'global_ep_iter']: + prune(model, update_thresholds_only=True) + if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: + # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + for name, params in model.named_parameters(): + if "score" in name: + scores = params + with torch.no_grad(): + scores.data = torch.clamp(scores.data, 0.0, 1.0) + # print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + + return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() + + +def validate(val_loader, model, criterion, args, writer, epoch): + # batch_time = AverageMeter("Time", ":6.3f", write_val=False) + # losses = AverageMeter("Loss", ":.3f", write_val=False) + # top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) + # top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) + # top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) + # progress = ProgressMeter( + # len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " + # ) + top1 = 0 + top5 = 0 + top10 = 0 + num_images = 1 + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + # for i, (images, target) in tqdm.tqdm( + # enumerate(val_loader), ascii=True, total=len(val_loader) + # ): + # time.sleep(0.1) + # return -1, -1, -1 + for i, (images, target) in enumerate(val_loader): + continue + images = images.to(args.gpu) + target = target.to(args.gpu) + + #print(images.shape, target.shape) + + # compute output + # output = model(images) + + # loss = criterion(output, target) + loss = torch.Tensor([0]) + + # measure accuracy and record loss + # acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) + acc1, acc5, acc10 = torch.Tensor([5]), torch.Tensor([5]), torch.Tensor([5]) + # losses.update(loss.item(), images.size(0)) + # top1.update(acc1.item(), images.size(0)) + # top5.update(acc5.item(), images.size(0)) + # top10.update(acc10.item(), images.size(0)) + # compute weighted sum for each accuracy so we can average it later + top1 += acc1.item()# * images.size(0) + top5 += acc5.item()# * images.size(0) + top10 += acc10.item()# * images.size(0) + num_images += 1#images.size(0) + + # measure elapsed time + # batch_time.update(time.time() - end) + batch_time = time.time() - end + end = time.time() + + if i % args.print_freq == 0: + # progress.display(i) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) + + # progress.display(len(val_loader)) + + # if writer is not None: + # progress.write_to_tensorboard( + # writer, prefix="test", global_step=epoch) + + print("Model top1 Accuracy: {}".format(top1/num_images)) + return top1/num_images, top5/num_images, top10/num_images + + if __name__ == "__main__": main() diff --git a/main_utils.py b/main_utils.py index 1cf44d9f..9de7f53b 100644 --- a/main_utils.py +++ b/main_utils.py @@ -51,6 +51,19 @@ import copy from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors +import time +import torch +# import tqdm +import copy +import pdb + +from utils.eval_utils import accuracy +# from utils.logging import AverageMeter, ProgressMeter +from utils.net_utils import get_regularization_loss, prune, get_layers + +from torch import optim +import psutil, sys + def print_layers(parser_args, model): @@ -319,7 +332,7 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b 20, 40], gamma=0.1) # NOTE: hard-coded ''' - train, validate, modifier = get_trainer(parser_args) + # train, validate, modifier = get_trainer(parser_args) # check the performance of loaded model (after rounding) if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: From 985355508b2831e08164c46562f67176fffd8567 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 9 May 2022 14:53:56 -0500 Subject: [PATCH 046/113] trying to see if imports are the issue --- ddp_debug_main.py | 192 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 ddp_debug_main.py diff --git a/ddp_debug_main.py b/ddp_debug_main.py new file mode 100644 index 00000000..5dc805bd --- /dev/null +++ b/ddp_debug_main.py @@ -0,0 +1,192 @@ +from args_helper import parser_args +import psutil + + +def main(): + print(parser_args) + print("\n\nBeginning of process.") + print_time() + set_seed(parser_args.seed * parser_args.trial_num) + # set_seed(parser_args.seed + parser_args.trial_num - 1) + + # world size = ngpus_per_node since we are assuming single node + ngpus_per_node = torch.cuda.device_count() + + if parser_args.multiprocessing_distributed: + # assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" + mp.spawn(main_worker, args=(ngpus_per_node,), nprocs=ngpus_per_node, join=True) + else: + # Simply call main_worker function + main_worker(parser_args.gpu, ngpus_per_node) + + +def setup_distributed(rank, ngpus_per_node): + os.environ['MASTER_ADDR'] = '127.0.0.1' + os.environ['MASTER_PORT'] = '{}'.format(parser_args.port) + + dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) + +def main_worker(gpu, ngpus_per_node): + parser_args.gpu = gpu + if parser_args.multiprocessing_distributed: + parser_args.rank = parser_args.gpu + setup_distributed(parser_args.rank, ngpus_per_node) + # if using ddp, divide batch size per gpu + parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) + + + #########################DATA LOADING CODE######################### + from torchvision import datasets, transforms + import torch.multiprocessing + from torch.utils.data import random_split + torch.multiprocessing.set_sharing_strategy("file_system") + data_root = parser_args.data + + traindir = os.path.join(data_root, 'train') + valdir = os.path.join(data_root, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if parser_args.use_full_data: + train_dataset = dataset + # use_full_data => we are not tuning hyperparameters + validation_dataset = test_dataset + else: + # train_size = 1000 + # val_size = len(dataset) - train_size + val_size = 10000 + train_size = len(dataset) - val_size + train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) + + if parser_args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + val_dataset = datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])) + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=parser_args.batch_size, + shuffle=(train_sampler is None), + num_workers=parser_args.num_workers, + pin_memory=True, sampler=train_sampler) + + val_loader = torch.utils.data.DataLoader( + val_dataset, + batch_size=parser_args.batch_size, shuffle=False, + num_workers=parser_args.num_workers, pin_memory=True) + + actual_val_loader = torch.utils.data.DataLoader( + validation_dataset, + batch_size=parser_args.batch_size, shuffle=False, + num_workers=parser_args.num_workers, pin_memory=True + ) + + #########################DATA LOADING CODE######################### + + + for epoch in range(1, 20): + print("STARTING TRAINING: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + if parser_args.multiprocessing_distributed: + train_loader.sampler.set_epoch(epoch) + + print("Skipping training, just gonna round") + print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, + #ratio=parser_args.noise_ratio, rank=parser_args.gpu) + if True:#(parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + my_validate(actual_val_loader) + acc1 = -1 + else: + acc1 = -1 + print("GPU: {} | acc1={}".format(parser_args.gpu, acc1)) + print("After Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + dist.barrier() + continue + + print("GPU:{} | WE DID IT!!!") + +def my_validate(val_loader): + # batch_time = AverageMeter("Time", ":6.3f", write_val=False) + # losses = AverageMeter("Loss", ":.3f", write_val=False) + # top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) + # top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) + # top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) + # progress = ProgressMeter( + # len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " + # ) + top1 = 0 + top5 = 0 + top10 = 0 + num_images = 1 + + # switch to evaluate mode + + with torch.no_grad(): + end = time.time() + # for i, (images, target) in tqdm.tqdm( + # enumerate(val_loader), ascii=True, total=len(val_loader) + # ): + # time.sleep(0.1) + # return -1, -1, -1 + for i, (images, target) in enumerate(val_loader): + continue + images = images.to(args.gpu) + target = target.to(args.gpu) + + #print(images.shape, target.shape) + + # compute output + # output = model(images) + + # loss = criterion(output, target) + loss = torch.Tensor([0]) + + # measure accuracy and record loss + # acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) + acc1, acc5, acc10 = torch.Tensor([5]), torch.Tensor([5]), torch.Tensor([5]) + # losses.update(loss.item(), images.size(0)) + # top1.update(acc1.item(), images.size(0)) + # top5.update(acc5.item(), images.size(0)) + # top10.update(acc10.item(), images.size(0)) + # compute weighted sum for each accuracy so we can average it later + top1 += acc1.item()# * images.size(0) + top5 += acc5.item()# * images.size(0) + top10 += acc10.item()# * images.size(0) + num_images += 1#images.size(0) + + # measure elapsed time + # batch_time.update(time.time() - end) + batch_time = time.time() - end + end = time.time() + + if i % args.print_freq == 0: + # progress.display(i) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) + + # progress.display(len(val_loader)) + + # if writer is not None: + # progress.write_to_tensorboard( + # writer, prefix="test", global_step=epoch) + + print("Model top1 Accuracy: {}".format(top1/num_images)) + return top1/num_images, top5/num_images, top10/num_images + + +if __name__ == "__main__": + main() \ No newline at end of file From a767fb1c06244287d65c038c8aba2fef5576c6aa Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 9 May 2022 20:01:21 +0000 Subject: [PATCH 047/113] removing more imports --- ddp_debug_main.py | 40 ++++++++++++++++++++++++++++++++++++++-- imagenet_exec.sh | 2 +- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index 5dc805bd..098f7954 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -1,3 +1,33 @@ +import pdb +import numpy as np +import os +import pathlib +import random +import time +import pandas as pd +# from torch.utils.tensorboard import SummaryWriter +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.optim +import torch.utils.data +import torch.distributed as dist +import torch.multiprocessing as mp + +import sys +import re + + +import copy +import time +import torch +# import tqdm +import copy +import pdb + +from torch import optim +import psutil, sys from args_helper import parser_args import psutil @@ -67,7 +97,7 @@ def main_worker(gpu, ngpus_per_node): train_size = len(dataset) - val_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) - if parser_args.distributed: + if parser_args.multiprocessing_distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None @@ -187,6 +217,12 @@ def my_validate(val_loader): print("Model top1 Accuracy: {}".format(top1/num_images)) return top1/num_images, top5/num_images, top10/num_images +def print_time(): + print("\n\n--------------------------------------") + print("TIME: The current time is: {}".format(time.ctime())) + print("TIME: The current time in seconds is: {}".format(time.time())) + print("--------------------------------------\n\n") if __name__ == "__main__": - main() \ No newline at end of file + main() + diff --git a/imagenet_exec.sh b/imagenet_exec.sh index baf2d31b..1b899083 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -25,5 +25,5 @@ BLOCK conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" log_root="resnet50_sp5" log_end="_log" -CUDA_VISIBLE_DEVICES=0,1,2,3 python main.py \ +CUDA_VISIBLE_DEVICES=0,1,2,3 python ddp_debug_main.py \ --config "$conf_file" > "$log_root$log_end" 2>&1 & From df656bc72fdc655d626a902d8222d95c0c174fcf Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 9 May 2022 15:03:39 -0500 Subject: [PATCH 048/113] adding set_seed --- ddp_debug_main.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index 098f7954..42f46fd3 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -50,6 +50,21 @@ def main(): main_worker(parser_args.gpu, ngpus_per_node) +# set seed for experiment +def set_seed(seed): + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + # set this=True if you want deterministic runs + torch.backends.cudnn.deterministic = False + # set this=False if you want deterministic runs + torch.backends.cudnn.benchmark = True + print("Seeded everything: {}".format(seed)) + + def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' os.environ['MASTER_PORT'] = '{}'.format(parser_args.port) From 1eeab6c81a26e648cf8dfb55a951914ca4137de6 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 9 May 2022 15:29:30 -0500 Subject: [PATCH 049/113] removing parser_args --- ddp_debug_main.py | 957 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 956 insertions(+), 1 deletion(-) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index 42f46fd3..125bb3b2 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -28,11 +28,966 @@ from torch import optim import psutil, sys -from args_helper import parser_args import psutil +parser = argparse.ArgumentParser(description="Pruning random networks") +# Config/Hyperparameters +parser.add_argument( + "--data", + default="data/datasets/", + help="path to dataset base directory" +) +parser.add_argument( + "--log-dir", + default=None, + help="Where to save the runs. If None use ./runs" + ) +parser.add_argument( + "--name", + default=None, + type=str, + help="Name of experiment" +) +parser.add_argument( + "--config", + default='configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_1_4_highreg.yml', + help="Config file to use" +) +parser.add_argument( + "--batch-size", + type=int, + default=64, + metavar="N", + help="Input batch size for training (default: 64)" +) +parser.add_argument( + "--epochs", + type=int, + default=10, + metavar="N", + help="Number of epochs to use in training (default: 10)" +) +parser.add_argument( + "--submask-size", + type=int, + default=1, + metavar="S", + help="Size of random 0/1 submask to create among a random set of weights in the network (default: 1)" +) +parser.add_argument( + "--metric", + type=str, + default="loss", + metavar="M", + help="Metric used to determine whether a weight or activation should be pruned (default: \"loss\")" +) +parser.add_argument( + "--pruning-strategy", + type=str, + default=None, + metavar="PS", + help="Strategy for pruning. Can be \"weights\", \"activations\", \"activations_and_weights\", \"simulated_annealing\", or None. If None, train a model via vanilla SGD (default: None)" +) +parser.add_argument( + "--how-to-prune", + type=str, + default="random", + help="How the pruning strategy will be executed. Can be \"random\", \"random_with_replacement\", \"layer\", or \"layer_reversed.\" This argument has changes how the weights or activations in the network are selected for pruning. \"random\" will choose the weights/activations without replacement, \"random_with_replacement\" will choose the weights/activations with replacement, and \"layer\" will choose every weight/activation in the network in layer by layer, and \"layer_reversed\" is the same as \"layer,\" but will start with the last layer and move to the first (default: \"random\")" +) +parser.add_argument( + "--start-from-nothing", + action="store_true", + default=False, + help="Prunes all weights in the network, but leaves one weight per layer to connect the input layer to the output layer. Only applies to masked layers (default: False)" +) +parser.add_argument( + "--flips", + type=list, + default=None, + nargs="+", + metavar="R", + help="List of epoch indices (starting from and including 0) where flipping occurs. At each milestone, 10% of the mask parameters are randomly chosen and they are flipped. Restarting is accomplished at the beginning of an epoch. Applies only when pruning weights (default: None)" +) +parser.add_argument( + "--lr", + type=float, + default=0.001, + metavar="LR", + help="Learning rate (default: 0.001)" +) +parser.add_argument( + "--lr-policy", + type=str, + default=None, + help="Learning rate scheduler" +) +parser.add_argument( + "--lr-gamma", + type=float, + default=0.1, + help="Multistep lr decay ratio (default: 0.1)" +) +parser.add_argument( + "--lr-adjust", + type=int, + default=50, + help="Multistep lr decay period (default: 50)" +) + +parser.add_argument( + "--momentum", + type=float, + default=0.9, + metavar="M", + help="Momentum (default: 0.9)" +) +parser.add_argument( + "--wd", + type=float, + default=0.0005, + metavar="WD", + help="Weight decay (default: 0.0005)" +) +parser.add_argument( + "--nesterov", + type=bool, + default=False, + metavar="N", + help="Nesterov acceleration (default: False)" +) +parser.add_argument( + "--milestones", + type=list, + default=[50, 100, 150, 200], + nargs="+", + metavar="M", + help="List of milestones where learning rate is multiplied by args.gamma (default: [50, 100, 150, 200])" +) +parser.add_argument( + "--gamma", + type=float, + default=0.1, + metavar="G", + help="Multiplicative factor to reduce the learning rate at every milestone (default: 0.1)" +) +parser.add_argument( + "--td", + type=float, + default=0.99, + metavar="TD", + help="Temperature decay constant for simulated annealing (default: 0.99)" +) +parser.add_argument( + "--temp", + type=int, + default=100000, + metavar="T", + help="Temperature used in simulated annealing (default: 100000)" +) +parser.add_argument( + "--max-iter", + type=int, + default=100000, + metavar="MI", + help="Maximum number of iterations to run simulated annealing for before terminating. It's recommended to set this to a value that at is at least as big as the number of parameters in the network (default: 100000)" +) +parser.add_argument( + "--algo", + type=str, + default='ep', + help="pruning algo to use |ep|pt_hack|pt_reg|hc|ep+greedy|greedy+ep|hc_iter|global_ep|global_ep_iter|imp" +) +parser.add_argument( + "--iter_start", + type=int, + default=0, + help="starting epoch for iterative pruning" +) +parser.add_argument( + "--iter-period", + type=int, + default=5, + help="period [epochs] for iterative pruning" +) +parser.add_argument( + "--optimizer", + type=str, + default='sgd', + help="optimizer option to use |sgd|adam|" +) +parser.add_argument( + '--evaluate-only', + action='store_true', + default=False, + help='just use rounding techniques to evaluate a saved model' +) +parser.add_argument( + "--round", + type=str, + default='naive', + help='rounding technique to use |naive|prob|pb|' + # naive: threshold(0.5), prob: probabilistic rounding, pb: pseudo-boolean paper's choice (RoundDown) +) +parser.add_argument( + '--noise', + action='store_true', + default=False, + help='flag that decides if we add noise to the rounded p_i' +) +parser.add_argument( + "--noise-ratio", + type=float, + default=0.0, + help="portion of score flipping" +) +parser.add_argument( + "--score-init", + type=str, + default="unif", + help="initial score for hypercube |unif|bern|" +) +parser.add_argument( + "--interpolate", + type=str, + default="prob", + help="way of interpolating masks/weights |prob|linear|" +) +parser.add_argument( + '--plot-hc-convergence', + action='store_true', + default=False, + help='flag that decides if we plot convergence of hc' +) +parser.add_argument( + "--hc-warmup", + default=9999, + type=int, + help="warmup epochs for hypercube" +) +parser.add_argument( + "--hc-period", + default=1, + type=int, + help="rounding period for hypercube" +) +parser.add_argument( + "--num-round", + type=int, + default=1, + help='number of different models testing in rounding' +) +# do we need it? +parser.add_argument( + "--num-test", + type=int, + default=1, + help='number of different models testing in prob rounding' +) +parser.add_argument( + "--save-model", + action='store_true', + default=False, + help='For Saving the current Model' +) +# Architecture and training +parser.add_argument( + "--arch", + type=str, + default="TwoLayerFC", + # KS: gotta find a better way to do this. causing circular import issues + # help="Model architecture: " + " | ".join(models.__dict__["__all__"]) + " | (default: TwoLayerFC)" +) +parser.add_argument( + "--width", + type=float, + default=1.0, + help="portion of additional width compared with original width" +) + + +parser.add_argument( + "--hidden-size", + type=int, + default=500, + metavar="H", + help="Number of nodes in the FC layers of the network" +) +parser.add_argument( + "--bias", + action="store_true", + default=False, + help="Boolean flag to indicate whether to use bias" +) +parser.add_argument( + "--freeze-weights", + action="store_true", + default=True, + help="Boolean flag to indicate whether weights should be frozen. Used when sparsifying (default: False)" +) +parser.add_argument( + "--conv-type", + type=str, + default=None, + help="What kind of sparsity to use" +) +parser.add_argument( + "--mode", + default="fan_in", + help="Weight initialization mode" +) +parser.add_argument( + "--nonlinearity", + default="relu", + help="Nonlinearity used by initialization" +) +parser.add_argument( + "--bn-type", + default=None, + help="BatchNorm type" +) +parser.add_argument( + "--init", + default="kaiming_normal", + help="Weight initialization modifications" +) +parser.add_argument( + "--no-bn-decay", + action="store_true", + default=False, + help="No batchnorm decay" +) +parser.add_argument( + "--scale-fan", + action="store_true", + default=False, + help="scale fan" +) +parser.add_argument( + "--first-layer-dense", + action="store_true", + help="First layer dense or sparse" +) +parser.add_argument( + "--dense-training", + action="store_true", + help="Train the network without pruning, and checkpoint along the way" +) +parser.add_argument( + "--last-layer-dense", + action="store_true", + help="Last layer dense or sparse" +) +parser.add_argument( + "--label-smoothing", + type=float, + help="Label smoothing to use, default 0.0", + default=None, +) +parser.add_argument( + "--first-layer-type", + type=str, + default=None, + help="Conv type of first layer" +) +parser.add_argument( + "--trainer", type=str, default="default", help="cs, ss, or standard training" +) +parser.add_argument( + "--score-init-constant", + type=float, + default=None, + help="Sample Baseline Subnet Init", +) +# represents percentage of weights THAT REMAIN each time ep, global_ep +# whereas it represents number of weights TO PRUNE when calling prune() +parser.add_argument( + "--prune-rate", + default=0.5, + help="Decides number of weights that REMAIN after sparse training.", + type=float, +) +parser.add_argument( # add for bottom K iterative pruning + "--prune-type", + default="FixThresholding", + help="Type of prune - fix thresholding (FixTHresholding), or prune bottem k percent (BottomK)", + type=str, +) +parser.add_argument( + "--dataset", + type=str, + default="MNIST", + help="Dataset to train the model with. Can be CIFAR10 or MNIST (default: MNIST)" +) +parser.add_argument( + "--loss", + type=str, + default="cross-entropy-loss", + help="which loss to use for pruning: cross-entropy-loss or zero-one-loss" +) +# # Save/Load +# parser.add_argument( +# "--save-dir", +# type=str, +# default=False, +# help="Directory to save the results of the experiment (default: \"./results\")" +# ) +parser.add_argument( + "--save-plot-data", + action="store_true", + default=False, + help="Boolean flag to indicate whether to save training data for plotting. The following data will be saved at every epoch: top-1 training set accuracy, top-1 validation set accuracy, epoch number (default: False)" +) +parser.add_argument( + "--data-dir", + type=str, + default="../data", + help="Directory of dataset to load in" +) +parser.add_argument( + "--load-ckpt", + type=str, + default=None, + help="Path to checkpoint to load from" +) +parser.add_argument( + "--log-interval", + type=int, + default=2, + metavar="N", + help="The number of batches to wait before logging training status. When pruning, this is the number of epochs before logging status (default: 100)" +) +parser.add_argument( + "--ckpt-interval", + type=int, + default=-1, + help="The number of epochs to train before saving the next checkpoint" +) +# Device settings +parser.add_argument( + "--seed", + type=int, + default=42, + metavar="S", + help="Random seed (default: 42)" +) +parser.add_argument( + "--seed-fixed-init", + type=int, + default=24, + metavar="S", + help="Random seed when used for fixing weight/score init (default:24)" +) +parser.add_argument( + "--trial-num", + type=int, + default=1, + help="Trial number (1,2, ...)" +) +parser.add_argument( + "--fixed-init", + action="store_true", + default=False, + help="fixed weight initialization" +) +parser.add_argument( + "--mode-connect", + action="store_true", + default=False, + help="Boolean flag to indicate whether to run mode connectivity" +) +parser.add_argument( + '--mode-connect-filename', + type=str, + default=None, + help='filename for state_dict used for mode connectivity' +) +parser.add_argument( + '--how-to-connect', + type=str, + default='prob', + help="procedure for interpolating the mask. Can be \"random\", which chooses a binary value for a mask coordinate with bern(alpha), \"score\" which uses the [0, 1] score for each coordinate for interpolation (the mask consists of continuous values rather than binary in this case), and \"round\" which is uses the mask obtained from the \"score\" option and applies naive rounding." +) +parser.add_argument( + "--no-cuda", + action="store_true", + default=False, + help="Disables CUDA training" +) +parser.add_argument( + "--gpu", + type=int, + default=0, + metavar="G", + help="Override the default choice for a CUDA-enabled GPU by specifying the GPU\"s integer index (i.e. \"0\" for \"cuda:0\")" +) +parser.add_argument( + "--num-workers", + type=int, + default=4, + metavar="W", + help="Number of workers" +) +parser.add_argument( + "--shift", + type=float, + default=0.0, + help="shift portion" +) +parser.add_argument( + "--num-trial", + type=int, + default=1, + help="number of trials for testing sharpness" +) +# WARNING: With DataParallel, this causes some issues +parser.add_argument( + "--pretrained", + dest="pretrained", + default=None, + type=str, + help="use pre-trained model", +) +parser.add_argument( + "--pretrained2", + dest="pretrained2", + default=None, + type=str, + help="use pre-trained model 2", +) +parser.add_argument( + "--save_every", + default=-1, + type=int, + help="Save every ___ epochs" +) +''' +# @ksreenivasan: commenting this for now. I think EP uses it. +parser.add_argument( + "--random-subnet", + action="store_true", + help="Whether or not to use a random subnet when fine tuning for lottery experiments", +) +''' +parser.add_argument( + "-e", + "--evaluate", + dest="evaluate", + action="store_true", + help="evaluate model on validation set", +) +parser.add_argument( + "--compare-rounding", + dest="compare-rounding", + action="store_true", + help="compare different rounding schemes", +) +parser.add_argument( + "--resume", + default=None, + type=str, + metavar="PATH", + help="path to latest checkpoint (default: none)", + ) +parser.add_argument( + "--width-mult", + default=1.0, + help="How much to vary the width of the network.", + type=float, +) +parser.add_argument( + "--start-epoch", + default=None, + type=int, + metavar="N", + help="manual epoch number (useful on restarts)", +) +parser.add_argument( + "--warmup_length", + default=0, + type=int, + help="Number of warmup iterations" +) +parser.add_argument( + "-p", + "--print-freq", + default=10, + type=int, + metavar="N", + help="print frequency (default: 10)", +) +parser.add_argument( + '--results-filename', + type=str, + default=None, + help='csv results filename' +) +parser.add_argument( + '--weight-training', + action='store_true', + default=False, + help='flag that decides if we are doing pruning or weight training' +) +parser.add_argument( + '--regularization', + default=None, + type=str, + help='which regularizer to add : |var_red_1|var_red_2|bin_cross_entropy|' +) +""" +var_red_1: lmbda * p^(alpha) (1-p)^(alpha') +var_red_2: w^2 p(1-p) +bin_cross_entropy: -plog(1-p)? +""" +parser.add_argument( + '--lmbda', + type=float, + default=0.001, + help='regularization coefficient lambda' +) +parser.add_argument( + "--alpha", + default=1.0, + type=float, + help="first exponent in regularizer", +) +parser.add_argument( + "--alpha_prime", + default=1.0, + type=float, + help="second exponent in regularizer", +) + +# Distributed training +parser.add_argument( + "--world-size", + default=-1, + type=int, + help="number of nodes for distributed training" +) +parser.add_argument( + "--rank", + default=-1, + type=int, + help="node rank for distributed training" +) +parser.add_argument( + "--dist-backend", + default="nccl", + type=str, + help="distributed backend" +) +parser.add_argument( + "--multiprocessing-distributed", + action="store_true", + help="Use multi-processing distributed training to launch " + "N processes per node, which has N GPUs. This is the " + "fastest way to use PyTorch for either single node or " + "multi node data parallel training" +) +parser.add_argument( + "--random-subnet", + action="store_true", + default=False, + help="Just initializes random subnetwork and then trains" +) +parser.add_argument( + "--hc-quantized", + action="store_true", + default=False, + help="round probablities in every iteration" +) +parser.add_argument( + "--quantize-threshold", + default=0.5, + type=float, + help="threhsold to use while quantizing scores in HC", +) +parser.add_argument( + "--checkpoint-at-prune", + action="store_true", + default=False, + help="save checkpoints every time we prune" +) +parser.add_argument( + "--skip-sanity-checks", + action="store_true", + default=False, + help="Enable this to skip sanity checks (save time)" +) +parser.add_argument( + "--skip-fine-tune", + action="store_true", + default=False, + help="Enable this to skip fine tuning (get pure pruned network)" +) +parser.add_argument( + "--shuffle", + action="store_true", + default=False, + help="shuffle weights/masks before sanity check" +) +parser.add_argument( + "--reinit", + action="store_true", + default=False, + help="reinit weights/masks before sanity check" +) +parser.add_argument( + "--chg_mask", + action="store_true", + default=False, + help="chg masks before sanity check" +) +parser.add_argument( + "--chg_weight", + action="store_true", + default=False, + help="chg weights before sanity check" +) +parser.add_argument( + "--fine-tune-optimizer", + type=str, + default='sgd', + help="optimizer option to use |sgd|adam| for fine-tuning weights" +) +parser.add_argument( + "--fine-tune-lr-policy", + type=str, + default=None, + help="Learning rate scheduler (for finetune)" +) +parser.add_argument( + "--fine-tune-lr", + type=float, + default=0.01, + metavar="LR", + help="Learning rate for fine-tuning weights" +) +parser.add_argument( + "--fine-tune-wd", + type=float, + default=0.0001, + metavar="WD", + help="Weight decay for fine-tuning weights" +) +# parser.add_argument( +# "--multigpu", +# default=None, +# type=str, +# help="Which GPUs to use for multigpu training, comma separated" +# ) +parser.add_argument( + "--rewind-score", + action="store_true", + default=False, + help="if set True, every time when we prune, we set the score back to the initial state" +) +parser.add_argument( + "--rewind-to-epoch", + default=-1, + type=int, + help="to rewind to some epoch, you have to explicitly set the argument. Otherwise the code will never cache the rewinded score" +) +parser.add_argument( + "--differentiate-clamp", + action="store_true", + default=False, + help="if set True, we project to [0, 1] in forward and therefore differentiate clamp " +) +parser.add_argument( + "--project-freq", + default=1, + type=int, + help="project scores to [0, 1] every k gradient steps" +) +parser.add_argument( + "--run_idx", + default=None, + help="index of run used for counting yml/log/save_folder" +) +parser.add_argument( + "--subfolder", + default=None, + help="subfolder within the location for saving the results" +) +# added parser args for IMP +parser.add_argument( + "--imp_rewind_iter", + default=1000, + type=int, + help="which iterations to rewind to" +) +parser.add_argument( + "--imp-resume-round", + default=-1, + type=int, + help="which round to resume to" +) +parser.add_argument( + "--imp-resume-epoch", + default=-1, + type=int, + help="which epoch to resume to" +) +parser.add_argument( + "--imp-resume-iter", + default=-1, + type=int, + help="which iter to resume to" +) +parser.add_argument( + "--imp-rewind-model", + default="short_imp/Liu_checkpoint_model_correct.pth" +) +parser.add_argument( + "--imp-no-rewind", + action="store_true", + default=False, + help="if set True, we run IMP algorithm without rewinding to previous states" +) +parser.add_argument( + "--imp-rounds", + type=int, + default=-1, + help="if set > 0, then ignore the epochs statement, and calculate epochs based on rounds * iter / round" +) +parser.add_argument( + "--smart_ratio", + type=float, + default=-1, + help="the pruning weights in [0, 1]. E.g. smart_ratio = 0.98 will end up with a 2\% weight remaining model" +) +parser.add_argument( + "--bottom-k-on-forward", + action="store_true", + default=False, + help="Enable this to use bottomK on forward for HC" +) +parser.add_argument( + "--target-sparsity", + default=0.5, + help="decides max percentage of weights that remain at the end of training", + type=float, +) +parser.add_argument( + "--lam_finetune_loss", + type=float, + default=-1, + help="lambda for finetune loss " +) +parser.add_argument( + "--num_step_finetune", + type=int, + default=10, + help="number of steps to check finetune loss " +) +parser.add_argument( + "--unflag-before-finetune", + action="store_true", + default=False, + help="Enable this to unprune weights if possible, before fine-tune" +) +parser.add_argument( + "--override-prune-rate", + action="store_true", + default=False, + help="Enable this to specify prune-rate manually" +) +parser.add_argument( + "--mixed_precision", + type=int, + default=0, + help="Use mixed precision or not" +) +parser.add_argument('--transformer_emsize', + type=int, default=200, + help='size of word embeddings' +) +parser.add_argument('--transformer_nhid', + type=int, + default=200, + help='number of hidden units per layer' +) +parser.add_argument('--transformer_nlayers', + type=int, + default=2, + help='number of layers' +) +parser.add_argument('--transformer_clip', + type=float, + default=0.25, + help='gradient clipping' +) +parser.add_argument('--transformer_bptt', + type=int, + default=35, + help='sequence length' +) +parser.add_argument('--transformer_dropout', + type=float, + default=0.2, + help='dropout applied to layers (0 = no dropout)' +) +parser.add_argument('--transformer_nhead', + type=int, + default=2, + help='the number of heads in the encoder/decoder of the transformer model' +) +parser.add_argument( + "--only-sanity", + action="store_true", + default=False, + help="Only run sanity checks on the files in specific directory or subdirectories" +) +parser.add_argument( + "--invert-sanity-check", + action="store_true", + default=False, + help="Enable this to run the inverted sanity check (for HC)" +) +parser.add_argument( + "--sanity-folder", + default=None, + type=str, + metavar="PATH", + help="directory(s) to access for only sanity check", +) +parser.add_argument( + "--sr-version", + default=1, + type=int, + help="smart ratio version number (1, 2, ...)", +) +parser.add_argument( + "--use-full-data", + action="store_true", + default=False, + help="Enable this use full train data and not leave anything for validation" +) +parser.add_argument( + "--port", + default=29500, + type=int, + help="Specify port to use for DDP", +) + def main(): + parser_args = parser.parse_args() + # get commands from command line + override_args = _parser.argv_to_vars(sys.argv) + + # load yaml file + yaml_txt = open(parser_args.config).read() + + # override args + loaded_yaml = yaml.load(yaml_txt, Loader=yaml.FullLoader) + for v in override_args: + loaded_yaml[v] = getattr(parser_args, v) + + print(f"=> Reading YAML config from {parser_args.config}") + parser_args.__dict__.update(loaded_yaml) + print(parser_args) print("\n\nBeginning of process.") print_time() From af1a52a1b973e0027ceeaf509c7e9262114e85b2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 9 May 2022 21:09:43 +0000 Subject: [PATCH 050/113] some more minor attempts at fixes --- .../resnet50/imagenet/resnet50_sparsity_5.yml | 2 +- data/imagenet.py | 23 +++-- ddp_debug_main.py | 16 ++-- main.py | 90 +++++-------------- 4 files changed, 43 insertions(+), 88 deletions(-) diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml index a7cf06bd..539940f1 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml @@ -56,7 +56,7 @@ regularization: L2 lmbda: 0.000001 # 1e-6 # ===== Hardware setup ===== # -workers: 12 +workers: 8 multiprocessing_distributed: True mixed_precision: True # gpu: 1 diff --git a/data/imagenet.py b/data/imagenet.py index 65f7d934..812f6ffb 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -3,16 +3,13 @@ import torch from torchvision import datasets, transforms import torch.multiprocessing -from args_helper import parser_args from torch.utils.data import random_split -torch.multiprocessing.set_sharing_strategy("file_system") - class ImageNet: def __init__(self, args): super(ImageNet, self).__init__() - data_root = parser_args.data + data_root = args.data traindir = os.path.join(data_root, 'train') valdir = os.path.join(data_root, 'val') @@ -28,7 +25,7 @@ def __init__(self, args): normalize, ])) - if parser_args.use_full_data: + if args.use_full_data: train_dataset = dataset # use_full_data => we are not tuning hyperparameters validation_dataset = test_dataset @@ -39,7 +36,7 @@ def __init__(self, args): train_size = len(dataset) - val_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) - if parser_args.multiprocessing_distributed: + if args.multiprocessing_distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None @@ -52,18 +49,18 @@ def __init__(self, args): ])) self.train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=parser_args.batch_size, - shuffle=(train_sampler is None), - num_workers=parser_args.num_workers, + train_dataset, batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers, pin_memory=True, sampler=train_sampler) self.val_loader = torch.utils.data.DataLoader( val_dataset, - batch_size=parser_args.batch_size, shuffle=False, - num_workers=parser_args.num_workers, pin_memory=True) + batch_size=args.batch_size, shuffle=False, + num_workers=args.num_workers, pin_memory=True) self.actual_val_loader = torch.utils.data.DataLoader( validation_dataset, - batch_size=parser_args.batch_size, shuffle=False, - num_workers=parser_args.num_workers, pin_memory=True + batch_size=args.batch_size, shuffle=False, + num_workers=args.num_workers, pin_memory=True ) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index 125bb3b2..385d0299 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -29,6 +29,12 @@ from torch import optim import psutil, sys import psutil +import argparse +import sys +import yaml +from configs import parser as _parser + +global parser_args parser = argparse.ArgumentParser(description="Pruning random networks") # Config/Hyperparameters @@ -999,10 +1005,10 @@ def main(): if parser_args.multiprocessing_distributed: # assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" - mp.spawn(main_worker, args=(ngpus_per_node,), nprocs=ngpus_per_node, join=True) + mp.spawn(main_worker, args=(ngpus_per_node,parser_args), nprocs=ngpus_per_node, join=True) else: # Simply call main_worker function - main_worker(parser_args.gpu, ngpus_per_node) + main_worker(parser_args.gpu, ngpus_per_node, parser_args) # set seed for experiment @@ -1020,17 +1026,17 @@ def set_seed(seed): print("Seeded everything: {}".format(seed)) -def setup_distributed(rank, ngpus_per_node): +def setup_distributed(rank, ngpus_per_node, parser_args): os.environ['MASTER_ADDR'] = '127.0.0.1' os.environ['MASTER_PORT'] = '{}'.format(parser_args.port) dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) -def main_worker(gpu, ngpus_per_node): +def main_worker(gpu, ngpus_per_node, parser_args): parser_args.gpu = gpu if parser_args.multiprocessing_distributed: parser_args.rank = parser_args.gpu - setup_distributed(parser_args.rank, ngpus_per_node) + setup_distributed(parser_args.rank, ngpus_per_node, parser_args) # if using ddp, divide batch size per gpu parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) diff --git a/main.py b/main.py index 518e79f1..656cefe2 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,10 @@ from main_utils import * import psutil +from torchvision import datasets, transforms +import torch.multiprocessing +from torch.utils.data import random_split +torch.multiprocessing.set_sharing_strategy("file_system") + def main(): @@ -142,90 +147,32 @@ def main_worker(gpu, ngpus_per_node): return - #########################DATA LOADING CODE######################### - from torchvision import datasets, transforms - import torch.multiprocessing - from torch.utils.data import random_split - torch.multiprocessing.set_sharing_strategy("file_system") - data_root = parser_args.data - - traindir = os.path.join(data_root, 'train') - valdir = os.path.join(data_root, 'val') - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) - - dataset = datasets.ImageFolder( - traindir, - transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - normalize, - ])) - - if parser_args.use_full_data: - train_dataset = dataset - # use_full_data => we are not tuning hyperparameters - validation_dataset = test_dataset - else: - # train_size = 1000 - # val_size = len(dataset) - train_size - val_size = 10000 - train_size = len(dataset) - val_size - train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) - - if parser_args.distributed: - train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) - else: - train_sampler = None - - val_dataset = datasets.ImageFolder(valdir, transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - normalize, - ])) - - train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=parser_args.batch_size, - shuffle=(train_sampler is None), - num_workers=parser_args.num_workers, - pin_memory=True, sampler=train_sampler) - - val_loader = torch.utils.data.DataLoader( - val_dataset, - batch_size=parser_args.batch_size, shuffle=False, - num_workers=parser_args.num_workers, pin_memory=True) - - actual_val_loader = torch.utils.data.DataLoader( - validation_dataset, - batch_size=parser_args.batch_size, shuffle=False, - num_workers=parser_args.num_workers, pin_memory=True - ) - - #########################DATA LOADING CODE######################### - # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): print("STARTING TRAINING: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) # lr_policy(epoch, iteration=None) - modifier(parser_args, epoch, model) - cur_lr = get_lr(optimizer) + # modifier(parser_args, epoch, model) + # cur_lr = get_lr(optimizer) print("Skipping training, just gonna round") print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, #ratio=parser_args.noise_ratio, rank=parser_args.gpu) if True:#(parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - validate(actual_val_loader, model, criterion, parser_args, writer, epoch) + # print("Pretend to validate") + # continue + # do_something() + my_validate(data.actual_val_loader, None, criterion, parser_args, writer, epoch) acc1 = -1 else: acc1 = -1 print("GPU: {} | acc1={}".format(parser_args.gpu, acc1)) print("After Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) dist.barrier() + import gc; gc.collect() + torch.cuda.empty_cache() continue @@ -545,7 +492,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() -def validate(val_loader, model, criterion, args, writer, epoch): +def my_validate(val_loader, model, criterion, args, writer, epoch): # batch_time = AverageMeter("Time", ":6.3f", write_val=False) # losses = AverageMeter("Loss", ":.3f", write_val=False) # top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) @@ -560,8 +507,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): num_images = 1 # switch to evaluate mode - model.eval() - + #model.eval() with torch.no_grad(): end = time.time() # for i, (images, target) in tqdm.tqdm( @@ -570,6 +516,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): # time.sleep(0.1) # return -1, -1, -1 for i, (images, target) in enumerate(val_loader): + break continue images = images.to(args.gpu) target = target.to(args.gpu) @@ -613,6 +560,11 @@ def validate(val_loader, model, criterion, args, writer, epoch): print("Model top1 Accuracy: {}".format(top1/num_images)) return top1/num_images, top5/num_images, top10/num_images +def do_something(): + time.sleep(5) + print("SLEPT FOR 5s") + return 1 + if __name__ == "__main__": main() From 33580a20854e6fd964477f417c906df0b6a9704c Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 10 May 2022 22:59:47 -0500 Subject: [PATCH 051/113] checking vanilla data loader memory --- ddp_debug_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index 125bb3b2..4534bf16 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -1039,7 +1039,7 @@ def main_worker(gpu, ngpus_per_node): from torchvision import datasets, transforms import torch.multiprocessing from torch.utils.data import random_split - torch.multiprocessing.set_sharing_strategy("file_system") + # torch.multiprocessing.set_sharing_strategy("file_system") data_root = parser_args.data traindir = os.path.join(data_root, 'train') From d45d5141ed3578f29909af6a120b39874e7f188a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 11 May 2022 04:46:04 +0000 Subject: [PATCH 052/113] looks like vanilla data loader works! --- ddp_debug_main.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index acb79736..597b406c 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -1115,7 +1115,7 @@ def main_worker(gpu, ngpus_per_node, parser_args): # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, #ratio=parser_args.noise_ratio, rank=parser_args.gpu) if True:#(parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - my_validate(actual_val_loader) + my_validate(actual_val_loader, parser_args, epoch) acc1 = -1 else: acc1 = -1 @@ -1126,7 +1126,7 @@ def main_worker(gpu, ngpus_per_node, parser_args): print("GPU:{} | WE DID IT!!!") -def my_validate(val_loader): +def my_validate(val_loader, parser_args, epoch): # batch_time = AverageMeter("Time", ":6.3f", write_val=False) # losses = AverageMeter("Loss", ":.3f", write_val=False) # top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) @@ -1139,7 +1139,6 @@ def my_validate(val_loader): top5 = 0 top10 = 0 num_images = 1 - # switch to evaluate mode with torch.no_grad(): @@ -1150,9 +1149,8 @@ def my_validate(val_loader): # time.sleep(0.1) # return -1, -1, -1 for i, (images, target) in enumerate(val_loader): - continue - images = images.to(args.gpu) - target = target.to(args.gpu) + images = images.to(parser_args.gpu) + target = target.to(parser_args.gpu) #print(images.shape, target.shape) @@ -1180,9 +1178,9 @@ def my_validate(val_loader): batch_time = time.time() - end end = time.time() - if i % args.print_freq == 0: + if i % parser_args.print_freq == 0: # progress.display(i) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item(), batch_time)) # progress.display(len(val_loader)) From 5096528e65988d330c41d2286c4114f41d0824f8 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 11 May 2022 10:56:26 -0500 Subject: [PATCH 053/113] moving args outside main --- ddp_debug_main.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index acb79736..ddd9ce68 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -977,23 +977,23 @@ help="Specify port to use for DDP", ) +parser_args = parser.parse_args() +# get commands from command line +override_args = _parser.argv_to_vars(sys.argv) -def main(): - parser_args = parser.parse_args() - # get commands from command line - override_args = _parser.argv_to_vars(sys.argv) +# load yaml file +yaml_txt = open(parser_args.config).read() - # load yaml file - yaml_txt = open(parser_args.config).read() +# override args +loaded_yaml = yaml.load(yaml_txt, Loader=yaml.FullLoader) +for v in override_args: + loaded_yaml[v] = getattr(parser_args, v) - # override args - loaded_yaml = yaml.load(yaml_txt, Loader=yaml.FullLoader) - for v in override_args: - loaded_yaml[v] = getattr(parser_args, v) +print(f"=> Reading YAML config from {parser_args.config}") +parser_args.__dict__.update(loaded_yaml) - print(f"=> Reading YAML config from {parser_args.config}") - parser_args.__dict__.update(loaded_yaml) +def main(): print(parser_args) print("\n\nBeginning of process.") print_time() From d1e22fd7f043ea1cb39a7660c5844fd806ef25bb Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 11 May 2022 11:46:48 -0500 Subject: [PATCH 054/113] trying to avoid mp.spawn() --- ddp_debug_main.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index 45ff9b3d..892c5435 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -976,6 +976,12 @@ type=int, help="Specify port to use for DDP", ) +parser.add_argument( + "--rank", + default=0, + type=int, + help="Specify rank/gpu for DDP", +) parser_args = parser.parse_args() # get commands from command line @@ -1002,13 +1008,14 @@ def main(): # world size = ngpus_per_node since we are assuming single node ngpus_per_node = torch.cuda.device_count() + main_worker(rank, ngpus_per_node, parser_args) - if parser_args.multiprocessing_distributed: - # assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" - mp.spawn(main_worker, args=(ngpus_per_node,parser_args), nprocs=ngpus_per_node, join=True) - else: - # Simply call main_worker function - main_worker(parser_args.gpu, ngpus_per_node, parser_args) + # if parser_args.multiprocessing_distributed: + # # assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" + # mp.spawn(main_worker, args=(ngpus_per_node,parser_args), nprocs=ngpus_per_node, join=True) + # else: + # # Simply call main_worker function + # main_worker(parser_args.gpu, ngpus_per_node, parser_args) # set seed for experiment From a91bd0695910c030ba25aaa21a9364dc4ede2ade Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 11 May 2022 16:59:23 +0000 Subject: [PATCH 055/113] trying to avoid mp.spawn() in main also --- ddp_debug_main.py | 10 ++-------- imagenet_exec.sh | 10 +++++++--- main.py | 8 +++----- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/ddp_debug_main.py b/ddp_debug_main.py index 892c5435..9e90e9f1 100644 --- a/ddp_debug_main.py +++ b/ddp_debug_main.py @@ -672,7 +672,7 @@ ) parser.add_argument( "--rank", - default=-1, + default=0, type=int, help="node rank for distributed training" ) @@ -976,12 +976,6 @@ type=int, help="Specify port to use for DDP", ) -parser.add_argument( - "--rank", - default=0, - type=int, - help="Specify rank/gpu for DDP", -) parser_args = parser.parse_args() # get commands from command line @@ -1008,7 +1002,7 @@ def main(): # world size = ngpus_per_node since we are assuming single node ngpus_per_node = torch.cuda.device_count() - main_worker(rank, ngpus_per_node, parser_args) + main_worker(parser_args.rank, ngpus_per_node, parser_args) # if parser_args.multiprocessing_distributed: # # assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" diff --git a/imagenet_exec.sh b/imagenet_exec.sh index 1b899083..0a8c25c9 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -23,7 +23,11 @@ done BLOCK conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" -log_root="resnet50_sp5" +log_root="ddp_debug_" log_end="_log" -CUDA_VISIBLE_DEVICES=0,1,2,3 python ddp_debug_main.py \ - --config "$conf_file" > "$log_root$log_end" 2>&1 & +for gpu in 0 1 2 3 +do + python ddp_debug_main.py \ + --rank "$gpu" \ + --config "$conf_file" > "$log_root$gpu$log_end" 2>&1 & +done diff --git a/main.py b/main.py index 656cefe2..62028554 100644 --- a/main.py +++ b/main.py @@ -38,8 +38,8 @@ def main_worker(gpu, ngpus_per_node): # if using ddp, divide batch size per gpu parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) - train, validate, modifier = get_trainer(parser_args) - model = get_model(parser_args) + # train, validate, modifier = get_trainer(parser_args) + #model = get_model(parser_args) if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: idty_str = get_idty_str(parser_args) @@ -160,7 +160,7 @@ def main_worker(gpu, ngpus_per_node): print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, #ratio=parser_args.noise_ratio, rank=parser_args.gpu) - if True:#(parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: # print("Pretend to validate") # continue # do_something() @@ -171,8 +171,6 @@ def main_worker(gpu, ngpus_per_node): print("GPU: {} | acc1={}".format(parser_args.gpu, acc1)) print("After Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) dist.barrier() - import gc; gc.collect() - torch.cuda.empty_cache() continue From 9cb4ae48f3aef97f773a09563ffbd4af5f8372e4 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 11 May 2022 12:05:02 -0500 Subject: [PATCH 056/113] changing main to get rank as arg --- imagenet_exec.sh | 6 +++--- main.py | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index 0a8c25c9..c18b90d9 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -27,7 +27,7 @@ log_root="ddp_debug_" log_end="_log" for gpu in 0 1 2 3 do - python ddp_debug_main.py \ - --rank "$gpu" \ - --config "$conf_file" > "$log_root$gpu$log_end" 2>&1 & + CUDA_VISIBLE_DEVICES="$gpu" python main.py \ + --rank "$gpu" \ + --config "$conf_file" > "$log_root$gpu$log_end" 2>&1 & done diff --git a/main.py b/main.py index 62028554..f7d55ac2 100644 --- a/main.py +++ b/main.py @@ -3,7 +3,7 @@ from torchvision import datasets, transforms import torch.multiprocessing from torch.utils.data import random_split -torch.multiprocessing.set_sharing_strategy("file_system") +# torch.multiprocessing.set_sharing_strategy("file_system") @@ -17,24 +17,24 @@ def main(): # world size = ngpus_per_node since we are assuming single node ngpus_per_node = torch.cuda.device_count() - if parser_args.multiprocessing_distributed: - # assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" - mp.spawn(main_worker, args=(ngpus_per_node,), nprocs=ngpus_per_node, join=True) - else: - # Simply call main_worker function - main_worker(parser_args.gpu, ngpus_per_node) + # if parser_args.multiprocessing_distributed: + # # assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" + # mp.spawn(main_worker, args=(ngpus_per_node,), nprocs=ngpus_per_node, join=True) + # else: + # # Simply call main_worker function + # main_worker(parser_args.gpu, ngpus_per_node) + main_worker(parser_args.rank, ngpus_per_node) -def main_worker(gpu, ngpus_per_node): +def main_worker(rank, ngpus_per_node): # NOTE: gpu = rank in the multiprocessing setting - parser_args.gpu = gpu + parser_args.gpu = rank if parser_args.gpu is not None: print("Use GPU: {} for training".format(parser_args.gpu)) if parser_args.multiprocessing_distributed: - parser_args.rank = parser_args.gpu - setup_distributed(parser_args.rank, ngpus_per_node) + setup_distributed(parser_args.gpu, ngpus_per_node) # if using ddp, divide batch size per gpu parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) From 3a7ea3d9d0bad4bf2c7da57667168e4dea1e8550 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 11 May 2022 17:16:09 +0000 Subject: [PATCH 057/113] bringing more functionality back in --- imagenet_exec.sh | 2 +- main.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index c18b90d9..5bf4dcb6 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -27,7 +27,7 @@ log_root="ddp_debug_" log_end="_log" for gpu in 0 1 2 3 do - CUDA_VISIBLE_DEVICES="$gpu" python main.py \ + python main.py \ --rank "$gpu" \ --config "$conf_file" > "$log_root$gpu$log_end" 2>&1 & done diff --git a/main.py b/main.py index f7d55ac2..8c0b65fe 100644 --- a/main.py +++ b/main.py @@ -38,8 +38,8 @@ def main_worker(rank, ngpus_per_node): # if using ddp, divide batch size per gpu parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) - # train, validate, modifier = get_trainer(parser_args) - #model = get_model(parser_args) + train, validate, modifier = get_trainer(parser_args) + model = get_model(parser_args) if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: idty_str = get_idty_str(parser_args) From 0726f752d70c570ae3c5b72268bb544a837054f9 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 11 May 2022 21:53:52 -0500 Subject: [PATCH 058/113] trying original val loop without mp.spawn --- imagenet_exec.sh | 4 +- main.py | 216 ++--------------------------------------------- 2 files changed, 10 insertions(+), 210 deletions(-) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index 5bf4dcb6..793bb6e6 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -28,6 +28,6 @@ log_end="_log" for gpu in 0 1 2 3 do python main.py \ - --rank "$gpu" \ - --config "$conf_file" > "$log_root$gpu$log_end" 2>&1 & + --rank "$gpu" \ + --config "$conf_file" > "$log_root$gpu$log_end" 2>&1 & done diff --git a/main.py b/main.py index 8c0b65fe..a68d1256 100644 --- a/main.py +++ b/main.py @@ -3,9 +3,6 @@ from torchvision import datasets, transforms import torch.multiprocessing from torch.utils.data import random_split -# torch.multiprocessing.set_sharing_strategy("file_system") - - def main(): print(parser_args) @@ -111,7 +108,8 @@ def main_worker(rank, ngpus_per_node): epoch_list, test_acc_before_round_list, test_acc_list, reg_loss_list, model_sparsity_list, val_acc_list, train_acc_list = [], [], [], [], [], [], [] # Save the initial model - #torch.save(model.state_dict(), result_root + 'init_model.pth') + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + torch.save(model.state_dict(), result_root + 'init_model.pth') # compute prune_rate to reach target_sparsity if not parser_args.override_prune_rate: @@ -152,19 +150,16 @@ def main_worker(rank, ngpus_per_node): print("STARTING TRAINING: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) - # lr_policy(epoch, iteration=None) - # modifier(parser_args, epoch, model) - # cur_lr = get_lr(optimizer) + lr_policy(epoch, iteration=None) + modifier(parser_args, epoch, model) + cur_lr = get_lr(optimizer) print("Skipping training, just gonna round") print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) - # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, - #ratio=parser_args.noise_ratio, rank=parser_args.gpu) + cp_model = round_model(model, parser_args.round, noise=parser_args.noise, + ratio=parser_args.noise_ratio, rank=parser_args.gpu) if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - # print("Pretend to validate") - # continue - # do_something() - my_validate(data.actual_val_loader, None, criterion, parser_args, writer, epoch) + validate(data.val_loader, model, criterion, parser_args, writer, epoch) acc1 = -1 else: acc1 = -1 @@ -362,201 +357,6 @@ def main_worker(rank, ngpus_per_node): cleanup_distributed() -def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler=None): - # batch_time = AverageMeter("Time", ":6.3f") - # data_time = AverageMeter("Data", ":6.3f") - # losses = AverageMeter("Loss", ":.3f") - # top1 = AverageMeter("Acc@1", ":6.2f") - # top5 = AverageMeter("Acc@5", ":6.2f") - # top10 = AverageMeter("Acc@10", ":6.2f") - # progress = ProgressMeter( - # len(train_loader), - # [batch_time, data_time, losses, top1, top5], - # prefix=f"GPU:[{args.gpu}] | Epoch: [{epoch}]", - # ) - top1 = 0 - top5 = 0 - top10 = 0 - num_images = 0 - - # switch to train mode - model.train() - - batch_size = train_loader.batch_size - num_batches = len(train_loader) - end = time.time() - # for i, (images, target) in tqdm.tqdm( - # enumerate(train_loader), ascii=True, total=len(train_loader) - # ): - print("(TRAINER)BEFORE TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - for i, (images, target) in enumerate(train_loader): - # print("(TRAINER)AFTER LOADING IMAGES: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - # print("Is it just the image?") - # print("(TRAINER): Size of images: {}".format(sys.getsizeof(images))) - # measure data loading time - data_time = time.time() - end - # print("Data Time: {}".format(data_time)) - # print(images.shape, target.shape) - - images = images.to(args.gpu) - target = target.to(args.gpu) - - # update score thresholds for global ep - if args.algo in ['global_ep', 'global_ep_iter']: - prune(model, update_thresholds_only=True) - - # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - if args.algo in ['hc', 'hc_iter', 'pt'] and i % args.project_freq == 0 and not args.differentiate_clamp: - for name, params in model.named_parameters(): - if "score" in name: - scores = params - with torch.no_grad(): - scores.data = torch.clamp(scores.data, 0.0, 1.0) - # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - - # compute output - if scaler is None: - output = model(images) - loss = criterion(output, target) - else: - with torch.cuda.amp.autocast(enabled=True): # mixed precision - output = model(images) - loss = criterion(output, target) - - if args.lam_finetune_loss > 0: - raise NotImplementedError # please check finetune_loss repo - - regularization_loss = torch.tensor(0) - # print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - if args.regularization: - regularization_loss =\ - get_regularization_loss(model, regularizer=args.regularization, - lmbda=args.lmbda, alpha=args.alpha, - alpha_prime=args.alpha_prime) - # print("(TRAINER)BEFORE REGULARIZATION COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - #print('regularization_loss: ', regularization_loss) - loss += regularization_loss - # print("(TRAINER)BEFORE ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - # measure accuracy and record loss - acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - # losses.update(loss.item(), images.size(0)) - # top1.update(acc1.item(), images.size(0)) - # top5.update(acc5.item(), images.size(0)) - # top10.update(acc10.item(), images.size(0)) - # compute weighted sum for each accuracy so we can average it later - top1 += acc1.item() * images.size(0) - top5 += acc5.item() * images.size(0) - top10 += acc10.item() * images.size(0) - num_images += images.size(0) - - # print("(TRAINER)AFTER ACCURACY COMPUTATION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - - # compute gradient and do SGD step - optimizer.zero_grad() - if scaler is None: - loss.backward() - optimizer.step() - else: - scaler.scale(loss).backward() - scaler.step(optimizer) - scaler.update() - - # measure elapsed time - # batch_time.update(time.time() - end) - batch_time = time.time() - end - end = time.time() - - if i % args.print_freq == 0: - t = (num_batches * epoch + i) * batch_size - # progress.display(i) - # progress.write_to_tensorboard( - # writer, prefix="train", global_step=t) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) - - print("(TRAINER)AFTER TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - # before completing training, clean up model based on latest scores - # update score thresholds for global ep - if args.algo in ['global_ep', 'global_ep_iter']: - prune(model, update_thresholds_only=True) - if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: - # print("(TRAINER)BEFORE PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - for name, params in model.named_parameters(): - if "score" in name: - scores = params - with torch.no_grad(): - scores.data = torch.clamp(scores.data, 0.0, 1.0) - # print("(TRAINER)AFTER PROJECTION: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) - - return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() - - -def my_validate(val_loader, model, criterion, args, writer, epoch): - # batch_time = AverageMeter("Time", ":6.3f", write_val=False) - # losses = AverageMeter("Loss", ":.3f", write_val=False) - # top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) - # top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) - # top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) - # progress = ProgressMeter( - # len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " - # ) - top1 = 0 - top5 = 0 - top10 = 0 - num_images = 1 - - # switch to evaluate mode - #model.eval() - with torch.no_grad(): - end = time.time() - # for i, (images, target) in tqdm.tqdm( - # enumerate(val_loader), ascii=True, total=len(val_loader) - # ): - # time.sleep(0.1) - # return -1, -1, -1 - for i, (images, target) in enumerate(val_loader): - break - continue - images = images.to(args.gpu) - target = target.to(args.gpu) - - #print(images.shape, target.shape) - - # compute output - # output = model(images) - - # loss = criterion(output, target) - loss = torch.Tensor([0]) - - # measure accuracy and record loss - # acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - acc1, acc5, acc10 = torch.Tensor([5]), torch.Tensor([5]), torch.Tensor([5]) - # losses.update(loss.item(), images.size(0)) - # top1.update(acc1.item(), images.size(0)) - # top5.update(acc5.item(), images.size(0)) - # top10.update(acc10.item(), images.size(0)) - # compute weighted sum for each accuracy so we can average it later - top1 += acc1.item()# * images.size(0) - top5 += acc5.item()# * images.size(0) - top10 += acc10.item()# * images.size(0) - num_images += 1#images.size(0) - - # measure elapsed time - # batch_time.update(time.time() - end) - batch_time = time.time() - end - end = time.time() - - if i % args.print_freq == 0: - # progress.display(i) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) - - # progress.display(len(val_loader)) - - # if writer is not None: - # progress.write_to_tensorboard( - # writer, prefix="test", global_step=epoch) - - print("Model top1 Accuracy: {}".format(top1/num_images)) - return top1/num_images, top5/num_images, top10/num_images def do_something(): time.sleep(5) From fdee77b850c7068e8aef35d334c7416f3da82293 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 11 May 2022 22:12:57 -0500 Subject: [PATCH 059/113] trying train without mp.spawn() --- main.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/main.py b/main.py index a68d1256..53abdb94 100644 --- a/main.py +++ b/main.py @@ -154,19 +154,19 @@ def main_worker(rank, ngpus_per_node): modifier(parser_args, epoch, model) cur_lr = get_lr(optimizer) - print("Skipping training, just gonna round") - print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) - cp_model = round_model(model, parser_args.round, noise=parser_args.noise, - ratio=parser_args.noise_ratio, rank=parser_args.gpu) - if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - validate(data.val_loader, model, criterion, parser_args, writer, epoch) - acc1 = -1 - else: - acc1 = -1 - print("GPU: {} | acc1={}".format(parser_args.gpu, acc1)) - print("After Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) - dist.barrier() - continue + # print("Skipping training, just gonna round") + # print("Before Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + # cp_model = round_model(model, parser_args.round, noise=parser_args.noise, + # ratio=parser_args.noise_ratio, rank=parser_args.gpu) + # if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + # validate(data.val_loader, model, criterion, parser_args, writer, epoch) + # acc1 = -1 + # else: + # acc1 = -1 + # print("GPU: {} | acc1={}".format(parser_args.gpu, acc1)) + # print("After Round: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + # dist.barrier() + # continue # save the score at the beginning of training epoch, so if we set parser.args.rewind_to_epoch to 0 @@ -182,11 +182,11 @@ def main_worker(rank, ngpus_per_node): # train for one epoch start_train = time.time() - print("BEFORE TRAIN LOOP: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + print("Before Train() Call: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) train_acc1, train_acc5, train_acc10, reg_loss = train( data.train_loader, model, criterion, optimizer, epoch, parser_args, writer=writer, scaler=scaler ) - print("AFTER TRAIN LOOP: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + print("After Train() Call: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) # train_time.update((time.time() - start_train) / 60) train_time = (time.time() - start_train) / 60 @@ -195,7 +195,7 @@ def main_worker(rank, ngpus_per_node): # evaluate on validation set if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: start_validation = time.time() - print("BEFORE VAL LOOP: GPU: {} | Epoch {} | Memory Usage: {}".format(parser_args.gpu, epoch, psutil.virtual_memory())) + print("Before Val() Call: GPU: {} | Epoch {} | Memory Usage: {}".format(parser_args.gpu, epoch, psutil.virtual_memory())) if parser_args.algo in ['hc', 'hc_iter']: br_acc1, br_acc5, br_acc10 = validate( data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding @@ -219,13 +219,13 @@ def main_worker(rank, ngpus_per_node): print('Acc: {}'.format(acc1)) # validation_time.update((time.time() - start_validation) / 60) validation_time = (time.time() - start_validation) / 60 - print("AFTER VAL LOOP: GPU: {} | Epoch {} | Memory Usage: {}".format(parser_args.gpu, epoch, psutil.virtual_memory())) + print("After Val() Call: GPU: {} | Epoch {} | Memory Usage: {}".format(parser_args.gpu, epoch, psutil.virtual_memory())) # prune the model every T_{prune} epochs if not parser_args.weight_training and parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: - print("BEFORE PRUNE: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + print("Before Prune() Call: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) prune(model) - print("AFTER PRUNE: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) + print("After Prune() Call: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) if parser_args.checkpoint_at_prune: if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: save_checkpoint_at_prune(model, parser_args) From e1dda18c17e7c23ce66eb5ad5fcedbfa9ff35f46 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 May 2022 03:13:36 +0000 Subject: [PATCH 060/113] minor bugfix --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 53abdb94..f514bd96 100644 --- a/main.py +++ b/main.py @@ -150,7 +150,6 @@ def main_worker(rank, ngpus_per_node): print("STARTING TRAINING: Epoch {} | Memory Usage: {}".format(epoch, psutil.virtual_memory())) if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) - lr_policy(epoch, iteration=None) modifier(parser_args, epoch, model) cur_lr = get_lr(optimizer) From 28929b39f95d0dcb89918b7068471bcf8c81688e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 May 2022 03:15:49 +0000 Subject: [PATCH 061/113] making logs camel case --- trainers/default.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index d7b8ae42..d8f291fc 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -41,7 +41,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # for i, (images, target) in tqdm.tqdm( # enumerate(train_loader), ascii=True, total=len(train_loader) # ): - print("(TRAINER)BEFORE TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + print("(TRAINER)Before Train Loop: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) for i, (images, target) in enumerate(train_loader): # print("(TRAINER)AFTER LOADING IMAGES: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # print("Is it just the image?") @@ -126,7 +126,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # writer, prefix="train", global_step=t) print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) - print("(TRAINER)AFTER TRAIN LOOP: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) + print("(TRAINER)After Train Loop: GPU:{} | Epoch {} | Memory Usage: {}".format(args.gpu, epoch, psutil.virtual_memory())) # before completing training, clean up model based on latest scores # update score thresholds for global ep if args.algo in ['global_ep', 'global_ep_iter']: From 87daf128908060b7cc640073b6b20259bf25b26d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 May 2022 05:54:24 +0000 Subject: [PATCH 062/113] minor bugfix --- main_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main_utils.py b/main_utils.py index 9de7f53b..743665b0 100644 --- a/main_utils.py +++ b/main_utils.py @@ -332,7 +332,7 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b 20, 40], gamma=0.1) # NOTE: hard-coded ''' - # train, validate, modifier = get_trainer(parser_args) + train, validate, modifier = get_trainer(parser_args) # check the performance of loaded model (after rounding) if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: From 27484350f9bb26bbb4d9c1d82b173673a82361f9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 May 2022 05:56:24 +0000 Subject: [PATCH 063/113] changing log file name --- imagenet_exec.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index 793bb6e6..8ff63f7f 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -23,7 +23,7 @@ done BLOCK conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" -log_root="ddp_debug_" +log_root="resnet50_sp5_gpu_" log_end="_log" for gpu in 0 1 2 3 do From 045387ac8732a75af038dc443c0adb45068e90be Mon Sep 17 00:00:00 2001 From: PaulCCCCCCH Date: Thu, 12 May 2022 11:17:30 +0400 Subject: [PATCH 064/113] distributed launch --- imagenet_exec_slurm.sh | 23 +++++++++++++++++++++++ main.py | 10 ++++++++++ 2 files changed, 33 insertions(+) create mode 100644 imagenet_exec_slurm.sh diff --git a/imagenet_exec_slurm.sh b/imagenet_exec_slurm.sh new file mode 100644 index 00000000..9bb24b69 --- /dev/null +++ b/imagenet_exec_slurm.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#SBATCH --job-name=hongyiwa-test # create a short name for your job +#SBATCH --output=hc_resnet50_imagenet_trial0.txt +#SBATCH --nodes=1 # node count +#SBATCH --ntasks-per-node=4 # total number of tasks across all nodes +#SBATCH --cpus-per-task=16 # cpu-cores per task (>1 if multi-threaded tasks) +#SBATCH --mem=64G # total memory per node (4 GB per cpu-core is default) +#SBATCH --gres=gpu:4 # number of gpus per node +#SBATCH --time=202:00:00 # total run time limit (HH:MM:SS) + +source /apps/local/conda_init.sh +conda activate pf2.0 + +conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" +log_root="resnet50_sp5_gpu_" +log_end="_log" + +python -m torch.distributed.launch --nproc_per_node=4 --use_env main.py --config "$conf_file" + +#torchrun main.py --config "$conf_file" +#python main.py --rank 1 --config "$conf_file" & +#python main.py --rank 2 --config "$conf_file" & +#python main.py --rank 3 --config "$conf_file" & diff --git a/main.py b/main.py index f514bd96..3141744f 100644 --- a/main.py +++ b/main.py @@ -4,6 +4,8 @@ import torch.multiprocessing from torch.utils.data import random_split +import os + def main(): print(parser_args) print("\n\nBeginning of process.") @@ -20,6 +22,14 @@ def main(): # else: # # Simply call main_worker function # main_worker(parser_args.gpu, ngpus_per_node) + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + parser_args.rank = int(os.environ["RANK"]) + parser_args.world_size = int(os.environ['WORLD_SIZE']) + parser_args.gpu = int(os.environ['LOCAL_RANK']) + elif 'SLURM_PROCID' in os.environ: + parser_args.rank = int(os.environ['SLURM_PROCID']) + parser_args.gpu = args.rank % torch.cuda.device_count() + main_worker(parser_args.rank, ngpus_per_node) From 8d822982732eda11700223f38aba2329cf923fdc Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 12 May 2022 13:56:00 -0500 Subject: [PATCH 065/113] fixing validate bug and trying to be careful with memory before finetune --- main.py | 12 ++++++++---- trainers/default.py | 21 ++++++++------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/main.py b/main.py index 3141744f..d88687ef 100644 --- a/main.py +++ b/main.py @@ -330,18 +330,21 @@ def main_worker(rank, ngpus_per_node): dist.barrier() print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - cp_model = copy.deepcopy(model) + if not parser_args.multiprocessing_distributed: + before_ft_model = copy.deepcopy(model) + else: + before_ft_model = model.module if not parser_args.skip_fine_tune: if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: print("Beginning fine-tuning") - cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, + model = finetune(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root) # print out the final acc if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - eval_and_print(validate, data.val_loader, cp_model, criterion, + eval_and_print(validate, data.val_loader, model, criterion, parser_args, writer=None, description='final model after finetuning') # save checkpoint after fine-tuning - torch.save(cp_model.state_dict(), result_root + 'model_after_finetune.pth') + torch.save(model.state_dict(), result_root + 'model_after_finetune.pth') else: if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: print("Skipping finetuning!!!") @@ -352,6 +355,7 @@ def main_worker(rank, ngpus_per_node): print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) if not parser_args.skip_sanity_checks: + # load before_ft_model into current model do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root) else: diff --git a/trainers/default.py b/trainers/default.py index d8f291fc..66b4513f 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -155,7 +155,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): top1 = 0 top5 = 0 top10 = 0 - num_images = 1 + num_images = 0 # switch to evaluate mode model.eval() @@ -165,33 +165,28 @@ def validate(val_loader, model, criterion, args, writer, epoch): # for i, (images, target) in tqdm.tqdm( # enumerate(val_loader), ascii=True, total=len(val_loader) # ): - # time.sleep(0.1) - # return -1, -1, -1 for i, (images, target) in enumerate(val_loader): - continue images = images.to(args.gpu) target = target.to(args.gpu) #print(images.shape, target.shape) # compute output - # output = model(images) + output = model(images) - # loss = criterion(output, target) - loss = torch.Tensor([0]) + loss = criterion(output, target) # measure accuracy and record loss - # acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - acc1, acc5, acc10 = torch.Tensor([5]), torch.Tensor([5]), torch.Tensor([5]) + acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) # losses.update(loss.item(), images.size(0)) # top1.update(acc1.item(), images.size(0)) # top5.update(acc5.item(), images.size(0)) # top10.update(acc10.item(), images.size(0)) # compute weighted sum for each accuracy so we can average it later - top1 += acc1.item()# * images.size(0) - top5 += acc5.item()# * images.size(0) - top10 += acc10.item()# * images.size(0) - num_images += 1#images.size(0) + top1 += acc1.item() * images.size(0) + top5 += acc5.item() * images.size(0) + top10 += acc10.item() * images.size(0) + num_images += images.size(0) # measure elapsed time # batch_time.update(time.time() - end) From 815bb557b345a44dd7915ae339c116f88a245f23 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 May 2022 22:58:16 +0000 Subject: [PATCH 066/113] getting rid of before round acc --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index d88687ef..c8976aa2 100644 --- a/main.py +++ b/main.py @@ -206,9 +206,10 @@ def main_worker(rank, ngpus_per_node): start_validation = time.time() print("Before Val() Call: GPU: {} | Epoch {} | Memory Usage: {}".format(parser_args.gpu, epoch, psutil.virtual_memory())) if parser_args.algo in ['hc', 'hc_iter']: - br_acc1, br_acc5, br_acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding - print('Acc before rounding: {}'.format(br_acc1)) + br_acc1, br_acc5, br_acc10 = -1, -1, -1 + #br_acc1, br_acc5, br_acc10 = validate( + # data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding + #print('Acc before rounding: {}'.format(br_acc1)) acc_avg = 0 for num_trial in range(parser_args.num_test): cp_model = round_model(model, parser_args.round, noise=parser_args.noise, From dd5a27eee64ed383f9be3f6ff6e2cf95f57cd94d Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 11:42:32 -0500 Subject: [PATCH 067/113] starting from scratch. implementing just GM imagenet from the pytorch example --- imagenet/imagenet_main.py | 467 ++++++++++++++++++++++++++++++ imagenet/imagenet_main_bkp.py | 454 +++++++++++++++++++++++++++++ imagenet/resnet.py | 520 ++++++++++++++++++++++++++++++++++ 3 files changed, 1441 insertions(+) create mode 100644 imagenet/imagenet_main.py create mode 100644 imagenet/imagenet_main_bkp.py create mode 100644 imagenet/resnet.py diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py new file mode 100644 index 00000000..fbaccadc --- /dev/null +++ b/imagenet/imagenet_main.py @@ -0,0 +1,467 @@ +import argparse +import os +import random +import shutil +import time +import warnings +from enum import Enum +import math +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +from torch.optim.lr_scheduler import StepLR +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torch.autograd as autograd +import torch.nn.functional as F +# import torchvision.models as models +import models + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('data', metavar='DIR', default='imagenet', + help='path to dataset (default: imagenet)') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') + +best_acc1 = 0 + + +def main(): + args = parser.parse_args() + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + + if args.gpu is not None: + print("Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + # create model + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("=> creating model '{}'".format(args.arch)) + # model = models.__dict__[args.arch]() + model = models.ResNet50() + + if not torch.cuda.is_available(): + print('using CPU, this will be slow') + elif args.distributed: + # For multiprocessing distributed, DistributedDataParallel constructor + # should always set the single device scope, otherwise, + # DistributedDataParallel will use all available devices. + if args.gpu is not None: + torch.cuda.set_device(args.gpu) + model.cuda(args.gpu) + # When using a single GPU per process and per + # DistributedDataParallel, we need to divide the batch size + # ourselves based on the total number of GPUs of the current node. + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + else: + model.cuda() + # DistributedDataParallel will divide and allocate batch_size to all + # available GPUs if device_ids are not set + model = torch.nn.parallel.DistributedDataParallel(model) + elif args.gpu is not None: + torch.cuda.set_device(args.gpu) + model = model.cuda(args.gpu) + else: + # DataParallel will divide and allocate batch_size to all available GPUs + if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + model.features = torch.nn.DataParallel(model.features) + model.cuda() + else: + model = torch.nn.DataParallel(model).cuda() + + # define loss function (criterion), optimizer, and learning rate scheduler + criterion = nn.CrossEntropyLoss().cuda(args.gpu) + + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + scheduler = StepLR(optimizer, step_size=30, gamma=0.1) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + if args.gpu is None: + checkpoint = torch.load(args.resume) + else: + # Map model to be loaded to specified single gpu. + loc = 'cuda:{}'.format(args.gpu) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + if args.gpu is not None: + # best_acc1 may be from a checkpoint from a different GPU + best_acc1 = best_acc1.to(args.gpu) + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + scheduler.load_state_dict(checkpoint['scheduler']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True) + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch, args) + + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args) + + scheduler.step() + + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer' : optimizer.state_dict(), + 'scheduler' : scheduler.state_dict() + }, is_best) + + +def train(train_loader, model, criterion, optimizer, epoch, args): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + + end = time.time() + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + if torch.cuda.is_available(): + target = target.cuda(args.gpu, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + +def validate(val_loader, model, criterion, args): + batch_time = AverageMeter('Time', ':6.3f', Summary.NONE) + losses = AverageMeter('Loss', ':.4e', Summary.NONE) + top1 = AverageMeter('Acc@1', ':6.2f', Summary.AVERAGE) + top5 = AverageMeter('Acc@5', ':6.2f', Summary.AVERAGE) + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + if torch.cuda.is_available(): + target = target.cuda(args.gpu, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + progress.display_summary() + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best.pth.tar') + +class Summary(Enum): + NONE = 0 + AVERAGE = 1 + SUM = 2 + COUNT = 3 + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f', summary_type=Summary.AVERAGE): + self.name = name + self.fmt = fmt + self.summary_type = summary_type + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + def summary(self): + fmtstr = '' + if self.summary_type is Summary.NONE: + fmtstr = '' + elif self.summary_type is Summary.AVERAGE: + fmtstr = '{name} {avg:.3f}' + elif self.summary_type is Summary.SUM: + fmtstr = '{name} {sum:.3f}' + elif self.summary_type is Summary.COUNT: + fmtstr = '{name} {count:.3f}' + else: + raise ValueError('invalid summary type %r' % self.summary_type) + + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + + def display_summary(self): + entries = [" *"] + entries += [meter.summary() for meter in self.meters] + print(' '.join(entries)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +""" +@ksreenivasan: Dumping new code here +""" + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py new file mode 100644 index 00000000..30fd04c8 --- /dev/null +++ b/imagenet/imagenet_main_bkp.py @@ -0,0 +1,454 @@ +import argparse +import os +import random +import shutil +import time +import warnings +from enum import Enum + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +from torch.optim.lr_scheduler import StepLR +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('data', metavar='DIR', default='imagenet', + help='path to dataset (default: imagenet)') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') + +best_acc1 = 0 + + +def main(): + args = parser.parse_args() + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + + if args.gpu is not None: + print("Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + # create model + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch]() + + if not torch.cuda.is_available(): + print('using CPU, this will be slow') + elif args.distributed: + # For multiprocessing distributed, DistributedDataParallel constructor + # should always set the single device scope, otherwise, + # DistributedDataParallel will use all available devices. + if args.gpu is not None: + torch.cuda.set_device(args.gpu) + model.cuda(args.gpu) + # When using a single GPU per process and per + # DistributedDataParallel, we need to divide the batch size + # ourselves based on the total number of GPUs of the current node. + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + else: + model.cuda() + # DistributedDataParallel will divide and allocate batch_size to all + # available GPUs if device_ids are not set + model = torch.nn.parallel.DistributedDataParallel(model) + elif args.gpu is not None: + torch.cuda.set_device(args.gpu) + model = model.cuda(args.gpu) + else: + # DataParallel will divide and allocate batch_size to all available GPUs + if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + model.features = torch.nn.DataParallel(model.features) + model.cuda() + else: + model = torch.nn.DataParallel(model).cuda() + + # define loss function (criterion), optimizer, and learning rate scheduler + criterion = nn.CrossEntropyLoss().cuda(args.gpu) + + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + scheduler = StepLR(optimizer, step_size=30, gamma=0.1) + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + if args.gpu is None: + checkpoint = torch.load(args.resume) + else: + # Map model to be loaded to specified single gpu. + loc = 'cuda:{}'.format(args.gpu) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + if args.gpu is not None: + # best_acc1 may be from a checkpoint from a different GPU + best_acc1 = best_acc1.to(args.gpu) + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + scheduler.load_state_dict(checkpoint['scheduler']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ])) + + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ])), + batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True) + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch, args) + + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args) + + scheduler.step() + + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0): + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_acc1': best_acc1, + 'optimizer' : optimizer.state_dict(), + 'scheduler' : scheduler.state_dict() + }, is_best) + + +def train(train_loader, model, criterion, optimizer, epoch, args): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + + end = time.time() + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + if torch.cuda.is_available(): + target = target.cuda(args.gpu, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + +def validate(val_loader, model, criterion, args): + batch_time = AverageMeter('Time', ':6.3f', Summary.NONE) + losses = AverageMeter('Loss', ':.4e', Summary.NONE) + top1 = AverageMeter('Acc@1', ':6.2f', Summary.AVERAGE) + top5 = AverageMeter('Acc@5', ':6.2f', Summary.AVERAGE) + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + if torch.cuda.is_available(): + target = target.cuda(args.gpu, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + progress.display_summary() + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best.pth.tar') + +class Summary(Enum): + NONE = 0 + AVERAGE = 1 + SUM = 2 + COUNT = 3 + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f', summary_type=Summary.AVERAGE): + self.name = name + self.fmt = fmt + self.summary_type = summary_type + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + def summary(self): + fmtstr = '' + if self.summary_type is Summary.NONE: + fmtstr = '' + elif self.summary_type is Summary.AVERAGE: + fmtstr = '{name} {avg:.3f}' + elif self.summary_type is Summary.SUM: + fmtstr = '{name} {sum:.3f}' + elif self.summary_type is Summary.COUNT: + fmtstr = '{name} {count:.3f}' + else: + raise ValueError('invalid summary type %r' % self.summary_type) + + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + + def display_summary(self): + entries = [" *"] + entries += [meter.summary() for meter in self.meters] + print(' '.join(entries)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/imagenet/resnet.py b/imagenet/resnet.py new file mode 100644 index 00000000..5e0637e9 --- /dev/null +++ b/imagenet/resnet.py @@ -0,0 +1,520 @@ +import torch +import torch.autograd as autograd +import torch.nn as nn +import torch.nn.functional as F +import numpy as np + +import math + +# BasicBlock {{{ +class BasicBlock(nn.Module): + M = 2 + expansion = 1 + + def __init__(self, builder, inplanes, planes, stride=1, downsample=None, base_width=64): + super(BasicBlock, self).__init__() + if base_width / 64 > 1: + raise ValueError("Base width >64 does not work for BasicBlock") + + self.conv1 = builder.conv3x3(inplanes, planes, stride) + self.bn1 = builder.batchnorm(planes) + self.relu = builder.activation() + self.conv2 = builder.conv3x3(planes, planes) + self.bn2 = builder.batchnorm(planes, last_bn=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + if self.bn1 is not None: + out = self.bn1(out) + + out = self.relu(out) + + out = self.conv2(out) + + if self.bn2 is not None: + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + M = 3 + expansion = 4 + + def __init__(self, builder, inplanes, planes, stride=1, downsample=None, base_width=64): + super(Bottleneck, self).__init__() + width = int(planes * base_width / 64) + self.conv1 = builder.conv1x1(inplanes, width) + self.bn1 = builder.batchnorm(width) + self.conv2 = builder.conv3x3(width, width, stride=stride) + self.bn2 = builder.batchnorm(width) + self.conv3 = builder.conv1x1(width, planes * self.expansion) + self.bn3 = builder.batchnorm(planes * self.expansion, last_bn=True) + self.relu = builder.activation() + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + + out = self.relu(out) + + return out + + +# Bottleneck }}} + +# ResNet {{{ +class ResNet(nn.Module): + def __init__(self, builder, block, layers, num_classes=1000, base_width=64): + args_first_layer_dense = False + args_last_layer_dense = False + args_bias = False + + self.inplanes = 64 + super(ResNet, self).__init__() + + self.base_width = base_width + if self.base_width // 64 > 1: + print(f"==> Using {self.base_width // 64}x wide model") + + if args_first_layer_dense: + self.conv1 = nn.Conv2d( + 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False + ) + else: + self.conv1 = builder.conv7x7(3, 64, stride=2, first_layer=True) + + self.bn1 = builder.batchnorm(64) + self.relu = builder.activation() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(builder, block, 64, layers[0]) + self.layer2 = self._make_layer(builder, block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(builder, block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(builder, block, 512, layers[3], stride=2) + self.avgpool = nn.AdaptiveAvgPool2d(1) + + # self.fc = nn.Linear(512 * block.expansion, num_classes) + if args_last_layer_dense: + self.fc = nn.Conv2d(512 * block.expansion, num_classes, 1) + else: + self.fc = builder.conv1x1(512 * block.expansion, num_classes) + + self.prunable_layer_names, self.prunable_biases = self.get_prunable_param_names() + + def _make_layer(self, builder, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + dconv = builder.conv1x1( + self.inplanes, planes * block.expansion, stride=stride + ) + dbn = builder.batchnorm(planes * block.expansion) + if dbn is not None: + downsample = nn.Sequential(dconv, dbn) + else: + downsample = dconv + + layers = [] + layers.append(block(builder, self.inplanes, planes, stride, downsample, base_width=self.base_width)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(builder, self.inplanes, planes, base_width=self.base_width)) + + return nn.Sequential(*layers) + + def get_prunable_param_names(model): + prunable_weights = [name + '.weight' for name, module in model.named_modules() if + isinstance(module, nn.modules.conv.Conv2d) or + isinstance(module, nn.modules.linear.Linear)] + if args_bias: + prunable_biases = [name + '.bias' for name, module in model.named_modules() if + isinstance(module, nn.modules.conv.Conv2d) or + isinstance(module, nn.modules.linear.Linear)] + else: + prunable_biases = [""] + + return prunable_weights, prunable_biases + + def forward(self, x): + # update score thresholds for global ep + # TODO: Don't need this for now + # if parser_args.algo in ['global_ep', 'global_ep_iter'] or parser_args.bottom_k_on_forward: + # prune(self, update_thresholds_only=True) + x = self.conv1(x) + + if self.bn1 is not None: + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = self.fc(x) + x = x.view(x.size(0), -1) + + return x + + +# ResNet }}} +def ResNet18(pretrained=False): + return ResNet(get_builder(), BasicBlock, [2, 2, 2, 2], 1000) + + +def ResNet50(pretrained=False): + return ResNet(get_builder(), Bottleneck, [3, 4, 6, 3], 1000) + + +def ResNet101(pretrained=False): + return ResNet(get_builder(), Bottleneck, [3, 4, 23, 3], 200) + #return ResNet(get_builder(), Bottleneck, [3, 4, 23, 3], 1000) + + +def WideResNet50_2(pretrained=False): + return ResNet( + get_builder(), Bottleneck, [3, 4, 6, 3], num_classes=1000, base_width=64 * 2 + ) + + +def WideResNet101_2(pretrained=False): + return ResNet( + get_builder(), Bottleneck, [3, 4, 23, 3], num_classes=1000, base_width=64 * 2 + ) + + +class Builder(object): + def __init__(self, conv_layer, bn_layer, first_layer=None, weight_init="signed_constant"): + self.conv_layer = conv_layer + self.bn_layer = bn_layer + # self.first_layer = first_layer or conv_layer + self.first_layer = conv_layer + self.weight_init = weight_init + + def conv(self, kernel_size, in_planes, out_planes, stride=1, first_layer=False, groups=1): + # conv_layer = self.first_layer if first_layer else self.conv_layer + conv_layer = self.conv_layer + + if first_layer: + print(f"==> Building first layer") + + if kernel_size == 3: + conv = conv_layer( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=True, + groups=groups + ) + elif kernel_size == 1: + conv = conv_layer( + in_planes, out_planes, + kernel_size=1, + stride=stride, + bias=True, + ) + elif kernel_size == 5: + conv = conv_layer( + in_planes, + out_planes, + kernel_size=5, + stride=stride, + padding=2, + bias=True, + ) + elif kernel_size == 7: + conv = conv_layer( + in_planes, + out_planes, + kernel_size=7, + stride=stride, + padding=3, + bias=True, + ) + else: + return None + + self._init_conv(conv) + + return conv + + def conv3x3(self, in_planes, out_planes, stride=1, first_layer=False, groups=1): + """3x3 convolution with padding""" + c = self.conv(3, in_planes, out_planes, stride=stride, first_layer=first_layer, groups=groups) + return c + + def conv1x1(self, in_planes, out_planes, stride=1, first_layer=False): + """1x1 convolution with padding""" + c = self.conv(1, in_planes, out_planes, stride=stride, first_layer=first_layer) + return c + + def linear(self, in_planes, out_planes): + l = SubnetLinear(in_planes, out_planes, bias=True) + self._init_conv(l) + return l + + def conv7x7(self, in_planes, out_planes, stride=1, first_layer=False): + """7x7 convolution with padding""" + c = self.conv(7, in_planes, out_planes, stride=stride, first_layer=first_layer) + return c + + def conv5x5(self, in_planes, out_planes, stride=1, first_layer=False): + """5x5 convolution with padding""" + c = self.conv(5, in_planes, out_planes, stride=stride, first_layer=first_layer) + return c + + def batchnorm(self, planes, last_bn=False, first_layer=False): + return self.bn_layer(planes) + + def activation(self): + # always ReLU + return (lambda: nn.ReLU(inplace=True))() + + def _init_conv(self, conv): + if weight_init == "signed_constant": + fan = nn.init._calculate_correct_fan(conv.weight, "fan_in") + # scale_fan = False always because this isn't EP + # if scale_fan: + # fan = fan * (1 - parser_args.prune_rate) + gain = nn.init.calculate_gain("relu") + std = gain / math.sqrt(fan) + conv.weight.data = conv.weight.data.sign() * std + + elif weight_init == "unsigned_constant": + fan = nn.init._calculate_correct_fan(conv.weight, "fan_in") + # scale_fan = False always because this isn't EP + # if parser_args.scale_fan: + # fan = fan * (1 - parser_args.prune_rate) + gain = nn.init.calculate_gain("relu") + std = gain / math.sqrt(fan) + conv.weight.data = torch.ones_like(conv.weight.data) * std + + elif weight_init == "kaiming_normal": + # scale_fan = False always because this isn't EP + # if parser_args.scale_fan: + # fan = nn.init._calculate_correct_fan(conv.weight, parser_args.mode) + # fan = fan * (1 - parser_args.prune_rate) + # gain = nn.init.calculate_gain(parser_args.nonlinearity) + # std = gain / math.sqrt(fan) + # with torch.no_grad(): + # conv.weight.data.normal_(0, std) + # else: + # nn.init.kaiming_normal_( + # conv.weight, mode=parser_args.mode, nonlinearity=parser_args.nonlinearity + # ) + nn.init.kaiming_normal_( + conv.weight, mode="fan_in", nonlinearity="relu" + ) + + elif weight_init == "kaiming_uniform": + nn.init.kaiming_uniform_( + conv.weight, mode="fan_in", nonlinearity="relu" + ) + + elif weight_init == "xavier_normal": + nn.init.xavier_normal_(conv.weight) + + elif weight_init == "xavier_constant": + fan_in, fan_out = nn.init._calculate_fan_in_and_fan_out(conv.weight) + std = math.sqrt(2.0 / float(fan_in + fan_out)) + conv.weight.data = conv.weight.data.sign() * std + + elif weight_init == "standard": + nn.init.kaiming_uniform_(conv.weight, a=math.sqrt(5)) + + else: + raise ValueError(f"{weight_init} is not an initialization option!") + + +def get_builder(): + conv_type = "SubnetConv" + bn_type = "NonAffineBatchNorm" # TODO: might change this if it doesn't affect weights + first_layer_type = None # TODO: I think + weight_init = "signed_constant" + + print("==> Conv Type: {}".format(conv_type)) + print("==> BN Type: {}".format(bn_type)) + + # need to fix this + # conv_layer = getattr(utils.conv_type, parser_args.conv_type) + # bn_layer = getattr(utils.bn_type, parser_args.bn_type) + conv_layer = SubnetConv + bn_layer = NonAffineBatchNorm + + if first_layer_type is not None: + first_layer = getattr(utils.conv_type, first_layer_type) + print(f"==> First Layer Type: {first_layer_type}") + else: + first_layer = None + + builder = Builder(conv_layer=conv_layer, bn_layer=bn_layer, first_layer=first_layer) + + return builder + +class GetSubnet(autograd.Function): + algo = 'hc_iter' + quantize_threshold = 0.5 + @staticmethod + def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_scores_prune_threshold=-np.inf): + if algo == 'ep': + # Get the supermask by sorting the scores and using the top k% + out = scores.clone() + _, idx = scores.flatten().sort() + j = int((1 - k) * scores.numel()) + # flat_out and out access the same memory. + flat_out = out.flatten() + flat_out[idx[:j]] = 0 + flat_out[idx[j:]] = 1 + + # repeat for bias + # Get the supermask by sorting the scores and using the top k% + bias_out = bias_scores.clone() + _, idx = bias_scores.flatten().sort() + j = int((1 - k) * bias_scores.numel()) + + # flat_out and out access the same memory. + bias_flat_out = bias_out.flatten() + bias_flat_out[idx[:j]] = 0 + bias_flat_out[idx[j:]] = 1 + + elif algo in ['global_ep', 'global_ep_iter']: + # define out, bias_out based on the layer's prune_threshold, bias_threshold + out = torch.gt(scores, torch.ones_like(scores)*scores_prune_threshold).float() + bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float() + + elif algo in ['hc', 'hc_iter']: + # round scores to {0, 1} + out = torch.gt(scores, torch.ones_like(scores)*quantize_threshold).float() + bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*quantize_threshold).float() + + else: + print("INVALID PRUNING ALGO") + print("EXITING") + exit() + + return out, bias_out + + @staticmethod + def backward(ctx, g_1, g_2): + # send the gradient g straight-through on the backward pass. + return g_1, g_2, None, None, None + + +# Not learning weights, finding subnet +class SubnetConv(nn.Conv2d): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + args_bias = False + algo = 'hc' + + # initialize flag (representing the pruned weights) + self.flag = nn.Parameter(torch.ones(self.weight.size())) + if args_bias: + self.bias_flag = nn.Parameter(torch.ones(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_flag = nn.Parameter(torch.Tensor(1)) + + # initialize the scores + self.scores = nn.Parameter(torch.Tensor(self.weight.size())) + if args_bias: + self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_scores = nn.Parameter(torch.Tensor(1)) + + # prune scores below this for global EP in bottom-k + self.scores_prune_threshold = -np.inf + self.bias_scores_prune_threshold = -np.inf + + if algo in ['hc', 'hc_iter']: + # score init is always uniform + nn.init.uniform_(self.scores, a=0.0, b=1.0) + nn.init.uniform_(self.bias_scores, a=0.0, b=1.0) + else: + nn.init.kaiming_uniform_(self.scores, a=math.sqrt(5)) + nn.init.uniform_(self.bias_scores, a=-1.0, b=1.0) # can't do kaiming here. picking U[-1, 1] for no real reason + + # NOTE: turn the gradient on the weights off + self.weight.requires_grad = False + self.flag.requires_grad = False + self.bias_flag.requires_grad = False + # TODO: Hacky. I'm trying to mimick frankle etc in that we have biases, but we don't prune them + self.bias.requires_grad = False + + def set_prune_rate(self, prune_rate): + self.prune_rate = prune_rate + + @property + def clamped_scores(self): + return self.scores.abs() + + def forward(self, x): + if algo in ['hc', 'hc_iter', 'transformer']: + subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, self.prune_rate) + subnet = subnet * self.flag.data.float() + bias_subnet = subnet * self.bias_flag.data.float() + elif algo in ['imp']: + # no STE, no subnet. Mask is handled outside + pass + elif algo in ['global_ep', 'global_ep_iter']: + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) + else: + # ep, global_ep, global_ep_iter, pt etc + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), self.prune_rate) + + if algo in ['imp']: + # no STE, no subnet. Mask is handled outside + w = self.weight + b = self.bias + else: + w = self.weight * subnet + if args_bias: + b = self.bias * bias_subnet + else: + b = self.bias + + x = F.conv2d( + x, w, b, self.stride, self.padding, self.dilation, self.groups + ) + + return x + +class NonAffineBatchNorm(nn.BatchNorm2d): + def __init__(self, dim): + super(NonAffineBatchNorm, self).__init__(dim, affine=False) + + +class AffineBatchNorm(nn.BatchNorm2d): + def __init__(self, dim): + super(AffineBatchNorm, self).__init__(dim, affine=True) From c00eca9b9220e7789c050f85a59aeb2b7a5ab585 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 13 May 2022 11:51:45 -0500 Subject: [PATCH 068/113] minor bugfixes to create model with scores --- imagenet/imagenet_main.py | 2 +- imagenet/{resnet.py => models.py} | 50 +++++++++++++++---------------- 2 files changed, 26 insertions(+), 26 deletions(-) rename imagenet/{resnet.py => models.py} (94%) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index fbaccadc..5be66bf4 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -464,4 +464,4 @@ def accuracy(output, target, topk=(1,)): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/imagenet/resnet.py b/imagenet/models.py similarity index 94% rename from imagenet/resnet.py rename to imagenet/models.py index 5e0637e9..9ae1a40d 100644 --- a/imagenet/resnet.py +++ b/imagenet/models.py @@ -93,9 +93,9 @@ def forward(self, x): # ResNet {{{ class ResNet(nn.Module): def __init__(self, builder, block, layers, num_classes=1000, base_width=64): - args_first_layer_dense = False - args_last_layer_dense = False - args_bias = False + self.args_first_layer_dense = False + self.args_last_layer_dense = False + self.args_bias = False self.inplanes = 64 super(ResNet, self).__init__() @@ -104,7 +104,7 @@ def __init__(self, builder, block, layers, num_classes=1000, base_width=64): if self.base_width // 64 > 1: print(f"==> Using {self.base_width // 64}x wide model") - if args_first_layer_dense: + if self.args_first_layer_dense: self.conv1 = nn.Conv2d( 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False ) @@ -121,7 +121,7 @@ def __init__(self, builder, block, layers, num_classes=1000, base_width=64): self.avgpool = nn.AdaptiveAvgPool2d(1) # self.fc = nn.Linear(512 * block.expansion, num_classes) - if args_last_layer_dense: + if self.args_last_layer_dense: self.fc = nn.Conv2d(512 * block.expansion, num_classes, 1) else: self.fc = builder.conv1x1(512 * block.expansion, num_classes) @@ -148,12 +148,12 @@ def _make_layer(self, builder, block, planes, blocks, stride=1): return nn.Sequential(*layers) - def get_prunable_param_names(model): - prunable_weights = [name + '.weight' for name, module in model.named_modules() if + def get_prunable_param_names(self): + prunable_weights = [name + '.weight' for name, module in self.named_modules() if isinstance(module, nn.modules.conv.Conv2d) or isinstance(module, nn.modules.linear.Linear)] - if args_bias: - prunable_biases = [name + '.bias' for name, module in model.named_modules() if + if self.args_bias: + prunable_biases = [name + '.bias' for name, module in self.named_modules() if isinstance(module, nn.modules.conv.Conv2d) or isinstance(module, nn.modules.linear.Linear)] else: @@ -301,7 +301,7 @@ def activation(self): return (lambda: nn.ReLU(inplace=True))() def _init_conv(self, conv): - if weight_init == "signed_constant": + if self.weight_init == "signed_constant": fan = nn.init._calculate_correct_fan(conv.weight, "fan_in") # scale_fan = False always because this isn't EP # if scale_fan: @@ -310,7 +310,7 @@ def _init_conv(self, conv): std = gain / math.sqrt(fan) conv.weight.data = conv.weight.data.sign() * std - elif weight_init == "unsigned_constant": + elif self.weight_init == "unsigned_constant": fan = nn.init._calculate_correct_fan(conv.weight, "fan_in") # scale_fan = False always because this isn't EP # if parser_args.scale_fan: @@ -319,7 +319,7 @@ def _init_conv(self, conv): std = gain / math.sqrt(fan) conv.weight.data = torch.ones_like(conv.weight.data) * std - elif weight_init == "kaiming_normal": + elif self.weight_init == "kaiming_normal": # scale_fan = False always because this isn't EP # if parser_args.scale_fan: # fan = nn.init._calculate_correct_fan(conv.weight, parser_args.mode) @@ -336,20 +336,20 @@ def _init_conv(self, conv): conv.weight, mode="fan_in", nonlinearity="relu" ) - elif weight_init == "kaiming_uniform": + elif self.weight_init == "kaiming_uniform": nn.init.kaiming_uniform_( conv.weight, mode="fan_in", nonlinearity="relu" ) - elif weight_init == "xavier_normal": + elif self.weight_init == "xavier_normal": nn.init.xavier_normal_(conv.weight) - elif weight_init == "xavier_constant": + elif self.weight_init == "xavier_constant": fan_in, fan_out = nn.init._calculate_fan_in_and_fan_out(conv.weight) std = math.sqrt(2.0 / float(fan_in + fan_out)) conv.weight.data = conv.weight.data.sign() * std - elif weight_init == "standard": + elif self.weight_init == "standard": nn.init.kaiming_uniform_(conv.weight, a=math.sqrt(5)) else: @@ -434,12 +434,12 @@ def backward(ctx, g_1, g_2): class SubnetConv(nn.Conv2d): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - args_bias = False - algo = 'hc' + self.args_bias = False + self.algo = 'hc' # initialize flag (representing the pruned weights) self.flag = nn.Parameter(torch.ones(self.weight.size())) - if args_bias: + if self.args_bias: self.bias_flag = nn.Parameter(torch.ones(self.bias.size())) else: # dummy variable just so other things don't break @@ -447,7 +447,7 @@ def __init__(self, *args, **kwargs): # initialize the scores self.scores = nn.Parameter(torch.Tensor(self.weight.size())) - if args_bias: + if self.args_bias: self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) else: # dummy variable just so other things don't break @@ -457,7 +457,7 @@ def __init__(self, *args, **kwargs): self.scores_prune_threshold = -np.inf self.bias_scores_prune_threshold = -np.inf - if algo in ['hc', 'hc_iter']: + if self.algo in ['hc', 'hc_iter']: # score init is always uniform nn.init.uniform_(self.scores, a=0.0, b=1.0) nn.init.uniform_(self.bias_scores, a=0.0, b=1.0) @@ -480,20 +480,20 @@ def clamped_scores(self): return self.scores.abs() def forward(self, x): - if algo in ['hc', 'hc_iter', 'transformer']: + if self.algo in ['hc', 'hc_iter', 'transformer']: subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, self.prune_rate) subnet = subnet * self.flag.data.float() bias_subnet = subnet * self.bias_flag.data.float() - elif algo in ['imp']: + elif self.algo in ['imp']: # no STE, no subnet. Mask is handled outside pass - elif algo in ['global_ep', 'global_ep_iter']: + elif self.algo in ['global_ep', 'global_ep_iter']: subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) else: # ep, global_ep, global_ep_iter, pt etc subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), self.prune_rate) - if algo in ['imp']: + if self.algo in ['imp']: # no STE, no subnet. Mask is handled outside w = self.weight b = self.bias From a40f048a4a67799d5807aa736e19336ad15ddbdf Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 15:49:54 -0500 Subject: [PATCH 069/113] deleting whitespace --- imagenet/imagenet_main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 5be66bf4..bcb6b91f 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -461,7 +461,5 @@ def accuracy(output, target, topk=(1,)): """ - - if __name__ == '__main__': main() From 721b71c945e9fed2eae3b42810d58ee4795ae42c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 13 May 2022 21:02:33 +0000 Subject: [PATCH 070/113] adding run script --- imagenet/imagenet_main.py | 2 +- imagenet/imagenet_main_bkp.py | 4 ++-- imagenet/run.sh | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 imagenet/run.sh diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index bcb6b91f..0416f65b 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -30,7 +30,7 @@ and callable(models.__dict__[name])) parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') -parser.add_argument('data', metavar='DIR', default='imagenet', +parser.add_argument('--data', metavar='DIR', default='/home/ubuntu/ILSVRC2012/', help='path to dataset (default: imagenet)') parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', choices=model_names, diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py index 30fd04c8..e8175e0b 100644 --- a/imagenet/imagenet_main_bkp.py +++ b/imagenet/imagenet_main_bkp.py @@ -25,7 +25,7 @@ and callable(models.__dict__[name])) parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') -parser.add_argument('data', metavar='DIR', default='imagenet', +parser.add_argument('--data', metavar='DIR', default='imagenet', help='path to dataset (default: imagenet)') parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', choices=model_names, @@ -451,4 +451,4 @@ def accuracy(output, target, topk=(1,)): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/imagenet/run.sh b/imagenet/run.sh new file mode 100644 index 00000000..655aca01 --- /dev/null +++ b/imagenet/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +python imagenet_main_bkp.py \ + -a resnet50 \ + --dist-url 'tcp://127.0.0.1:2500' \ + --dist-backend 'nccl' \ + --multiprocessing-distributed \ + --world-size 1 \ + --rank 0 \ + --data '/home/ubuntu/ILSVRC2012' + From 94dc5efd98f9c33b78e898bdd79d7b05709d01ec Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 16:16:51 -0500 Subject: [PATCH 071/113] trying to add mixed_precision to see if I can increase batch size to 1k --- imagenet/imagenet_main_bkp.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py index e8175e0b..30cb3440 100644 --- a/imagenet/imagenet_main_bkp.py +++ b/imagenet/imagenet_main_bkp.py @@ -75,6 +75,10 @@ 'N processes per node, which has N GPUs. This is the ' 'fastest way to use PyTorch for either single node or ' 'multi node data parallel training') +parser.add_argument("--mixed_precision", + action='store_true', + default=False, + help="Use mixed precision or not") best_acc1 = 0 @@ -178,6 +182,12 @@ def main_worker(gpu, ngpus_per_node, args): """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" scheduler = StepLR(optimizer, step_size=30, gamma=0.1) + + # adding a scaler + if args.mixed_precision: + scaler = torch.cuda.amp.GradScaler(enabled=True) # mixed precision + else: + scaler = None # optionally resume from a checkpoint if args.resume: @@ -296,8 +306,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args): target = target.cuda(args.gpu, non_blocking=True) # compute output - output = model(images) - loss = criterion(output, target) + if scaler is None: + output = model(images) + loss = criterion(output, target) + else: + with torch.cuda.amp.autocast(enabled=True): + output = model(images) + loss = criterion(output, target) # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 5)) @@ -307,8 +322,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args): # compute gradient and do SGD step optimizer.zero_grad() - loss.backward() - optimizer.step() + if scaler is None: + loss.backward() + optimizer.step() + else: + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() # measure elapsed time batch_time.update(time.time() - end) From 8a87780203378393b015912b05297fbf477bbebd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 13 May 2022 21:23:38 +0000 Subject: [PATCH 072/113] okay looks like mixed precision did the trick! --- imagenet/imagenet_main_bkp.py | 6 +++--- imagenet/run.sh | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py index 30cb3440..d6f0cf70 100644 --- a/imagenet/imagenet_main_bkp.py +++ b/imagenet/imagenet_main_bkp.py @@ -75,7 +75,7 @@ 'N processes per node, which has N GPUs. This is the ' 'fastest way to use PyTorch for either single node or ' 'multi node data parallel training') -parser.add_argument("--mixed_precision", +parser.add_argument("--mixed-precision", action='store_true', default=False, help="Use mixed precision or not") @@ -257,7 +257,7 @@ def main_worker(gpu, ngpus_per_node, args): train_sampler.set_epoch(epoch) # train for one epoch - train(train_loader, model, criterion, optimizer, epoch, args) + train(train_loader, model, criterion, optimizer, epoch, args, scaler) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args) @@ -281,7 +281,7 @@ def main_worker(gpu, ngpus_per_node, args): }, is_best) -def train(train_loader, model, criterion, optimizer, epoch, args): +def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): batch_time = AverageMeter('Time', ':6.3f') data_time = AverageMeter('Data', ':6.3f') losses = AverageMeter('Loss', ':.4e') diff --git a/imagenet/run.sh b/imagenet/run.sh index 655aca01..91d01ec4 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -7,5 +7,7 @@ python imagenet_main_bkp.py \ --multiprocessing-distributed \ --world-size 1 \ --rank 0 \ + --batch-size 1024 \ + --mixed-precision \ --data '/home/ubuntu/ILSVRC2012' From ae77d7bbfd83c8dd83f282915cdfbad0e9025f4d Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 18:08:08 -0500 Subject: [PATCH 073/113] adding timing logs and checking if it works for GM --- imagenet/imagenet_main.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 0416f65b..00f63b0a 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -80,6 +80,10 @@ 'N processes per node, which has N GPUs. This is the ' 'fastest way to use PyTorch for either single node or ' 'multi node data parallel training') +parser.add_argument("--mixed-precision", + action='store_true', + default=False, + help="Use mixed precision or not") best_acc1 = 0 @@ -184,6 +188,12 @@ def main_worker(gpu, ngpus_per_node, args): """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" scheduler = StepLR(optimizer, step_size=30, gamma=0.1) + + # adding a scaler for mixed-precision training + if args.mixed_precision: + scaler = torch.cuda.amp.GradScaler(enabled=True) # mixed precision + else: + scaler = None # optionally resume from a checkpoint if args.resume: @@ -252,11 +262,18 @@ def main_worker(gpu, ngpus_per_node, args): if args.distributed: train_sampler.set_epoch(epoch) + start_train = time.time() # train for one epoch - train(train_loader, model, criterion, optimizer, epoch, args) + train(train_loader, model, criterion, optimizer, epoch, args, scaler) + + train_time = train_time = (time.time() - start_train) / 60 + print("Epoch: {} | Train Time {}".format(train_time)) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args) + + epoch_time = (time.time() - start_train) / 60 + print("Epoch: {} | Train + Val Time {}".format(epoch_time)) scheduler.step() @@ -277,7 +294,7 @@ def main_worker(gpu, ngpus_per_node, args): }, is_best) -def train(train_loader, model, criterion, optimizer, epoch, args): +def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): batch_time = AverageMeter('Time', ':6.3f') data_time = AverageMeter('Data', ':6.3f') losses = AverageMeter('Loss', ':.4e') From 8d718910599ce22e495d30abbcfd506feab7c468 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 18:10:48 -0500 Subject: [PATCH 074/113] adding few more timing logs --- imagenet/imagenet_main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 00f63b0a..c1fe0440 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -262,18 +262,19 @@ def main_worker(gpu, ngpus_per_node, args): if args.distributed: train_sampler.set_epoch(epoch) + print_time("Epoch: {} | Starting Train".format(epoch)) start_train = time.time() # train for one epoch train(train_loader, model, criterion, optimizer, epoch, args, scaler) train_time = train_time = (time.time() - start_train) / 60 - print("Epoch: {} | Train Time {}".format(train_time)) + print("Epoch: {} | Train Time {}".format(epoch, train_time)) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args) epoch_time = (time.time() - start_train) / 60 - print("Epoch: {} | Train + Val Time {}".format(epoch_time)) + print("Epoch: {} | Train + Val Time {}".format(epoch, epoch_time)) scheduler.step() @@ -476,6 +477,12 @@ def accuracy(output, target, topk=(1,)): """ @ksreenivasan: Dumping new code here """ +def print_time(msg): + print("\n\n----------------------------------------------------------------------------") + print("{}".format(msg)) + print("TIME: The current time is: {}".format(time.ctime())) + print("TIME: The current time in seconds is: {}".format(time.time())) + print("----------------------------------------------------------------------------\n\n") if __name__ == '__main__': From 123411d71b8d2c004564c6122928dc80f6b402d0 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 13 May 2022 23:24:26 +0000 Subject: [PATCH 075/113] minor bugfixes in subnetconv --- imagenet/imagenet_main.py | 1 - imagenet/models.py | 7 ++++--- imagenet/run.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index c1fe0440..35be1a78 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -33,7 +33,6 @@ parser.add_argument('--data', metavar='DIR', default='/home/ubuntu/ILSVRC2012/', help='path to dataset (default: imagenet)') parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', - choices=model_names, help='model architecture: ' + ' | '.join(model_names) + ' (default: resnet18)') diff --git a/imagenet/models.py b/imagenet/models.py index 9ae1a40d..a076c387 100644 --- a/imagenet/models.py +++ b/imagenet/models.py @@ -382,10 +382,10 @@ def get_builder(): return builder class GetSubnet(autograd.Function): - algo = 'hc_iter' - quantize_threshold = 0.5 @staticmethod def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_scores_prune_threshold=-np.inf): + algo = 'hc_iter' + quantize_threshold = 0.5 if algo == 'ep': # Get the supermask by sorting the scores and using the top k% out = scores.clone() @@ -436,6 +436,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.args_bias = False self.algo = 'hc' + self.prune_rate = 0.5 # initialize flag (representing the pruned weights) self.flag = nn.Parameter(torch.ones(self.weight.size())) @@ -499,7 +500,7 @@ def forward(self, x): b = self.bias else: w = self.weight * subnet - if args_bias: + if self.args_bias: b = self.bias * bias_subnet else: b = self.bias diff --git a/imagenet/run.sh b/imagenet/run.sh index 91d01ec4..204beeef 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -python imagenet_main_bkp.py \ +python imagenet_main.py \ -a resnet50 \ --dist-url 'tcp://127.0.0.1:2500' \ --dist-backend 'nccl' \ @@ -9,5 +9,5 @@ python imagenet_main_bkp.py \ --rank 0 \ --batch-size 1024 \ --mixed-precision \ - --data '/home/ubuntu/ILSVRC2012' + --data '/home/ubuntu/ILSVRC2012' > "resnet50_imagenet_log" 2>&1 & From bf6dbe454e6f7052d3aa85e288f0b39bc8575504 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 18:25:26 -0500 Subject: [PATCH 076/113] adding scaler in main training loop --- imagenet/imagenet_main.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 35be1a78..170cc22c 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -319,8 +319,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): target = target.cuda(args.gpu, non_blocking=True) # compute output - output = model(images) - loss = criterion(output, target) + if scaler is None: + output = model(images) + loss = criterion(output, target) + else: + with torch.cuda.amp.autocast(enabled=True): + output = model(images) + loss = criterion(output, target) # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 5)) @@ -330,8 +335,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): # compute gradient and do SGD step optimizer.zero_grad() - loss.backward() - optimizer.step() + if scaler is None: + loss.backward() + optimizer.step() + else: + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() # measure elapsed time batch_time.update(time.time() - end) From 5f6aafec5aba6c3662d41690868b96147849d063 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 20:36:37 -0500 Subject: [PATCH 077/113] adding timing logs to original pytorch example to compare --- imagenet/imagenet_main.py | 6 +++--- imagenet/imagenet_main_bkp.py | 16 ++++++++++++++++ imagenet/run.sh | 3 ++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 170cc22c..3b28b030 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -140,10 +140,10 @@ def main_worker(gpu, ngpus_per_node, args): world_size=args.world_size, rank=args.rank) # create model if args.pretrained: - print("=> using pre-trained model '{}'".format(args.arch)) + print("==> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) else: - print("=> creating model '{}'".format(args.arch)) + print("==> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() model = models.ResNet50() @@ -274,7 +274,7 @@ def main_worker(gpu, ngpus_per_node, args): epoch_time = (time.time() - start_train) / 60 print("Epoch: {} | Train + Val Time {}".format(epoch, epoch_time)) - + scheduler.step() diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py index d6f0cf70..c045b872 100644 --- a/imagenet/imagenet_main_bkp.py +++ b/imagenet/imagenet_main_bkp.py @@ -256,14 +256,22 @@ def main_worker(gpu, ngpus_per_node, args): if args.distributed: train_sampler.set_epoch(epoch) + print_time("Epoch: {} | Starting Train".format(epoch)) + start_train = time.time() + # train for one epoch train(train_loader, model, criterion, optimizer, epoch, args, scaler) + train_time = train_time = (time.time() - start_train) / 60 + print("Epoch: {} | Train Time {}".format(epoch, train_time)) + # evaluate on validation set acc1 = validate(val_loader, model, criterion, args) scheduler.step() + epoch_time = (time.time() - start_train) / 60 + print("Epoch: {} | Train + Val Time {}".format(epoch, epoch_time)) # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 @@ -469,6 +477,14 @@ def accuracy(output, target, topk=(1,)): res.append(correct_k.mul_(100.0 / batch_size)) return res +def print_time(msg): + print("\n\n----------------------------------------------------------------------------") + print("{}".format(msg)) + print("TIME: The current time is: {}".format(time.ctime())) + print("TIME: The current time in seconds is: {}".format(time.time())) + print("----------------------------------------------------------------------------\n\n") + + if __name__ == '__main__': main() diff --git a/imagenet/run.sh b/imagenet/run.sh index 204beeef..8c8de5c4 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -python imagenet_main.py \ +python imagenet_main_bkp.py \ -a resnet50 \ --dist-url 'tcp://127.0.0.1:2500' \ --dist-backend 'nccl' \ @@ -8,6 +8,7 @@ python imagenet_main.py \ --world-size 1 \ --rank 0 \ --batch-size 1024 \ + --workers 12 \ --mixed-precision \ --data '/home/ubuntu/ILSVRC2012' > "resnet50_imagenet_log" 2>&1 & From 6e40fca1eac117d78fdf2f693d0695c4273dd0b5 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 22:21:12 -0500 Subject: [PATCH 078/113] adding lists to store results --- imagenet/imagenet_main.py | 48 ++++++++++++++++++++++++++++----------- imagenet/models.py | 2 +- imagenet/run.sh | 2 ++ 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 3b28b030..4aab949e 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -257,6 +257,12 @@ def main_worker(gpu, ngpus_per_node, args): validate(val_loader, model, criterion, args) return + epoch_list = [] + test_acc_list = [] + model_sparsity_list = [] + val_acc_list = [] + train_acc_list = [] + for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) @@ -264,7 +270,7 @@ def main_worker(gpu, ngpus_per_node, args): print_time("Epoch: {} | Starting Train".format(epoch)) start_train = time.time() # train for one epoch - train(train_loader, model, criterion, optimizer, epoch, args, scaler) + train_acc1 = train(train_loader, model, criterion, optimizer, epoch, args, scaler) train_time = train_time = (time.time() - start_train) / 60 print("Epoch: {} | Train Time {}".format(epoch, train_time)) @@ -275,23 +281,37 @@ def main_worker(gpu, ngpus_per_node, args): epoch_time = (time.time() - start_train) / 60 print("Epoch: {} | Train + Val Time {}".format(epoch, epoch_time)) + epoch_list.append(epoch) + train_acc_list.append(train_acc1) + test_acc_list.append(acc1) + val_acc_list.append(acc1) + scheduler.step() - + results_df = pd.DataFrame({'epoch': epoch_list, + 'test_acc': test_acc_list, + 'val_acc': val_acc_list, + 'train_acc': train_acc_list,}) + # 'regularization_loss': reg_loss_list, + # 'model_sparsity': model_sparsity_list}) + # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) - - if not args.multiprocessing_distributed or (args.multiprocessing_distributed - and args.rank % ngpus_per_node == 0): - save_checkpoint({ - 'epoch': epoch + 1, - 'arch': args.arch, - 'state_dict': model.state_dict(), - 'best_acc1': best_acc1, - 'optimizer' : optimizer.state_dict(), - 'scheduler' : scheduler.state_dict() - }, is_best) + results_df.to_csv("acc_and_sparsity.csv", index=False) + + save_flag = ((epoch+1)%10 == 0) or (epoch > 85) + if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed + and args.rank % ngpus_per_node == 0)): + torch.save(model.state_dict(), 'model_before_fineune_epoch_{}.pth'.format(epoch)) + # save_checkpoint({ + # 'epoch': epoch + 1, + # 'arch': args.arch, + # 'state_dict': model.state_dict(), + # 'best_acc1': best_acc1, + # 'optimizer' : optimizer.state_dict(), + # 'scheduler' : scheduler.state_dict() + # }, is_best) def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): @@ -350,6 +370,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): if i % args.print_freq == 0: progress.display(i) + return top1.avg + def validate(val_loader, model, criterion, args): batch_time = AverageMeter('Time', ':6.3f', Summary.NONE) diff --git a/imagenet/models.py b/imagenet/models.py index a076c387..6f2e56c8 100644 --- a/imagenet/models.py +++ b/imagenet/models.py @@ -493,7 +493,7 @@ def forward(self, x): else: # ep, global_ep, global_ep_iter, pt etc subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), self.prune_rate) - + if self.algo in ['imp']: # no STE, no subnet. Mask is handled outside w = self.weight diff --git a/imagenet/run.sh b/imagenet/run.sh index 8c8de5c4..1c2cf863 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -10,5 +10,7 @@ python imagenet_main_bkp.py \ --batch-size 1024 \ --workers 12 \ --mixed-precision \ + --epochs 88 \ + --lr 0.4 \ --data '/home/ubuntu/ILSVRC2012' > "resnet50_imagenet_log" 2>&1 & From 02d3a786e764d02cb0cd7892a033ed1048f5f2be Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 23:01:45 -0500 Subject: [PATCH 079/113] adding regularizer and some other features --- imagenet/imagenet_main.py | 58 ++++++++++++++++++++++++++++++++++++--- imagenet/run.sh | 1 + 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 4aab949e..1b5e98b4 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -83,6 +83,10 @@ action='store_true', default=False, help="Use mixed precision or not") +parser.add_argument('--lmbda', + type=float, + default=0.001, + help='regularization coefficient lambda') best_acc1 = 0 @@ -282,16 +286,16 @@ def main_worker(gpu, ngpus_per_node, args): print("Epoch: {} | Train + Val Time {}".format(epoch, epoch_time)) epoch_list.append(epoch) - train_acc_list.append(train_acc1) - test_acc_list.append(acc1) - val_acc_list.append(acc1) + train_acc_list.append(train_acc1.item()) + test_acc_list.append(acc1.item()) + val_acc_list.append(acc1.item()) scheduler.step() results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, - 'train_acc': train_acc_list,}) + 'train_acc': train_acc_list}) # 'regularization_loss': reg_loss_list, # 'model_sparsity': model_sparsity_list}) @@ -338,6 +342,16 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): if torch.cuda.is_available(): target = target.cuda(args.gpu, non_blocking=True) + for name, params in model.named_parameters(): + # make sure param_name ends with .weight or .bias + if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): + with torch.no_grad(): + params.data = torch.clamp(params.data, 0.0, 1.0) + + regularization_loss = torch.tensor(0) + regularization_loss = get_regularization_loss(model, lmbda=args.lmbda) + loss += regularization_loss + # compute output if scaler is None: output = model(images) @@ -515,6 +529,42 @@ def print_time(msg): print("TIME: The current time in seconds is: {}".format(time.time())) print("----------------------------------------------------------------------------\n\n") +def get_regularization_loss(model, args): + conv_layers, linear_layers = get_layers(args.arch, model) + regularization_loss = torch.tensor(0.).to(args.gpu) + + # reg_loss = ||p||_2^2 + for name, params in model.named_parameters(): + if ".bias_score" in nameZ + # do nothing, because I'm pretending there are no biases + regularization_loss += 0 + + elif ".score" in name: + regularization_loss += torch.norm(params, p=2)**2 + regularization_loss = args.lmbda * regularization_loss + return regularization_loss + +# return layer objects of conv layers and linear layers so we can parse them +# efficiently +def get_layers(arch='ResNet50', dist_model=None): + if isinstance(dist_model, nn.parallel.DistributedDataParallel): + model = dist_model.module + else: + model = dist_model + + if arch == 'ResNet50': + conv_layers = [model.conv1] + for layer in [model.layer1, model.layer2, model.layer3, model.layer4]: + for basic_block_id in [i for i in range(len(layer))]: + conv_layers.append(layer[basic_block_id].conv1) + conv_layers.append(layer[basic_block_id].conv2) + conv_layers.append(layer[basic_block_id].conv3) + # handle shortcut + # if len(layer[basic_block_id].shortcut) > 0: + # conv_layers.append(layer[basic_block_id].shortcut[0]) + linear_layers = [model.fc] + + return (conv_layers, linear_layers) if __name__ == '__main__': main() diff --git a/imagenet/run.sh b/imagenet/run.sh index 1c2cf863..a63220ff 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -12,5 +12,6 @@ python imagenet_main_bkp.py \ --mixed-precision \ --epochs 88 \ --lr 0.4 \ + --lmbda 0.000001 \ --data '/home/ubuntu/ILSVRC2012' > "resnet50_imagenet_log" 2>&1 & From 1b7313c050f57eb805c1d4af77579f436bbf3d51 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 23:18:43 -0500 Subject: [PATCH 080/113] adding imports --- imagenet/imagenet_main.py | 1 + imagenet/run.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 1b5e98b4..8f09a388 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -7,6 +7,7 @@ from enum import Enum import math import numpy as np +import pandas as pd import torch import torch.nn as nn diff --git a/imagenet/run.sh b/imagenet/run.sh index a63220ff..f6d09c9b 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -1,7 +1,7 @@ #!/bin/bash -python imagenet_main_bkp.py \ - -a resnet50 \ +python imagenet_main.py \ + --arch resnet50 \ --dist-url 'tcp://127.0.0.1:2500' \ --dist-backend 'nccl' \ --multiprocessing-distributed \ @@ -13,5 +13,5 @@ python imagenet_main_bkp.py \ --epochs 88 \ --lr 0.4 \ --lmbda 0.000001 \ - --data '/home/ubuntu/ILSVRC2012' > "resnet50_imagenet_log" 2>&1 & + --data '/home/ubuntu/ILSVRC2012' #> "resnet50_imagenet_log" 2>&1 & From 9834796c0c7c3633e3cb23f15cf5b2f66ba32c99 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 14 May 2022 04:28:59 +0000 Subject: [PATCH 081/113] minor bugfixes --- imagenet/imagenet_main.py | 5 +++-- imagenet/run.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 8f09a388..ba3b139e 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -8,6 +8,7 @@ import math import numpy as np import pandas as pd +import re import torch import torch.nn as nn @@ -350,7 +351,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): params.data = torch.clamp(params.data, 0.0, 1.0) regularization_loss = torch.tensor(0) - regularization_loss = get_regularization_loss(model, lmbda=args.lmbda) + regularization_loss = get_regularization_loss(model, args) loss += regularization_loss # compute output @@ -536,7 +537,7 @@ def get_regularization_loss(model, args): # reg_loss = ||p||_2^2 for name, params in model.named_parameters(): - if ".bias_score" in nameZ + if ".bias_score" in name: # do nothing, because I'm pretending there are no biases regularization_loss += 0 diff --git a/imagenet/run.sh b/imagenet/run.sh index f6d09c9b..4a2a8d9b 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -1,7 +1,7 @@ #!/bin/bash python imagenet_main.py \ - --arch resnet50 \ + --arch ResNet50 \ --dist-url 'tcp://127.0.0.1:2500' \ --dist-backend 'nccl' \ --multiprocessing-distributed \ From 0e0662a6f19eeb5a58ac63255461256fdbae8362 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 13 May 2022 23:33:40 -0500 Subject: [PATCH 082/113] adding additional project step and reordering reg_loss --- imagenet/imagenet_main.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index ba3b139e..7deed20f 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -344,16 +344,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): if torch.cuda.is_available(): target = target.cuda(args.gpu, non_blocking=True) + # project to [0, 1] in every gradient step for name, params in model.named_parameters(): - # make sure param_name ends with .weight or .bias + # make sure param_name ends with .scores and not bias_scores if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): with torch.no_grad(): params.data = torch.clamp(params.data, 0.0, 1.0) - regularization_loss = torch.tensor(0) - regularization_loss = get_regularization_loss(model, args) - loss += regularization_loss - # compute output if scaler is None: output = model(images) @@ -363,6 +360,10 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): output = model(images) loss = criterion(output, target) + regularization_loss = torch.tensor(0) + regularization_loss = get_regularization_loss(model, args) + loss += regularization_loss + # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 5)) losses.update(loss.item(), images.size(0)) @@ -386,6 +387,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): if i % args.print_freq == 0: progress.display(i) + # project to [0, 1] before returning model + for name, params in model.named_parameters(): + # make sure param_name ends with .scores and not bias_scores + if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): + with torch.no_grad(): + params.data = torch.clamp(params.data, 0.0, 1.0) + return top1.avg From 9205fb9717b338c2cf8a9a5010664aa5c809b3f0 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 14 May 2022 04:37:27 +0000 Subject: [PATCH 083/113] adding a commented print to check regularization_loss is working --- imagenet/imagenet_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 7deed20f..84006391 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -362,6 +362,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): regularization_loss = torch.tensor(0) regularization_loss = get_regularization_loss(model, args) + # print("Regularization loss: {}".format(regularization_loss.item())) loss += regularization_loss # measure accuracy and record loss From 73f36ab7fdd6ed8a23875264e831ab4278e4d05e Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sat, 14 May 2022 01:34:58 -0500 Subject: [PATCH 084/113] trying switch_to_wt to sanity check finetune --- imagenet/imagenet_main.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 7deed20f..804cd34a 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -187,7 +187,9 @@ def main_worker(gpu, ngpus_per_node, args): # define loss function (criterion), optimizer, and learning rate scheduler criterion = nn.CrossEntropyLoss().cuda(args.gpu) - optimizer = torch.optim.SGD(model.parameters(), args.lr, + print("Switching to wt to see if that works well!") + switch_to_wt(model) + optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) @@ -576,5 +578,24 @@ def get_layers(arch='ResNet50', dist_model=None): return (conv_layers, linear_layers) + +# switches off gradients for scores and flags and switches it on for weights and biases +def switch_to_wt(model): + print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') + + for name, params in model.named_parameters(): + # make sure param_name ends with .weight or .bias + if re.match('.*\.weight', name): + params.requires_grad = True + elif re.match('.*\.bias$', name): + params.requires_grad = True + elif "score" in name: + params.requires_grad = False + else: + # flags and everything else + params.requires_grad = False + + return model + if __name__ == '__main__': main() From 04be2b88cef312f6ad898b8753ac2e8c31ac5b27 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sat, 14 May 2022 01:40:31 -0500 Subject: [PATCH 085/113] adding round_all_ones before switch_to_wt --- imagenet/imagenet_main.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 45de2fbe..0f02dc99 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -187,7 +187,9 @@ def main_worker(gpu, ngpus_per_node, args): # define loss function (criterion), optimizer, and learning rate scheduler criterion = nn.CrossEntropyLoss().cuda(args.gpu) - print("Switching to wt to see if that works well!") + print("GPU: {} | Switching to wt to see if that works well!".format(args.gpu)) + print("GPU: {} | First round model to all ones score".format(args.gpu)) + model = round_model(model, round_scheme='naive') switch_to_wt(model) optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.lr, momentum=args.momentum, @@ -598,5 +600,35 @@ def switch_to_wt(model): return model +def round_model(model, round_scheme='naive'): + print("Rounding model with scheme: {}".format(round_scheme)) + cp_model = copy.deepcopy(model) + if isinstance(model, nn.parallel.DistributedDataParallel): + # cp_model = copy.deepcopy(model.module) + named_params = cp_model.module.named_parameters() + else: + # cp_model = copy.deepcopy(model) + named_params = cp_model.named_parameters() + for name, params in named_params: + if ".score" in name: + if round_scheme == 'naive': + params.data = torch.gt(params.data, torch.ones_like( + params.data)*parser_args.quantize_threshold).int().float() + elif round_scheme == 'prob': + params.data = torch.clamp(params.data, 0.0, 1.0) + params.data = torch.bernoulli(params.data).float() + elif round_scheme == 'all_ones': + params.data = torch.ones_like(params.data) + else: + print("INVALID ROUNDING") + print("EXITING") + exit() + + # if isinstance(model, nn.parallel.DistributedDataParallel): + # cp_model = nn.parallel.DistributedDataParallel( + # cp_model, device_ids=[parser_args.gpu], find_unused_parameters=False) + + return cp_model + if __name__ == '__main__': main() From d2dbd2b2b971857d84ca4febf8c02c58b7fcf6ef Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 14 May 2022 21:57:16 +0000 Subject: [PATCH 086/113] trying gm with switch_to_wt to sanity check the code --- imagenet/imagenet_main.py | 9 ++++++--- imagenet/run.sh | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 0f02dc99..02f7b8f4 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -9,6 +9,7 @@ import numpy as np import pandas as pd import re +import copy import torch import torch.nn as nn @@ -149,6 +150,7 @@ def main_worker(gpu, ngpus_per_node, args): print("==> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) else: + # args.arch = 'resnet50' print("==> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() model = models.ResNet50() @@ -189,7 +191,7 @@ def main_worker(gpu, ngpus_per_node, args): print("GPU: {} | Switching to wt to see if that works well!".format(args.gpu)) print("GPU: {} | First round model to all ones score".format(args.gpu)) - model = round_model(model, round_scheme='naive') + model = round_model(model, round_scheme='all_ones') switch_to_wt(model) optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.lr, momentum=args.momentum, @@ -365,7 +367,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): loss = criterion(output, target) regularization_loss = torch.tensor(0) - regularization_loss = get_regularization_loss(model, args) + #regularization_loss = get_regularization_loss(model, args) # print("Regularization loss: {}".format(regularization_loss.item())) loss += regularization_loss @@ -601,6 +603,7 @@ def switch_to_wt(model): return model def round_model(model, round_scheme='naive'): + quantize_threshold=0.5 print("Rounding model with scheme: {}".format(round_scheme)) cp_model = copy.deepcopy(model) if isinstance(model, nn.parallel.DistributedDataParallel): @@ -613,7 +616,7 @@ def round_model(model, round_scheme='naive'): if ".score" in name: if round_scheme == 'naive': params.data = torch.gt(params.data, torch.ones_like( - params.data)*parser_args.quantize_threshold).int().float() + params.data)*quantize_threshold).int().float() elif round_scheme == 'prob': params.data = torch.clamp(params.data, 0.0, 1.0) params.data = torch.bernoulli(params.data).float() diff --git a/imagenet/run.sh b/imagenet/run.sh index 4a2a8d9b..36bc12f2 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -12,6 +12,6 @@ python imagenet_main.py \ --mixed-precision \ --epochs 88 \ --lr 0.4 \ - --lmbda 0.000001 \ + --lmbda 0.000000 \ --data '/home/ubuntu/ILSVRC2012' #> "resnet50_imagenet_log" 2>&1 & From 910486ca47f44f0863bc0dcb0f5de309530abdc1 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sat, 14 May 2022 16:58:00 -0500 Subject: [PATCH 087/113] making sure we save model.module not DDP model itself --- imagenet/imagenet_main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 02f7b8f4..d8f23568 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -310,12 +310,13 @@ def main_worker(gpu, ngpus_per_node, args): # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) - results_df.to_csv("acc_and_sparsity.csv", index=False) + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0): + results_df.to_csv("acc_and_sparsity.csv", index=False) save_flag = ((epoch+1)%10 == 0) or (epoch > 85) - if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed - and args.rank % ngpus_per_node == 0)): - torch.save(model.state_dict(), 'model_before_fineune_epoch_{}.pth'.format(epoch)) + if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0)): + torch.save(model.module.state_dict(), 'model_before_fineune_epoch_{}.pth'.format(epoch)) # save_checkpoint({ # 'epoch': epoch + 1, # 'arch': args.arch, From dc77a55b12c5413b9d2e005739740ca3dc34c59a Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sun, 15 May 2022 13:12:30 -0500 Subject: [PATCH 088/113] adding switch_to_prune() and trying kaiming normal init for wt training --- imagenet/imagenet_main.py | 15 +++++++++++++++ imagenet/models.py | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index d8f23568..892e4442 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -603,6 +603,21 @@ def switch_to_wt(model): return model +# switches off gradients for weights and biases and switches it on for scores and flags +def switch_to_prune(model): + print('Switching to pruning by switching off requires_grad for weights and switching it on for scores.') + + for name, params in model.named_parameters(): + # make sure param_name ends with .weight or .bias + if re.match('.*\.scores', name) and re.match('.*\.bias_scores', name): + params.requires_grad = True + else: + # weights, biases, bias_scores, flags and everything else + params.requires_grad = False + + return model + + def round_model(model, round_scheme='naive'): quantize_threshold=0.5 print("Rounding model with scheme: {}".format(round_scheme)) diff --git a/imagenet/models.py b/imagenet/models.py index 6f2e56c8..eba5dd95 100644 --- a/imagenet/models.py +++ b/imagenet/models.py @@ -358,9 +358,9 @@ def _init_conv(self, conv): def get_builder(): conv_type = "SubnetConv" - bn_type = "NonAffineBatchNorm" # TODO: might change this if it doesn't affect weights + bn_type = "AffineBatchNorm" # TODO: might change this if it causes problems later first_layer_type = None # TODO: I think - weight_init = "signed_constant" + weight_init = "kaiming_normal" print("==> Conv Type: {}".format(conv_type)) print("==> BN Type: {}".format(bn_type)) From f1233e1fd5401cc526ac8124aa08865372d6d5af Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sun, 15 May 2022 13:37:32 -0500 Subject: [PATCH 089/113] adding prune(), get_sparsity etc. this might nearly be all that we need for GM if things work nicely with affineBN and bias --- imagenet/imagenet_main.py | 152 +++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 4 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 892e4442..d889bebb 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -90,6 +90,18 @@ type=float, default=0.001, help='regularization coefficient lambda') +parser.add_argument("--iter-period", + type=int, + default=5, + help="period [epochs] for iterative pruning") +parser.add_argument("--invert-sanity-check", + action="store_true", + default=False, + help="Enable this to run the inverted sanity check (for HC)") +parser.add_argument("--prune-rate", + default=0.5, + type=float, + help="Decides fraction of weights TO PRUNE when calling prune()") best_acc1 = 0 @@ -279,6 +291,9 @@ def main_worker(gpu, ngpus_per_node, args): if args.distributed: train_sampler.set_epoch(epoch) + if epoch % (args.iter_period) == 0 and epoch != 0: + prune(model, args) + print_time("Epoch: {} | Starting Train".format(epoch)) start_train = time.time() # train for one epoch @@ -293,19 +308,23 @@ def main_worker(gpu, ngpus_per_node, args): epoch_time = (time.time() - start_train) / 60 print("Epoch: {} | Train + Val Time {}".format(epoch, epoch_time)) + # check sparsity of model + cp_model = round_model(model, args.round) + avg_sparsity = get_model_sparsity(cp_model, threshold=0, args=args) + epoch_list.append(epoch) train_acc_list.append(train_acc1.item()) test_acc_list.append(acc1.item()) val_acc_list.append(acc1.item()) + model_sparsity_list.append(avg_sparsity) scheduler.step() results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, - 'train_acc': train_acc_list}) - # 'regularization_loss': reg_loss_list, - # 'model_sparsity': model_sparsity_list}) + 'train_acc': train_acc_list, + 'model_sparsity': model_sparsity_list}) # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 @@ -629,7 +648,7 @@ def round_model(model, round_scheme='naive'): # cp_model = copy.deepcopy(model) named_params = cp_model.named_parameters() for name, params in named_params: - if ".score" in name: + if re.match('.*\.scores', name): if round_scheme == 'naive': params.data = torch.gt(params.data, torch.ones_like( params.data)*quantize_threshold).int().float() @@ -649,5 +668,130 @@ def round_model(model, round_scheme='naive'): return cp_model + +def prune(model, args, update_thresholds_only=False, update_scores=False): + print("Pruning Model:") + + scores_threshold = bias_scores_threshold = -np.inf + conv_layers, linear_layers = get_layers(args.arch, model) + + # prune the bottom k of scores + num_active_weights = 0 + num_active_biases = 0 + active_scores_list = [] + active_bias_scores_list = [] + + for layer in (conv_layers + linear_layers): + num_active_weights += layer.flag.data.sum().item() + active_scores = (layer.scores.data[layer.flag.data == 1]).clone() + active_scores_list.append(active_scores) + if args.bias: + num_active_biases += layer.bias_flag.data.sum().item() + active_biases = ( + layer.bias_scores.data[layer.bias_flag.data == 1]).clone() + active_bias_scores_list.append(active_biases) + + number_of_weights_to_prune = np.ceil( + args.prune_rate * num_active_weights).astype(int) + number_of_biases_to_prune = np.ceil( + args.prune_rate * num_active_biases).astype(int) + + agg_scores = torch.cat(active_scores_list) + agg_bias_scores = torch.cat( + active_bias_scores_list) if parser_args.bias else torch.tensor([]) + + # if invert_sanity_check, then threshold is based on sorted scores in descending order, and we prune all scores ABOVE it + scores_threshold = torch.sort( + torch.abs(agg_scores), descending=args.invert_sanity_check).values[number_of_weights_to_prune-1].item() + + if args.bias: + bias_scores_threshold = torch.sort( + torch.abs(agg_bias_scores), descending=args.invert_sanity_check).values[number_of_biases_to_prune-1].item() + else: + bias_scores_threshold = -1 + + if update_thresholds_only: + for layer in (conv_layers + linear_layers): + layer.scores_prune_threshold = scores_threshold + if parser_args.bias: + layer.bias_scores_prune_threshold = bias_scores_threshold + + else: + for layer in (conv_layers + linear_layers): + if args.invert_sanity_check: + layer.flag.data = (layer.flag.data + torch.lt(layer.scores.abs(), # TODO + torch.ones_like(layer.scores)*scores_threshold).int() == 2).int() + else: + layer.flag.data = (layer.flag.data + torch.gt(layer.scores.abs(), # TODO + torch.ones_like(layer.scores)*scores_threshold).int() == 2).int() + if update_scores: + layer.scores.data = layer.scores.data * layer.flag.data + if args.bias: + if args.invert_sanity_check: + layer.bias_flag.data = (layer.bias_flag.data + torch.lt(layer.bias_scores, torch.ones_like( + layer.bias_scores)*bias_scores_threshold).int() == 2).int() + else: + layer.bias_flag.data = (layer.bias_flag.data + torch.gt(layer.bias_scores, torch.ones_like( + layer.bias_scores)*bias_scores_threshold).int() == 2).int() + if update_scores: + layer.bias_scores.data = layer.bias_scores.data * layer.bias_flag.data + + + return scores_threshold, bias_scores_threshold + + +# returns num_nonzero elements, total_num_elements so that it is easier to compute +# average sparsity in the end +def get_layer_sparsity(layer, threshold=0, args): + # assume the model is rounded, compute effective scores + eff_scores = layer.scores * layer.flag + if args.bias: + eff_bias_scores = layer.bias_scores * layer.bias_flag + num_middle = torch.sum(torch.gt(eff_scores, + torch.ones_like(eff_scores)*threshold) * + torch.lt(eff_scores, + torch.ones_like(eff_scores.detach()*(1-threshold)).int())) + if num_middle > 0: + print("WARNING: Model scores are not binary. Sparsity number is unreliable.") + raise ValueError + w_numer, w_denom = eff_scores.detach().sum().item(), eff_scores.detach().flatten().numel() + + if args.bias: + b_numer, b_denom = eff_bias_scores.detach().sum().item(), eff_bias_scores.detach().flatten().numel() + else: + b_numer, b_denom = 0, 0 + + return w_numer, w_denom, b_numer, b_denom + + +# returns avg_sparsity = number of non-zero weights! +def get_model_sparsity(model, threshold=0, args): + conv_layers, linear_layers = get_layers(parser_args.arch, model) + numer = 0 + denom = 0 + + # TODO: find a nicer way to do this (skip dropout) + # TODO: Update: can't use .children() or .named_modules() because of the way things are wrapped in builder + for conv_layer in conv_layers: + w_numer, w_denom, b_numer, b_denom = get_layer_sparsity( + conv_layer, threshold, args) + numer += w_numer + denom += w_denom + if parser_args.bias: + numer += b_numer + denom += b_denom + + for lin_layer in linear_layers: + w_numer, w_denom, b_numer, b_denom = get_layer_sparsity( + lin_layer, threshold, args) + numer += w_numer + denom += w_denom + if parser_args.bias: + numer += b_numer + denom += b_denom + # print('Overall sparsity: {}/{} ({:.2f} %)'.format((int)(numer), denom, 100*numer/denom)) + return 100*numer/denom + + if __name__ == '__main__': main() From 5b447a203f4acd6da175f69ad72bc3a4c1310336 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 29 May 2022 13:40:47 -0500 Subject: [PATCH 090/113] adding model strings to make comparing easier --- imagenet/my_model.txt | 176 +++++++++++++++++++++++++++++++++++++ imagenet/pytorch_model.txt | 176 +++++++++++++++++++++++++++++++++++++ 2 files changed, 352 insertions(+) create mode 100644 imagenet/my_model.txt create mode 100644 imagenet/pytorch_model.txt diff --git a/imagenet/my_model.txt b/imagenet/my_model.txt new file mode 100644 index 00000000..b0061d4a --- /dev/null +++ b/imagenet/my_model.txt @@ -0,0 +1,176 @@ +ResNet( + (conv1): SubnetConv(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3)) + (bn1): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False) + (layer1): Sequential( + (0): Bottleneck( + (conv1): SubnetConv(64, 64, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + (downsample): Sequential( + (0): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1)) + (1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + ) + ) + (1): Bottleneck( + (conv1): SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (2): Bottleneck( + (conv1): SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + ) + (layer2): Sequential( + (0): Bottleneck( + (conv1): SubnetConv(256, 128, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (bn2): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + (downsample): Sequential( + (0): SubnetConv(256, 512, kernel_size=(1, 1), stride=(2, 2)) + (1): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + ) + ) + (1): Bottleneck( + (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (2): Bottleneck( + (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (3): Bottleneck( + (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + ) + (layer3): Sequential( + (0): Bottleneck( + (conv1): SubnetConv(512, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + (downsample): Sequential( + (0): SubnetConv(512, 1024, kernel_size=(1, 1), stride=(2, 2)) + (1): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + ) + ) + (1): Bottleneck( + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (2): Bottleneck( + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (3): Bottleneck( + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (4): Bottleneck( + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (5): Bottleneck( + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + ) + (layer4): Sequential( + (0): Bottleneck( + (conv1): SubnetConv(1024, 512, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (bn2): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + (downsample): Sequential( + (0): SubnetConv(1024, 2048, kernel_size=(1, 1), stride=(2, 2)) + (1): NonAffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + ) + ) + (1): Bottleneck( + (conv1): SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (2): Bottleneck( + (conv1): SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1)) + (bn1): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (bn2): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1)) + (bn3): NonAffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + ) + (avgpool): AdaptiveAvgPool2d(output_size=1) + (fc): SubnetConv(2048, 1000, kernel_size=(1, 1), stride=(1, 1)) +) \ No newline at end of file diff --git a/imagenet/pytorch_model.txt b/imagenet/pytorch_model.txt new file mode 100644 index 00000000..b5ad38fc --- /dev/null +++ b/imagenet/pytorch_model.txt @@ -0,0 +1,176 @@ +ResNet( + (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) + (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False) + (layer1): Sequential( + (0): Bottleneck( + (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + (downsample): Sequential( + (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + ) + (1): Bottleneck( + (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (2): Bottleneck( + (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + ) + (layer2): Sequential( + (0): Bottleneck( + (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + (downsample): Sequential( + (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False) + (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + ) + (1): Bottleneck( + (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (2): Bottleneck( + (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (3): Bottleneck( + (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + ) + (layer3): Sequential( + (0): Bottleneck( + (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + (downsample): Sequential( + (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False) + (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + ) + (1): Bottleneck( + (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (2): Bottleneck( + (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (3): Bottleneck( + (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (4): Bottleneck( + (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (5): Bottleneck( + (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + ) + (layer4): Sequential( + (0): Bottleneck( + (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + (downsample): Sequential( + (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False) + (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + ) + (1): Bottleneck( + (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + (2): Bottleneck( + (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (relu): ReLU(inplace=True) + ) + ) + (avgpool): AdaptiveAvgPool2d(output_size=(1, 1)) + (fc): Linear(in_features=2048, out_features=1000, bias=True) +) \ No newline at end of file From 9cf910587a051628da8902020a8c713432b1d563 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 29 May 2022 13:52:29 -0500 Subject: [PATCH 091/113] updating model with bias=False and affine bn --- imagenet/my_model.txt | 214 +++++++++++++++++++++--------------------- 1 file changed, 107 insertions(+), 107 deletions(-) diff --git a/imagenet/my_model.txt b/imagenet/my_model.txt index b0061d4a..4309f752 100644 --- a/imagenet/my_model.txt +++ b/imagenet/my_model.txt @@ -1,176 +1,176 @@ ResNet( - (conv1): SubnetConv(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3)) - (bn1): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) + (bn1): AffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False) (layer1): Sequential( (0): Bottleneck( - (conv1): SubnetConv(64, 64, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) (downsample): Sequential( - (0): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1)) - (1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (0): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (1): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (1): Bottleneck( - (conv1): SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) (2): Bottleneck( - (conv1): SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) ) (layer2): Sequential( (0): Bottleneck( - (conv1): SubnetConv(256, 128, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - (bn2): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) (downsample): Sequential( - (0): SubnetConv(256, 512, kernel_size=(1, 1), stride=(2, 2)) - (1): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (0): SubnetConv(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False) + (1): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (1): Bottleneck( - (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) (2): Bottleneck( - (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) (3): Bottleneck( - (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) ) (layer3): Sequential( (0): Bottleneck( - (conv1): SubnetConv(512, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) (downsample): Sequential( - (0): SubnetConv(512, 1024, kernel_size=(1, 1), stride=(2, 2)) - (1): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (0): SubnetConv(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False) + (1): AffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (1): Bottleneck( - (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) (2): Bottleneck( - (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) (3): Bottleneck( - (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) (4): Bottleneck( - (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) (5): Bottleneck( - (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) ) (layer4): Sequential( (0): Bottleneck( - (conv1): SubnetConv(1024, 512, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - (bn2): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) (downsample): Sequential( - (0): SubnetConv(1024, 2048, kernel_size=(1, 1), stride=(2, 2)) - (1): NonAffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (0): SubnetConv(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False) + (1): AffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (1): Bottleneck( - (conv1): SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) (2): Bottleneck( - (conv1): SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1)) - (bn1): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (bn2): NonAffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) - (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1)) - (bn3): NonAffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True) + (conv1): SubnetConv(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn1): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv2): SubnetConv(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn2): AffineBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv3): SubnetConv(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn3): AffineBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (relu): ReLU(inplace=True) ) ) (avgpool): AdaptiveAvgPool2d(output_size=1) - (fc): SubnetConv(2048, 1000, kernel_size=(1, 1), stride=(1, 1)) + (fc): SubnetConv(2048, 1000, kernel_size=(1, 1), stride=(1, 1), bias=False) ) \ No newline at end of file From c416362e8be43bc7a368b8a2baa8ebb8e4a5ef56 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 29 May 2022 14:12:40 -0500 Subject: [PATCH 092/113] trying to mimic torchvision resnet50 --- imagenet/imagenet_main.py | 10 +++++----- imagenet/models.py | 6 +++--- imagenet/run.sh | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index d889bebb..bd840f8e 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -25,12 +25,12 @@ import torchvision.datasets as datasets import torch.autograd as autograd import torch.nn.functional as F -# import torchvision.models as models +import torchvision.models as torchvision_models import models -model_names = sorted(name for name in models.__dict__ +model_names = sorted(name for name in torchvision_models.__dict__ if name.islower() and not name.startswith("__") - and callable(models.__dict__[name])) + and callable(torchvision_models.__dict__[name])) parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') parser.add_argument('--data', metavar='DIR', default='/home/ubuntu/ILSVRC2012/', @@ -742,7 +742,7 @@ def prune(model, args, update_thresholds_only=False, update_scores=False): # returns num_nonzero elements, total_num_elements so that it is easier to compute # average sparsity in the end -def get_layer_sparsity(layer, threshold=0, args): +def get_layer_sparsity(layer, threshold=0, args=None): # assume the model is rounded, compute effective scores eff_scores = layer.scores * layer.flag if args.bias: @@ -765,7 +765,7 @@ def get_layer_sparsity(layer, threshold=0, args): # returns avg_sparsity = number of non-zero weights! -def get_model_sparsity(model, threshold=0, args): +def get_model_sparsity(model, threshold=0, args=None): conv_layers, linear_layers = get_layers(parser_args.arch, model) numer = 0 denom = 0 diff --git a/imagenet/models.py b/imagenet/models.py index eba5dd95..deb5c93d 100644 --- a/imagenet/models.py +++ b/imagenet/models.py @@ -369,7 +369,7 @@ def get_builder(): # conv_layer = getattr(utils.conv_type, parser_args.conv_type) # bn_layer = getattr(utils.bn_type, parser_args.bn_type) conv_layer = SubnetConv - bn_layer = NonAffineBatchNorm + bn_layer = AffineBatchNorm if first_layer_type is not None: first_layer = getattr(utils.conv_type, first_layer_type) @@ -437,6 +437,8 @@ def __init__(self, *args, **kwargs): self.args_bias = False self.algo = 'hc' self.prune_rate = 0.5 + # resnet50 has bias=False because of BN layers + self.bias = None # initialize flag (representing the pruned weights) self.flag = nn.Parameter(torch.ones(self.weight.size())) @@ -470,8 +472,6 @@ def __init__(self, *args, **kwargs): self.weight.requires_grad = False self.flag.requires_grad = False self.bias_flag.requires_grad = False - # TODO: Hacky. I'm trying to mimick frankle etc in that we have biases, but we don't prune them - self.bias.requires_grad = False def set_prune_rate(self, prune_rate): self.prune_rate = prune_rate diff --git a/imagenet/run.sh b/imagenet/run.sh index 36bc12f2..715cb8c8 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -2,9 +2,9 @@ python imagenet_main.py \ --arch ResNet50 \ - --dist-url 'tcp://127.0.0.1:2500' \ - --dist-backend 'nccl' \ - --multiprocessing-distributed \ + #--dist-url 'tcp://127.0.0.1:2500' \ + #--dist-backend 'nccl' \ + #--multiprocessing-distributed \ --world-size 1 \ --rank 0 \ --batch-size 1024 \ From 51594fcfa18856bbf523aea6ccff2ab91b764beb Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sun, 29 May 2022 14:27:07 -0500 Subject: [PATCH 093/113] changing last layer to linear --- imagenet/models.py | 94 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/imagenet/models.py b/imagenet/models.py index deb5c93d..8d672162 100644 --- a/imagenet/models.py +++ b/imagenet/models.py @@ -124,7 +124,8 @@ def __init__(self, builder, block, layers, num_classes=1000, base_width=64): if self.args_last_layer_dense: self.fc = nn.Conv2d(512 * block.expansion, num_classes, 1) else: - self.fc = builder.conv1x1(512 * block.expansion, num_classes) + # self.fc = builder.conv1x1(512 * block.expansion, num_classes) + self.fc = builder.linear(512 * block.expansion, num_classes) self.prunable_layer_names, self.prunable_biases = self.get_prunable_param_names() @@ -360,7 +361,7 @@ def get_builder(): conv_type = "SubnetConv" bn_type = "AffineBatchNorm" # TODO: might change this if it causes problems later first_layer_type = None # TODO: I think - weight_init = "kaiming_normal" + weight_init = "kaiming_normal" # TODO: this is only for debugging wt training print("==> Conv Type: {}".format(conv_type)) print("==> BN Type: {}".format(bn_type)) @@ -472,6 +473,8 @@ def __init__(self, *args, **kwargs): self.weight.requires_grad = False self.flag.requires_grad = False self.bias_flag.requires_grad = False + # TODO: Hacky. I'm trying to mimic frankle etc in that we have biases, but we don't prune them + self.bias.requires_grad = False def set_prune_rate(self, prune_rate): self.prune_rate = prune_rate @@ -504,13 +507,98 @@ def forward(self, x): b = self.bias * bias_subnet else: b = self.bias - + x = F.conv2d( x, w, b, self.stride, self.padding, self.dilation, self.groups ) return x + +class SubnetLinear(nn.Linear): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # TODO: hacky. trying to mimic frankle in having biases but not pruning them + self.args_bias = False + self.algo = 'hc' + self.prune_rate = 0.5 + # resnet50 has bias=True only for the FC layer + + # initialize flag (representing the pruned weights) + self.flag = nn.Parameter(torch.ones(self.weight.size())) + if self.args_bias: + self.bias_flag = nn.Parameter(torch.ones(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_flag = nn.Parameter(torch.Tensor(1)) + + # initialize the scores + self.scores = nn.Parameter(torch.Tensor(self.weight.size())) + if self.args_bias: + self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_scores = nn.Parameter(torch.Tensor(1)) + + # prune scores below this for global EP in bottom-k + self.scores_prune_threshold = -np.inf + self.bias_scores_prune_threshold = -np.inf + + if self.algo in ['hc', 'hc_iter']: + # score init is always uniform + nn.init.uniform_(self.scores, a=0.0, b=1.0) + nn.init.uniform_(self.bias_scores, a=0.0, b=1.0) + else: + nn.init.kaiming_uniform_(self.scores, a=math.sqrt(5)) + nn.init.uniform_(self.bias_scores, a=-1.0, b=1.0) # can't do kaiming here. picking U[-1, 1] for no real reason + + # NOTE: turn the gradient on the weights off + self.weight.requires_grad = False + self.flag.requires_grad = False + self.bias_flag.requires_grad = False + # TODO: Hacky. I'm trying to mimic frankle etc in that we have biases, but we don't prune them + self.bias.requires_grad = False + + def set_prune_rate(self, prune_rate): + self.prune_rate = prune_rate + + @property + def clamped_scores(self): + return self.scores.abs() + + def forward(self, x): + if parser_args.algo in ['hc', 'hc_iter']: + # don't need a mask here. the scores are directly multiplied with weights + subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, parser_args.prune_rate) + subnet = subnet * self.flag.data.float() + bias_subnet = subnet * self.bias_flag.data.float() + else: + subnet = self.scores * self.flag.data.float() + bias_subnet = self.bias_scores * self.bias_flag.data.float() + elif parser_args.algo in ['imp']: + # no STE, no subnet. Mask is handled outside + pass + elif parser_args.algo in ['global_ep', 'global_ep_iter']: + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) + else: + # ep, global_ep, global_ep_iter, pt etc + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), parser_args.prune_rate) + + if parser_args.algo in ['imp']: + # no STE, no subnet. Mask is handled outside + w = self.weight + b = self.bias + else: + w = self.weight * subnet + if parser_args.bias: + b = self.bias * bias_subnet + else: + b = self.bias + + x = F.linear(x, w, b) + return x + + class NonAffineBatchNorm(nn.BatchNorm2d): def __init__(self, dim): super(NonAffineBatchNorm, self).__init__(dim, affine=False) From 8d52ef8380b2faa7ecd270cf1bae21100cd4b0da Mon Sep 17 00:00:00 2001 From: root Date: Sun, 29 May 2022 14:31:42 -0500 Subject: [PATCH 094/113] minor bugfixes adding bias to final fc layer --- imagenet/models.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/imagenet/models.py b/imagenet/models.py index 8d672162..fe630f61 100644 --- a/imagenet/models.py +++ b/imagenet/models.py @@ -473,8 +473,6 @@ def __init__(self, *args, **kwargs): self.weight.requires_grad = False self.flag.requires_grad = False self.bias_flag.requires_grad = False - # TODO: Hacky. I'm trying to mimic frankle etc in that we have biases, but we don't prune them - self.bias.requires_grad = False def set_prune_rate(self, prune_rate): self.prune_rate = prune_rate @@ -572,9 +570,6 @@ def forward(self, x): subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, parser_args.prune_rate) subnet = subnet * self.flag.data.float() bias_subnet = subnet * self.bias_flag.data.float() - else: - subnet = self.scores * self.flag.data.float() - bias_subnet = self.bias_scores * self.bias_flag.data.float() elif parser_args.algo in ['imp']: # no STE, no subnet. Mask is handled outside pass From 63f38ceed43cb35c84003ad952b40a1759846bbe Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sun, 29 May 2022 14:37:55 -0500 Subject: [PATCH 095/113] plugging in subnetconv model into original script to compare --- imagenet/imagenet_main_bkp.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py index c045b872..9521b397 100644 --- a/imagenet/imagenet_main_bkp.py +++ b/imagenet/imagenet_main_bkp.py @@ -19,6 +19,7 @@ import torchvision.transforms as transforms import torchvision.datasets as datasets import torchvision.models as models +import models as mymodels model_names = sorted(name for name in models.__dict__ if name.islower() and not name.startswith("__") @@ -140,7 +141,10 @@ def main_worker(gpu, ngpus_per_node, args): model = models.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) - model = models.__dict__[args.arch]() + # model = models.__dict__[args.arch]() + # TODO: trying to compare vision model with my model + model = mymodels.ResNet50() + model = switch_to_wt(model) if not torch.cuda.is_available(): print('using CPU, this will be slow') @@ -485,6 +489,24 @@ def print_time(msg): print("----------------------------------------------------------------------------\n\n") +# switches off gradients for scores and flags and switches it on for weights and biases +def switch_to_wt(model): + print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') + + for name, params in model.named_parameters(): + # make sure param_name ends with .weight or .bias + if re.match('.*\.weight', name): + params.requires_grad = True + elif re.match('.*\.bias$', name): + params.requires_grad = True + elif "score" in name: + params.requires_grad = False + else: + # flags and everything else + params.requires_grad = False + + return model + if __name__ == '__main__': main() From 0b6cfdc12ac68e872fb5437c84e100f5bcd24133 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 30 May 2022 15:45:48 -0500 Subject: [PATCH 096/113] minor bugfixes and adding weight init to builder --- imagenet/imagenet_main.py | 1 + imagenet/imagenet_main_bkp.py | 7 ++++--- imagenet/models.py | 15 ++++++++------- imagenet/run.sh | 21 ++++++++++++++++++--- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index bd840f8e..f38be938 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -157,6 +157,7 @@ def main_worker(gpu, ngpus_per_node, args): args.rank = args.rank * ngpus_per_node + gpu dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) + # create model if args.pretrained: print("==> using pre-trained model '{}'".format(args.arch)) diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py index 9521b397..daea184c 100644 --- a/imagenet/imagenet_main_bkp.py +++ b/imagenet/imagenet_main_bkp.py @@ -5,6 +5,7 @@ import time import warnings from enum import Enum +import re import torch import torch.nn as nn @@ -141,10 +142,10 @@ def main_worker(gpu, ngpus_per_node, args): model = models.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) - # model = models.__dict__[args.arch]() + model = models.__dict__[args.arch]() # TODO: trying to compare vision model with my model - model = mymodels.ResNet50() - model = switch_to_wt(model) + #model = mymodels.ResNet50() + #model = switch_to_wt(model) if not torch.cuda.is_available(): print('using CPU, this will be slow') diff --git a/imagenet/models.py b/imagenet/models.py index fe630f61..8aa95162 100644 --- a/imagenet/models.py +++ b/imagenet/models.py @@ -180,6 +180,7 @@ def forward(self, x): x = self.layer4(x) x = self.avgpool(x) + x = torch.flatten(x, 1) x = self.fc(x) x = x.view(x.size(0), -1) @@ -378,7 +379,7 @@ def get_builder(): else: first_layer = None - builder = Builder(conv_layer=conv_layer, bn_layer=bn_layer, first_layer=first_layer) + builder = Builder(conv_layer=conv_layer, bn_layer=bn_layer, first_layer=first_layer, weight_init=weight_init) return builder @@ -565,27 +566,27 @@ def clamped_scores(self): return self.scores.abs() def forward(self, x): - if parser_args.algo in ['hc', 'hc_iter']: + if self.algo in ['hc', 'hc_iter']: # don't need a mask here. the scores are directly multiplied with weights - subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, parser_args.prune_rate) + subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, self.prune_rate) subnet = subnet * self.flag.data.float() bias_subnet = subnet * self.bias_flag.data.float() - elif parser_args.algo in ['imp']: + elif self.algo in ['imp']: # no STE, no subnet. Mask is handled outside pass elif parser_args.algo in ['global_ep', 'global_ep_iter']: subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) else: # ep, global_ep, global_ep_iter, pt etc - subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), parser_args.prune_rate) + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), self.prune_rate) - if parser_args.algo in ['imp']: + if self.algo in ['imp']: # no STE, no subnet. Mask is handled outside w = self.weight b = self.bias else: w = self.weight * subnet - if parser_args.bias: + if self.args_bias: b = self.bias * bias_subnet else: b = self.bias diff --git a/imagenet/run.sh b/imagenet/run.sh index 715cb8c8..5f2d1a24 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -1,10 +1,11 @@ #!/bin/bash +:< "resnet50_imagenet_log" 2>&1 & +BLOCK + +python imagenet_main_bkp.py \ + --arch resnet50 \ + #--dist-url 'tcp://127.0.0.1:2500' \ + #--dist-backend 'nccl' \ + #--multiprocessing-distributed \ + #--world-size 1 \ + #--rank 0 \ + --batch-size 256 \ + --workers 12 \ + --epochs 5 \ + --lr 0.4 \ + --data '/data/imagenet/' From d2639600d5948a861f0bee0cfe35e61a68f0f71d Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 30 May 2022 15:47:26 -0500 Subject: [PATCH 097/113] adding round_model to complete comparison between mymodel and torchvision model --- imagenet/imagenet_main_bkp.py | 39 ++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py index daea184c..5a69fecb 100644 --- a/imagenet/imagenet_main_bkp.py +++ b/imagenet/imagenet_main_bkp.py @@ -142,10 +142,11 @@ def main_worker(gpu, ngpus_per_node, args): model = models.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) - model = models.__dict__[args.arch]() + # model = models.__dict__[args.arch]() # TODO: trying to compare vision model with my model - #model = mymodels.ResNet50() - #model = switch_to_wt(model) + model = mymodels.ResNet50() + model = round_model(model, round_scheme='all_ones') + model = switch_to_wt(model) if not torch.cuda.is_available(): print('using CPU, this will be slow') @@ -509,5 +510,37 @@ def switch_to_wt(model): return model +def round_model(model, round_scheme='naive'): + quantize_threshold=0.5 + print("Rounding model with scheme: {}".format(round_scheme)) + cp_model = copy.deepcopy(model) + if isinstance(model, nn.parallel.DistributedDataParallel): + # cp_model = copy.deepcopy(model.module) + named_params = cp_model.module.named_parameters() + else: + # cp_model = copy.deepcopy(model) + named_params = cp_model.named_parameters() + for name, params in named_params: + if re.match('.*\.scores', name): + if round_scheme == 'naive': + params.data = torch.gt(params.data, torch.ones_like( + params.data)*quantize_threshold).int().float() + elif round_scheme == 'prob': + params.data = torch.clamp(params.data, 0.0, 1.0) + params.data = torch.bernoulli(params.data).float() + elif round_scheme == 'all_ones': + params.data = torch.ones_like(params.data) + else: + print("INVALID ROUNDING") + print("EXITING") + exit() + + # if isinstance(model, nn.parallel.DistributedDataParallel): + # cp_model = nn.parallel.DistributedDataParallel( + # cp_model, device_ids=[parser_args.gpu], find_unused_parameters=False) + + return cp_model + + if __name__ == '__main__': main() From 19d7044869a866be31c4c77aaee6f1e29423111a Mon Sep 17 00:00:00 2001 From: root Date: Tue, 31 May 2022 11:07:12 -0500 Subject: [PATCH 098/113] it works! looks like the models are near identical now. --- imagenet/imagenet_main_bkp.py | 9 +++++---- imagenet/run.sh | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/imagenet/imagenet_main_bkp.py b/imagenet/imagenet_main_bkp.py index 5a69fecb..3032d480 100644 --- a/imagenet/imagenet_main_bkp.py +++ b/imagenet/imagenet_main_bkp.py @@ -6,6 +6,7 @@ import warnings from enum import Enum import re +import copy import torch import torch.nn as nn @@ -142,11 +143,11 @@ def main_worker(gpu, ngpus_per_node, args): model = models.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) - # model = models.__dict__[args.arch]() + model = models.__dict__[args.arch]() # TODO: trying to compare vision model with my model - model = mymodels.ResNet50() - model = round_model(model, round_scheme='all_ones') - model = switch_to_wt(model) + # model = mymodels.ResNet50() + # model = round_model(model, round_scheme='all_ones') + # model = switch_to_wt(model) if not torch.cuda.is_available(): print('using CPU, this will be slow') diff --git a/imagenet/run.sh b/imagenet/run.sh index 5f2d1a24..0d316203 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -19,11 +19,11 @@ BLOCK python imagenet_main_bkp.py \ --arch resnet50 \ - #--dist-url 'tcp://127.0.0.1:2500' \ - #--dist-backend 'nccl' \ - #--multiprocessing-distributed \ - #--world-size 1 \ - #--rank 0 \ + --dist-url 'tcp://127.0.0.1:2500' \ + --dist-backend 'nccl' \ + --multiprocessing-distributed \ + --world-size 1 \ + --rank 0 \ --batch-size 256 \ --workers 12 \ --epochs 5 \ From 3a7ad5e1fb4d48d2ff92bc29fa83adaa182fdc79 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 1 Jun 2022 10:09:47 -0500 Subject: [PATCH 099/113] changing model back to signed_constant for GM --- imagenet/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagenet/models.py b/imagenet/models.py index 8aa95162..7b8a811e 100644 --- a/imagenet/models.py +++ b/imagenet/models.py @@ -362,7 +362,7 @@ def get_builder(): conv_type = "SubnetConv" bn_type = "AffineBatchNorm" # TODO: might change this if it causes problems later first_layer_type = None # TODO: I think - weight_init = "kaiming_normal" # TODO: this is only for debugging wt training + weight_init = "signed_constant" print("==> Conv Type: {}".format(conv_type)) print("==> BN Type: {}".format(bn_type)) From cfd62cb85b7189f80cadbd2790c713f9b937135d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 1 Jun 2022 16:17:13 +0000 Subject: [PATCH 100/113] turns out you need to switch_to_wt before instantiating ddp otherwise ddp complains about unused parameters. weird. --- imagenet/imagenet_main.py | 11 ++++++----- imagenet/run.sh | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index f38be938..746df1b7 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -167,6 +167,11 @@ def main_worker(gpu, ngpus_per_node, args): print("==> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() model = models.ResNet50() + print("GPU: {} | Switching to wt to see if that works well!".format(args.gpu)) + print("GPU: {} | First round model to all ones score".format(args.gpu)) + model = round_model(model, round_scheme='all_ones') + model = switch_to_wt(model) + if not torch.cuda.is_available(): print('using CPU, this will be slow') @@ -202,10 +207,6 @@ def main_worker(gpu, ngpus_per_node, args): # define loss function (criterion), optimizer, and learning rate scheduler criterion = nn.CrossEntropyLoss().cuda(args.gpu) - print("GPU: {} | Switching to wt to see if that works well!".format(args.gpu)) - print("GPU: {} | First round model to all ones score".format(args.gpu)) - model = round_model(model, round_scheme='all_ones') - switch_to_wt(model) optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) @@ -388,7 +389,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): loss = criterion(output, target) regularization_loss = torch.tensor(0) - #regularization_loss = get_regularization_loss(model, args) + # regularization_loss = get_regularization_loss(model, args) # print("Regularization loss: {}".format(regularization_loss.item())) loss += regularization_loss diff --git a/imagenet/run.sh b/imagenet/run.sh index 0d316203..fa2a0d09 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -:< "resnet50_imagenet_log" 2>&1 & -BLOCK +#BLOCK +:< Date: Wed, 1 Jun 2022 12:21:51 -0500 Subject: [PATCH 101/113] adding finetune feature --- imagenet/imagenet_main.py | 69 +++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 746df1b7..06c493db 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -102,6 +102,19 @@ default=0.5, type=float, help="Decides fraction of weights TO PRUNE when calling prune()") +parser.add_argument("--target-sparsity", + default=5, + type=float, + help="Decides target sparsity in % when running GM") +parser.add_argument("--finetune", + action="store_true", + default=False, + help="Enable this to run the FT step after finding the subnetwork") +parser.add_argument('--checkpoint', + default='', + type=str, + metavar='PATH', + help='path to latest checkpoint (default: none)') best_acc1 = 0 @@ -167,12 +180,22 @@ def main_worker(gpu, ngpus_per_node, args): print("==> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() model = models.ResNet50() - print("GPU: {} | Switching to wt to see if that works well!".format(args.gpu)) - print("GPU: {} | First round model to all ones score".format(args.gpu)) - model = round_model(model, round_scheme='all_ones') + # print("GPU: {} | Switching to wt to see if that works well!".format(args.gpu)) + # print("GPU: {} | First round model to all ones score".format(args.gpu)) + # model = round_model(model, round_scheme='all_ones') + # model = switch_to_wt(model) + + if args.finetune: + print("Finetuning Rare Gem. Loading checkpoint") + ckpt = torch.load(args.checkpoint) + model.load_state_dict(ckpt) + print("Successfully loaded checkpoint: {}".format(ckpt)) + model = round_model(model, round_scheme='naive') model = switch_to_wt(model) + args.prune_rate = get_prune_rate(args.target_sparsity, args.iter_period) + if not torch.cuda.is_available(): print('using CPU, this will be slow') elif args.distributed: @@ -293,7 +316,7 @@ def main_worker(gpu, ngpus_per_node, args): if args.distributed: train_sampler.set_epoch(epoch) - if epoch % (args.iter_period) == 0 and epoch != 0: + if not args.finetune and epoch % (args.iter_period) == 0 and epoch != 0: prune(model, args) print_time("Epoch: {} | Starting Train".format(epoch)) @@ -335,7 +358,7 @@ def main_worker(gpu, ngpus_per_node, args): if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0): results_df.to_csv("acc_and_sparsity.csv", index=False) - save_flag = ((epoch+1)%10 == 0) or (epoch > 85) + save_flag = ((epoch+1)%10 == 0) or (epoch > 85) or (epoch == args.epochs-1) if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0)): torch.save(model.module.state_dict(), 'model_before_fineune_epoch_{}.pth'.format(epoch)) # save_checkpoint({ @@ -372,12 +395,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): if torch.cuda.is_available(): target = target.cuda(args.gpu, non_blocking=True) - # project to [0, 1] in every gradient step - for name, params in model.named_parameters(): - # make sure param_name ends with .scores and not bias_scores - if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): - with torch.no_grad(): - params.data = torch.clamp(params.data, 0.0, 1.0) + if not args.finetune: + # project to [0, 1] in every gradient step + for name, params in model.named_parameters(): + # make sure param_name ends with .scores and not bias_scores + if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): + with torch.no_grad(): + params.data = torch.clamp(params.data, 0.0, 1.0) # compute output if scaler is None: @@ -416,12 +440,13 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): if i % args.print_freq == 0: progress.display(i) - # project to [0, 1] before returning model - for name, params in model.named_parameters(): - # make sure param_name ends with .scores and not bias_scores - if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): - with torch.no_grad(): - params.data = torch.clamp(params.data, 0.0, 1.0) + if not args.finetune: + # project to [0, 1] before returning model + for name, params in model.named_parameters(): + # make sure param_name ends with .scores and not bias_scores + if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): + with torch.no_grad(): + params.data = torch.clamp(params.data, 0.0, 1.0) return top1.avg @@ -795,5 +820,15 @@ def get_model_sparsity(model, threshold=0, args=None): return 100*numer/denom +def get_prune_rate(target_sparsity=0.5, iter_period=5): + print("Computing prune_rate for target_sparsity {} with iter_period {}".format( + target_sparsity, iter_period)) + max_epochs = parser_args.epochs + num_prune_iterations = np.floor((max_epochs-1)/iter_period) + # if algo is HC, iter_HC or anything that uses prune() then, prune_rate represents number of weights to prune + prune_rate = 1 - np.exp(np.log(target_sparsity/100)/num_prune_iterations) + return prune_rate + + if __name__ == '__main__': main() From 188735d8d516ca5a8e61aa67adf0e1f15040d70f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 1 Jun 2022 21:51:09 +0000 Subject: [PATCH 102/113] finetuning seems to work well! --- imagenet/imagenet_main.py | 39 ++++++++++++++++++++++----------------- imagenet/run.sh | 10 ++++++---- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 06c493db..b344e07d 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -115,6 +115,11 @@ type=str, metavar='PATH', help='path to latest checkpoint (default: none)') +parser.add_argument("--bias", + action="store_true", + default=False, + help="Enable this to allow pruning biases") + best_acc1 = 0 @@ -187,14 +192,14 @@ def main_worker(gpu, ngpus_per_node, args): if args.finetune: print("Finetuning Rare Gem. Loading checkpoint") - ckpt = torch.load(args.checkpoint) - model.load_state_dict(ckpt) - print("Successfully loaded checkpoint: {}".format(ckpt)) - model = round_model(model, round_scheme='naive') + #ckpt = torch.load(args.checkpoint) + #model.load_state_dict(ckpt) + print("Successfully loaded checkpoint: {}".format(args.checkpoint)) + model = round_model(model, round_scheme='all_ones') model = switch_to_wt(model) - args.prune_rate = get_prune_rate(args.target_sparsity, args.iter_period) + args.prune_rate = get_prune_rate(args.target_sparsity, args.iter_period, args.epochs) if not torch.cuda.is_available(): print('using CPU, this will be slow') @@ -318,6 +323,7 @@ def main_worker(gpu, ngpus_per_node, args): if not args.finetune and epoch % (args.iter_period) == 0 and epoch != 0: prune(model, args) + torch.distributed.barrier() print_time("Epoch: {} | Starting Train".format(epoch)) start_train = time.time() @@ -334,7 +340,7 @@ def main_worker(gpu, ngpus_per_node, args): print("Epoch: {} | Train + Val Time {}".format(epoch, epoch_time)) # check sparsity of model - cp_model = round_model(model, args.round) + cp_model = round_model(model, 'naive') avg_sparsity = get_model_sparsity(cp_model, threshold=0, args=args) epoch_list.append(epoch) @@ -412,10 +418,10 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): output = model(images) loss = criterion(output, target) - regularization_loss = torch.tensor(0) - # regularization_loss = get_regularization_loss(model, args) - # print("Regularization loss: {}".format(regularization_loss.item())) - loss += regularization_loss + if not args.finetune: + regularization_loss = get_regularization_loss(model, args) + # print("Regularization loss: {}".format(regularization_loss.item())) + loss += regularization_loss # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 5)) @@ -725,7 +731,7 @@ def prune(model, args, update_thresholds_only=False, update_scores=False): agg_scores = torch.cat(active_scores_list) agg_bias_scores = torch.cat( - active_bias_scores_list) if parser_args.bias else torch.tensor([]) + active_bias_scores_list) if args.bias else torch.tensor([]) # if invert_sanity_check, then threshold is based on sorted scores in descending order, and we prune all scores ABOVE it scores_threshold = torch.sort( @@ -740,7 +746,7 @@ def prune(model, args, update_thresholds_only=False, update_scores=False): if update_thresholds_only: for layer in (conv_layers + linear_layers): layer.scores_prune_threshold = scores_threshold - if parser_args.bias: + if args.bias: layer.bias_scores_prune_threshold = bias_scores_threshold else: @@ -793,7 +799,7 @@ def get_layer_sparsity(layer, threshold=0, args=None): # returns avg_sparsity = number of non-zero weights! def get_model_sparsity(model, threshold=0, args=None): - conv_layers, linear_layers = get_layers(parser_args.arch, model) + conv_layers, linear_layers = get_layers(args.arch, model) numer = 0 denom = 0 @@ -804,7 +810,7 @@ def get_model_sparsity(model, threshold=0, args=None): conv_layer, threshold, args) numer += w_numer denom += w_denom - if parser_args.bias: + if args.bias: numer += b_numer denom += b_denom @@ -813,17 +819,16 @@ def get_model_sparsity(model, threshold=0, args=None): lin_layer, threshold, args) numer += w_numer denom += w_denom - if parser_args.bias: + if args.bias: numer += b_numer denom += b_denom # print('Overall sparsity: {}/{} ({:.2f} %)'.format((int)(numer), denom, 100*numer/denom)) return 100*numer/denom -def get_prune_rate(target_sparsity=0.5, iter_period=5): +def get_prune_rate(target_sparsity=0.5, iter_period=5, max_epochs=88): print("Computing prune_rate for target_sparsity {} with iter_period {}".format( target_sparsity, iter_period)) - max_epochs = parser_args.epochs num_prune_iterations = np.floor((max_epochs-1)/iter_period) # if algo is HC, iter_HC or anything that uses prune() then, prune_rate represents number of weights to prune prune_rate = 1 - np.exp(np.log(target_sparsity/100)/num_prune_iterations) diff --git a/imagenet/run.sh b/imagenet/run.sh index fa2a0d09..4b618329 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -9,10 +9,11 @@ python imagenet_main.py \ --world-size 1 \ --rank 0 \ --batch-size 1024 \ - --workers 12 \ + --workers 8 \ --mixed-precision \ - --epochs 5 \ + --epochs 3 \ --lr 0.4 \ + --finetune \ --lmbda 0.000000 \ --data '/home/ubuntu/ILSVRC2012' #> "resnet50_imagenet_log" 2>&1 & #BLOCK @@ -25,8 +26,9 @@ python imagenet_main_bkp.py \ --multiprocessing-distributed \ --world-size 1 \ --rank 0 \ - --batch-size 256 \ - --workers 12 \ + --batch-size 1024 \ + --workers 8 \ + --mixed-precision \ --epochs 5 \ --lr 0.4 \ --data '/home/ubuntu/ILSVRC2012' From ccf2867cc5f38bef644b128c8241b541e5c2acb1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Jun 2022 20:51:32 +0000 Subject: [PATCH 103/113] adding save_model before finetune and bugfixes --- imagenet/imagenet_main.py | 28 +++++++++++++++------------- imagenet/run.sh | 26 ++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index b344e07d..60eaae59 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -185,20 +185,16 @@ def main_worker(gpu, ngpus_per_node, args): print("==> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() model = models.ResNet50() - # print("GPU: {} | Switching to wt to see if that works well!".format(args.gpu)) - # print("GPU: {} | First round model to all ones score".format(args.gpu)) - # model = round_model(model, round_scheme='all_ones') - # model = switch_to_wt(model) - + if not args.finetune: + model = switch_to_prune(model) if args.finetune: print("Finetuning Rare Gem. Loading checkpoint") - #ckpt = torch.load(args.checkpoint) - #model.load_state_dict(ckpt) + ckpt = torch.load(args.checkpoint, map_location='cuda:{}'.format(args.gpu)) + model.load_state_dict(ckpt) print("Successfully loaded checkpoint: {}".format(args.checkpoint)) - model = round_model(model, round_scheme='all_ones') + model = round_model(model, round_scheme='naive') model = switch_to_wt(model) - args.prune_rate = get_prune_rate(args.target_sparsity, args.iter_period, args.epochs) if not torch.cuda.is_available(): @@ -361,12 +357,17 @@ def main_worker(gpu, ngpus_per_node, args): is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) + if not args.finetune: + results_filename = "acc_and_sparsity.csv" + else: + results_filename = "acc_and_sparsity_finetune.csv" + if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0): - results_df.to_csv("acc_and_sparsity.csv", index=False) + results_df.to_csv(results_filename, index=False) save_flag = ((epoch+1)%10 == 0) or (epoch > 85) or (epoch == args.epochs-1) if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0)): - torch.save(model.module.state_dict(), 'model_before_fineune_epoch_{}.pth'.format(epoch)) + torch.save(model.module.state_dict(), 'model_before_finetune_epoch_{}.pth'.format(epoch)) # save_checkpoint({ # 'epoch': epoch + 1, # 'arch': args.arch, @@ -444,6 +445,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): end = time.time() if i % args.print_freq == 0: + print("Regularization Loss={} | Total Loss={}".format(regularization_loss, loss)) progress.display(i) if not args.finetune: @@ -660,8 +662,8 @@ def switch_to_prune(model): print('Switching to pruning by switching off requires_grad for weights and switching it on for scores.') for name, params in model.named_parameters(): - # make sure param_name ends with .weight or .bias - if re.match('.*\.scores', name) and re.match('.*\.bias_scores', name): + # make sure param_name ends with .scores and not .bias_scores + if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): params.requires_grad = True else: # weights, biases, bias_scores, flags and everything else diff --git a/imagenet/run.sh b/imagenet/run.sh index 4b618329..e1948d48 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -1,6 +1,23 @@ #!/bin/bash #:< "resnet50_imagenet_log" 2>&1 & + --checkpoint 'model_before_finetune_epoch_87.pth' \ + --lmbda 0.0 \ + --data '/home/ubuntu/ILSVRC2012' #BLOCK :< Date: Fri, 3 Jun 2022 15:59:46 -0500 Subject: [PATCH 104/113] trying cifar10 on ddp to debug --- imagenet/imagenet_main_cifar.py | 892 ++++++++++++++++++++++++++++++++ imagenet/models_cifar.py | 694 +++++++++++++++++++++++++ 2 files changed, 1586 insertions(+) create mode 100644 imagenet/imagenet_main_cifar.py create mode 100644 imagenet/models_cifar.py diff --git a/imagenet/imagenet_main_cifar.py b/imagenet/imagenet_main_cifar.py new file mode 100644 index 00000000..c5ea22dd --- /dev/null +++ b/imagenet/imagenet_main_cifar.py @@ -0,0 +1,892 @@ +import argparse +import os +import random +import shutil +import time +import warnings +from enum import Enum +import math +import numpy as np +import pandas as pd +import re +import copy + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +from torch.optim.lr_scheduler import StepLR +import torch.multiprocessing as mp +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torch.autograd as autograd +import torch.nn.functional as F +import torchvision.models as torchvision_models +import models + +model_names = sorted(name for name in torchvision_models.__dict__ + if name.islower() and not name.startswith("__") + and callable(torchvision_models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('--data', metavar='DIR', default='/home/ubuntu/ILSVRC2012/', + help='path to dataset (default: imagenet)') +parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', + help='mini-batch size (default: 256), this is the total ' + 'batch size of all GPUs on the current node when ' + 'using Data Parallel or Distributed Data Parallel') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='initial learning rate', dest='lr') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)', + dest='weight_decay') +parser.add_argument('-p', '--print-freq', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--world-size', default=-1, type=int, + help='number of nodes for distributed training') +parser.add_argument('--rank', default=-1, type=int, + help='node rank for distributed training') +parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, + help='url used to set up distributed training') +parser.add_argument('--dist-backend', default='nccl', type=str, + help='distributed backend') +parser.add_argument('--seed', default=None, type=int, + help='seed for initializing training. ') +parser.add_argument('--gpu', default=None, type=int, + help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +parser.add_argument("--mixed-precision", + action='store_true', + default=False, + help="Use mixed precision or not") +parser.add_argument('--lmbda', + type=float, + default=0.001, + help='regularization coefficient lambda') +parser.add_argument("--iter-period", + type=int, + default=5, + help="period [epochs] for iterative pruning") +parser.add_argument("--invert-sanity-check", + action="store_true", + default=False, + help="Enable this to run the inverted sanity check (for HC)") +parser.add_argument("--prune-rate", + default=0.5, + type=float, + help="Decides fraction of weights TO PRUNE when calling prune()") +parser.add_argument("--target-sparsity", + default=5, + type=float, + help="Decides target sparsity in % when running GM") +parser.add_argument("--finetune", + action="store_true", + default=False, + help="Enable this to run the FT step after finding the subnetwork") +parser.add_argument('--checkpoint', + default='', + type=str, + metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument("--bias", + action="store_true", + default=False, + help="Enable this to allow pruning biases") + + +best_acc1 = 0 + + +def main(): + args = parser.parse_args() + + if args.seed is not None: + random.seed(args.seed) + torch.manual_seed(args.seed) + cudnn.deterministic = True + warnings.warn('You have chosen to seed training. ' + 'This will turn on the CUDNN deterministic setting, ' + 'which can slow down your training considerably! ' + 'You may see unexpected behavior when restarting ' + 'from checkpoints.') + + if args.gpu is not None: + warnings.warn('You have chosen a specific GPU. This will completely ' + 'disable data parallelism.') + + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + + +def main_worker(gpu, ngpus_per_node, args): + global best_acc1 + args.gpu = gpu + + if args.gpu is not None: + print("Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + # create model + if args.pretrained: + print("==> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + # args.arch = 'resnet50' + print("==> creating model '{}'".format(args.arch)) + # model = models.__dict__[args.arch]() + model = models.ResNet50() + if not args.finetune: + model = switch_to_prune(model) + if args.finetune: + print("Finetuning Rare Gem. Loading checkpoint") + ckpt = torch.load(args.checkpoint, map_location='cuda:{}'.format(args.gpu)) + model.load_state_dict(ckpt) + print("Successfully loaded checkpoint: {}".format(args.checkpoint)) + model = round_model(model, round_scheme='naive') + model = switch_to_wt(model) + + args.prune_rate = get_prune_rate(args.target_sparsity, args.iter_period, args.epochs) + + if not torch.cuda.is_available(): + print('using CPU, this will be slow') + elif args.distributed: + # For multiprocessing distributed, DistributedDataParallel constructor + # should always set the single device scope, otherwise, + # DistributedDataParallel will use all available devices. + if args.gpu is not None: + torch.cuda.set_device(args.gpu) + model.cuda(args.gpu) + # When using a single GPU per process and per + # DistributedDataParallel, we need to divide the batch size + # ourselves based on the total number of GPUs of the current node. + args.batch_size = int(args.batch_size / ngpus_per_node) + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + else: + model.cuda() + # DistributedDataParallel will divide and allocate batch_size to all + # available GPUs if device_ids are not set + model = torch.nn.parallel.DistributedDataParallel(model) + elif args.gpu is not None: + torch.cuda.set_device(args.gpu) + model = model.cuda(args.gpu) + else: + # DataParallel will divide and allocate batch_size to all available GPUs + if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): + model.features = torch.nn.DataParallel(model.features) + model.cuda() + else: + model = torch.nn.DataParallel(model).cuda() + + # define loss function (criterion), optimizer, and learning rate scheduler + criterion = nn.CrossEntropyLoss().cuda(args.gpu) + + optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" + scheduler = StepLR(optimizer, step_size=30, gamma=0.1) + + # adding a scaler for mixed-precision training + if args.mixed_precision: + scaler = torch.cuda.amp.GradScaler(enabled=True) # mixed precision + else: + scaler = None + + # optionally resume from a checkpoint + if args.resume: + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + if args.gpu is None: + checkpoint = torch.load(args.resume) + else: + # Map model to be loaded to specified single gpu. + loc = 'cuda:{}'.format(args.gpu) + checkpoint = torch.load(args.resume, map_location=loc) + args.start_epoch = checkpoint['epoch'] + best_acc1 = checkpoint['best_acc1'] + if args.gpu is not None: + # best_acc1 may be from a checkpoint from a different GPU + best_acc1 = best_acc1.to(args.gpu) + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + scheduler.load_state_dict(checkpoint['scheduler']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + + # Data loading code + # traindir = os.path.join(args.data, 'train') + # valdir = os.path.join(args.data, 'val') + # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + # std=[0.229, 0.224, 0.225]) + + # train_dataset = datasets.ImageFolder( + # traindir, + # transforms.Compose([ + # transforms.RandomResizedCrop(224), + # transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), + # normalize, + # ])) + + # if args.distributed: + # train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + # else: + # train_sampler = None + + # train_loader = torch.utils.data.DataLoader( + # train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + # num_workers=args.workers, pin_memory=True, sampler=train_sampler) + + # val_loader = torch.utils.data.DataLoader( + # datasets.ImageFolder(valdir, transforms.Compose([ + # transforms.Resize(256), + # transforms.CenterCrop(224), + # transforms.ToTensor(), + # normalize, + # ])), + # batch_size=args.batch_size, shuffle=False, + # num_workers=args.workers, pin_memory=True) + + data_root = os.path.join(".", "cifar10") + + use_cuda = torch.cuda.is_available() + + # Data loading code + kwargs = {"num_workers": args.num_workers, "pin_memory": True} if use_cuda else {} + + normalize = transforms.Normalize( + mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262] + ) + + train_dataset = torchvision.datasets.CIFAR10( + root=data_root, + train=True, + download=True, + transform=transforms.Compose( + [ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ] + ), + ) + + test_dataset = torchvision.datasets.CIFAR10( + root=data_root, + train=False, + download=True, + transform=transforms.Compose([transforms.ToTensor(), normalize]), + ) + + if parser_args.multiprocessing_distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + + train_loader = torch.utils.data.DataLoader( + train_dataset, + batch_size=args.batch_size, + shuffle=(train_sampler is None), + sampler=train_sampler, + **kwargs + ) + + val_loader = torch.utils.data.DataLoader( + test_dataset, + batch_size=args.batch_size, + shuffle=False, **kwargs + ) + + if args.evaluate: + validate(val_loader, model, criterion, args) + return + + epoch_list = [] + test_acc_list = [] + model_sparsity_list = [] + val_acc_list = [] + train_acc_list = [] + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + + if not args.finetune and epoch % (args.iter_period) == 0 and epoch != 0: + prune(model, args) + torch.distributed.barrier() + + print_time("Epoch: {} | Starting Train".format(epoch)) + start_train = time.time() + # train for one epoch + train_acc1 = train(train_loader, model, criterion, optimizer, epoch, args, scaler) + + train_time = train_time = (time.time() - start_train) / 60 + print("Epoch: {} | Train Time {}".format(epoch, train_time)) + + # evaluate on validation set + acc1 = validate(val_loader, model, criterion, args) + + epoch_time = (time.time() - start_train) / 60 + print("Epoch: {} | Train + Val Time {}".format(epoch, epoch_time)) + + # check sparsity of model + cp_model = round_model(model, 'naive') + avg_sparsity = get_model_sparsity(cp_model, threshold=0, args=args) + + epoch_list.append(epoch) + train_acc_list.append(train_acc1.item()) + test_acc_list.append(acc1.item()) + val_acc_list.append(acc1.item()) + model_sparsity_list.append(avg_sparsity) + + scheduler.step() + + results_df = pd.DataFrame({'epoch': epoch_list, + 'test_acc': test_acc_list, + 'val_acc': val_acc_list, + 'train_acc': train_acc_list, + 'model_sparsity': model_sparsity_list}) + + # remember best acc@1 and save checkpoint + is_best = acc1 > best_acc1 + best_acc1 = max(acc1, best_acc1) + + if not args.finetune: + results_filename = "acc_and_sparsity.csv" + else: + results_filename = "acc_and_sparsity_finetune.csv" + + if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0): + results_df.to_csv(results_filename, index=False) + + save_flag = ((epoch+1)%10 == 0) or (epoch > 85) or (epoch == args.epochs-1) + if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0)): + torch.save(model.module.state_dict(), 'model_before_finetune_epoch_{}.pth'.format(epoch)) + # save_checkpoint({ + # 'epoch': epoch + 1, + # 'arch': args.arch, + # 'state_dict': model.state_dict(), + # 'best_acc1': best_acc1, + # 'optimizer' : optimizer.state_dict(), + # 'scheduler' : scheduler.state_dict() + # }, is_best) + + +def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): + batch_time = AverageMeter('Time', ':6.3f') + data_time = AverageMeter('Data', ':6.3f') + losses = AverageMeter('Loss', ':.4e') + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix="Epoch: [{}]".format(epoch)) + + # switch to train mode + model.train() + + end = time.time() + for i, (images, target) in enumerate(train_loader): + # measure data loading time + data_time.update(time.time() - end) + + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + if torch.cuda.is_available(): + target = target.cuda(args.gpu, non_blocking=True) + + if not args.finetune: + # project to [0, 1] in every gradient step + for name, params in model.named_parameters(): + # make sure param_name ends with .scores and not bias_scores + if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): + with torch.no_grad(): + params.data = torch.clamp(params.data, 0.0, 1.0) + + # compute output + if scaler is None: + output = model(images) + loss = criterion(output, target) + else: + with torch.cuda.amp.autocast(enabled=True): + output = model(images) + loss = criterion(output, target) + + if not args.finetune: + regularization_loss = get_regularization_loss(model, args) + # print("Regularization loss: {}".format(regularization_loss.item())) + loss += regularization_loss + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + if scaler is None: + loss.backward() + optimizer.step() + else: + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + print("Regularization Loss={} | Total Loss={}".format(regularization_loss, loss)) + progress.display(i) + + if not args.finetune: + # project to [0, 1] before returning model + for name, params in model.named_parameters(): + # make sure param_name ends with .scores and not bias_scores + if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): + with torch.no_grad(): + params.data = torch.clamp(params.data, 0.0, 1.0) + + return top1.avg + + +def validate(val_loader, model, criterion, args): + batch_time = AverageMeter('Time', ':6.3f', Summary.NONE) + losses = AverageMeter('Loss', ':.4e', Summary.NONE) + top1 = AverageMeter('Acc@1', ':6.2f', Summary.AVERAGE) + top5 = AverageMeter('Acc@5', ':6.2f', Summary.AVERAGE) + progress = ProgressMeter( + len(val_loader), + [batch_time, losses, top1, top5], + prefix='Test: ') + + # switch to evaluate mode + model.eval() + + with torch.no_grad(): + end = time.time() + for i, (images, target) in enumerate(val_loader): + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + if torch.cuda.is_available(): + target = target.cuda(args.gpu, non_blocking=True) + + # compute output + output = model(images) + loss = criterion(output, target) + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + progress.display(i) + + progress.display_summary() + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best.pth.tar') + +class Summary(Enum): + NONE = 0 + AVERAGE = 1 + SUM = 2 + COUNT = 3 + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f', summary_type=Summary.AVERAGE): + self.name = name + self.fmt = fmt + self.summary_type = summary_type + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + def summary(self): + fmtstr = '' + if self.summary_type is Summary.NONE: + fmtstr = '' + elif self.summary_type is Summary.AVERAGE: + fmtstr = '{name} {avg:.3f}' + elif self.summary_type is Summary.SUM: + fmtstr = '{name} {sum:.3f}' + elif self.summary_type is Summary.COUNT: + fmtstr = '{name} {count:.3f}' + else: + raise ValueError('invalid summary type %r' % self.summary_type) + + return fmtstr.format(**self.__dict__) + + +class ProgressMeter(object): + def __init__(self, num_batches, meters, prefix=""): + self.batch_fmtstr = self._get_batch_fmtstr(num_batches) + self.meters = meters + self.prefix = prefix + + def display(self, batch): + entries = [self.prefix + self.batch_fmtstr.format(batch)] + entries += [str(meter) for meter in self.meters] + print('\t'.join(entries)) + + def display_summary(self): + entries = [" *"] + entries += [meter.summary() for meter in self.meters] + print(' '.join(entries)) + + def _get_batch_fmtstr(self, num_batches): + num_digits = len(str(num_batches // 1)) + fmt = '{:' + str(num_digits) + 'd}' + return '[' + fmt + '/' + fmt.format(num_batches) + ']' + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +""" +@ksreenivasan: Dumping new code here +""" +def print_time(msg): + print("\n\n----------------------------------------------------------------------------") + print("{}".format(msg)) + print("TIME: The current time is: {}".format(time.ctime())) + print("TIME: The current time in seconds is: {}".format(time.time())) + print("----------------------------------------------------------------------------\n\n") + +def get_regularization_loss(model, args): + conv_layers, linear_layers = get_layers(args.arch, model) + regularization_loss = torch.tensor(0.).to(args.gpu) + + # reg_loss = ||p||_2^2 + for name, params in model.named_parameters(): + if ".bias_score" in name: + # do nothing, because I'm pretending there are no biases + regularization_loss += 0 + + elif ".score" in name: + regularization_loss += torch.norm(params, p=2)**2 + regularization_loss = args.lmbda * regularization_loss + return regularization_loss + +# return layer objects of conv layers and linear layers so we can parse them +# efficiently +def get_layers(arch='ResNet50', dist_model=None): + if isinstance(dist_model, nn.parallel.DistributedDataParallel): + model = dist_model.module + else: + model = dist_model + + if arch == 'ResNet50': + conv_layers = [model.conv1] + for layer in [model.layer1, model.layer2, model.layer3, model.layer4]: + for basic_block_id in [i for i in range(len(layer))]: + conv_layers.append(layer[basic_block_id].conv1) + conv_layers.append(layer[basic_block_id].conv2) + conv_layers.append(layer[basic_block_id].conv3) + # handle shortcut + # if len(layer[basic_block_id].shortcut) > 0: + # conv_layers.append(layer[basic_block_id].shortcut[0]) + linear_layers = [model.fc] + + return (conv_layers, linear_layers) + + +# switches off gradients for scores and flags and switches it on for weights and biases +def switch_to_wt(model): + print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') + + for name, params in model.named_parameters(): + # make sure param_name ends with .weight or .bias + if re.match('.*\.weight', name): + params.requires_grad = True + elif re.match('.*\.bias$', name): + params.requires_grad = True + elif "score" in name: + params.requires_grad = False + else: + # flags and everything else + params.requires_grad = False + + return model + +# switches off gradients for weights and biases and switches it on for scores and flags +def switch_to_prune(model): + print('Switching to pruning by switching off requires_grad for weights and switching it on for scores.') + + for name, params in model.named_parameters(): + # make sure param_name ends with .scores and not .bias_scores + if re.match('.*\.scores', name) and not re.match('.*\.bias_scores', name): + params.requires_grad = True + else: + # weights, biases, bias_scores, flags and everything else + params.requires_grad = False + + return model + + +def round_model(model, round_scheme='naive'): + quantize_threshold=0.5 + print("Rounding model with scheme: {}".format(round_scheme)) + cp_model = copy.deepcopy(model) + if isinstance(model, nn.parallel.DistributedDataParallel): + # cp_model = copy.deepcopy(model.module) + named_params = cp_model.module.named_parameters() + else: + # cp_model = copy.deepcopy(model) + named_params = cp_model.named_parameters() + for name, params in named_params: + if re.match('.*\.scores', name): + if round_scheme == 'naive': + params.data = torch.gt(params.data, torch.ones_like( + params.data)*quantize_threshold).int().float() + elif round_scheme == 'prob': + params.data = torch.clamp(params.data, 0.0, 1.0) + params.data = torch.bernoulli(params.data).float() + elif round_scheme == 'all_ones': + params.data = torch.ones_like(params.data) + else: + print("INVALID ROUNDING") + print("EXITING") + exit() + + # if isinstance(model, nn.parallel.DistributedDataParallel): + # cp_model = nn.parallel.DistributedDataParallel( + # cp_model, device_ids=[parser_args.gpu], find_unused_parameters=False) + + return cp_model + + +def prune(model, args, update_thresholds_only=False, update_scores=False): + print("Pruning Model:") + + scores_threshold = bias_scores_threshold = -np.inf + conv_layers, linear_layers = get_layers(args.arch, model) + + # prune the bottom k of scores + num_active_weights = 0 + num_active_biases = 0 + active_scores_list = [] + active_bias_scores_list = [] + + for layer in (conv_layers + linear_layers): + num_active_weights += layer.flag.data.sum().item() + active_scores = (layer.scores.data[layer.flag.data == 1]).clone() + active_scores_list.append(active_scores) + if args.bias: + num_active_biases += layer.bias_flag.data.sum().item() + active_biases = ( + layer.bias_scores.data[layer.bias_flag.data == 1]).clone() + active_bias_scores_list.append(active_biases) + + number_of_weights_to_prune = np.ceil( + args.prune_rate * num_active_weights).astype(int) + number_of_biases_to_prune = np.ceil( + args.prune_rate * num_active_biases).astype(int) + + agg_scores = torch.cat(active_scores_list) + agg_bias_scores = torch.cat( + active_bias_scores_list) if args.bias else torch.tensor([]) + + # if invert_sanity_check, then threshold is based on sorted scores in descending order, and we prune all scores ABOVE it + scores_threshold = torch.sort( + torch.abs(agg_scores), descending=args.invert_sanity_check).values[number_of_weights_to_prune-1].item() + + if args.bias: + bias_scores_threshold = torch.sort( + torch.abs(agg_bias_scores), descending=args.invert_sanity_check).values[number_of_biases_to_prune-1].item() + else: + bias_scores_threshold = -1 + + if update_thresholds_only: + for layer in (conv_layers + linear_layers): + layer.scores_prune_threshold = scores_threshold + if args.bias: + layer.bias_scores_prune_threshold = bias_scores_threshold + + else: + for layer in (conv_layers + linear_layers): + if args.invert_sanity_check: + layer.flag.data = (layer.flag.data + torch.lt(layer.scores.abs(), # TODO + torch.ones_like(layer.scores)*scores_threshold).int() == 2).int() + else: + layer.flag.data = (layer.flag.data + torch.gt(layer.scores.abs(), # TODO + torch.ones_like(layer.scores)*scores_threshold).int() == 2).int() + if update_scores: + layer.scores.data = layer.scores.data * layer.flag.data + if args.bias: + if args.invert_sanity_check: + layer.bias_flag.data = (layer.bias_flag.data + torch.lt(layer.bias_scores, torch.ones_like( + layer.bias_scores)*bias_scores_threshold).int() == 2).int() + else: + layer.bias_flag.data = (layer.bias_flag.data + torch.gt(layer.bias_scores, torch.ones_like( + layer.bias_scores)*bias_scores_threshold).int() == 2).int() + if update_scores: + layer.bias_scores.data = layer.bias_scores.data * layer.bias_flag.data + + + return scores_threshold, bias_scores_threshold + + +# returns num_nonzero elements, total_num_elements so that it is easier to compute +# average sparsity in the end +def get_layer_sparsity(layer, threshold=0, args=None): + # assume the model is rounded, compute effective scores + eff_scores = layer.scores * layer.flag + if args.bias: + eff_bias_scores = layer.bias_scores * layer.bias_flag + num_middle = torch.sum(torch.gt(eff_scores, + torch.ones_like(eff_scores)*threshold) * + torch.lt(eff_scores, + torch.ones_like(eff_scores.detach()*(1-threshold)).int())) + if num_middle > 0: + print("WARNING: Model scores are not binary. Sparsity number is unreliable.") + raise ValueError + w_numer, w_denom = eff_scores.detach().sum().item(), eff_scores.detach().flatten().numel() + + if args.bias: + b_numer, b_denom = eff_bias_scores.detach().sum().item(), eff_bias_scores.detach().flatten().numel() + else: + b_numer, b_denom = 0, 0 + + return w_numer, w_denom, b_numer, b_denom + + +# returns avg_sparsity = number of non-zero weights! +def get_model_sparsity(model, threshold=0, args=None): + conv_layers, linear_layers = get_layers(args.arch, model) + numer = 0 + denom = 0 + + # TODO: find a nicer way to do this (skip dropout) + # TODO: Update: can't use .children() or .named_modules() because of the way things are wrapped in builder + for conv_layer in conv_layers: + w_numer, w_denom, b_numer, b_denom = get_layer_sparsity( + conv_layer, threshold, args) + numer += w_numer + denom += w_denom + if args.bias: + numer += b_numer + denom += b_denom + + for lin_layer in linear_layers: + w_numer, w_denom, b_numer, b_denom = get_layer_sparsity( + lin_layer, threshold, args) + numer += w_numer + denom += w_denom + if args.bias: + numer += b_numer + denom += b_denom + # print('Overall sparsity: {}/{} ({:.2f} %)'.format((int)(numer), denom, 100*numer/denom)) + return 100*numer/denom + + +def get_prune_rate(target_sparsity=0.5, iter_period=5, max_epochs=88): + print("Computing prune_rate for target_sparsity {} with iter_period {}".format( + target_sparsity, iter_period)) + num_prune_iterations = np.floor((max_epochs-1)/iter_period) + # if algo is HC, iter_HC or anything that uses prune() then, prune_rate represents number of weights to prune + prune_rate = 1 - np.exp(np.log(target_sparsity/100)/num_prune_iterations) + return prune_rate + + +if __name__ == '__main__': + main() diff --git a/imagenet/models_cifar.py b/imagenet/models_cifar.py new file mode 100644 index 00000000..f1983d15 --- /dev/null +++ b/imagenet/models_cifar.py @@ -0,0 +1,694 @@ +import torch +import torch.autograd as autograd +import torch.nn as nn +import torch.nn.functional as F +import numpy as np + +import math + +class LambdaLayer(nn.Module): + def __init__(self, lambd): + super(LambdaLayer, self).__init__() + self.lambd = lambd + + def forward(self, x): + return self.lambd(x) + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, builder, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = builder.conv3x3(in_planes, planes, stride=stride) + self.bn1 = builder.batchnorm(planes) + self.conv2 = builder.conv3x3(planes, planes, stride=1) + self.bn2 = builder.batchnorm(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = LambdaLayer(lambda x: + F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes//4, planes//4), "constant", 0)) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, builder, block, num_blocks, times=1): + # the times is added, because in smart ratio paper, by default they use twice the channel as standard implementation. + # so to reproduce the result in their paper, we create this resnet32_double, and set times=2 here + # by default times is always 1. + super(ResNet, self).__init__() + self.in_planes = 16 * times + self.builder = builder + + self.conv1 = builder.conv3x3(3, 16 * times, stride=1, first_layer=True) + self.bn1 = builder.batchnorm(16 * times) + self.layer1 = self._make_layer(block, 16 * times, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 32 * times, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 64 * times, num_blocks[2], stride=2) + # self.avgpool = nn.AdaptiveAvgPool2d(1) + + num_classes = 10 + + self.fc = builder.conv1x1(64 * block.expansion * times, num_classes) # 10 = num_classes for cifar10 + + self.prunable_layer_names, self.prunable_biases = self.get_prunable_param_names() + + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.builder, self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + + return nn.Sequential(*layers) + + def get_prunable_param_names(model): + prunable_weights = [name + '.weight' for name, module in model.named_modules() if + isinstance(module, torch.nn.modules.conv.Conv2d) or + isinstance(module, torch.nn.modules.linear.Linear)] + prunable_biases = [""] + + return prunable_weights, prunable_biases + + def forward(self, x): + # update score thresholds for global ep + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = F.avg_pool2d(out, out.size()[3]) + out = self.fc(out) + return out.flatten(1) + + +def resnet20(): + return ResNet(get_builder(), BasicBlock, [3, 3, 3]) + + + +#######imagenet models############ + +# # BasicBlock {{{ +# class BasicBlock(nn.Module): +# M = 2 +# expansion = 1 + +# def __init__(self, builder, inplanes, planes, stride=1, downsample=None, base_width=64): +# super(BasicBlock, self).__init__() +# if base_width / 64 > 1: +# raise ValueError("Base width >64 does not work for BasicBlock") + +# self.conv1 = builder.conv3x3(inplanes, planes, stride) +# self.bn1 = builder.batchnorm(planes) +# self.relu = builder.activation() +# self.conv2 = builder.conv3x3(planes, planes) +# self.bn2 = builder.batchnorm(planes, last_bn=True) +# self.downsample = downsample +# self.stride = stride + +# def forward(self, x): +# residual = x + +# out = self.conv1(x) +# if self.bn1 is not None: +# out = self.bn1(out) + +# out = self.relu(out) + +# out = self.conv2(out) + +# if self.bn2 is not None: +# out = self.bn2(out) + +# if self.downsample is not None: +# residual = self.downsample(x) + +# out += residual +# out = self.relu(out) + +# return out + + +# class Bottleneck(nn.Module): +# M = 3 +# expansion = 4 + +# def __init__(self, builder, inplanes, planes, stride=1, downsample=None, base_width=64): +# super(Bottleneck, self).__init__() +# width = int(planes * base_width / 64) +# self.conv1 = builder.conv1x1(inplanes, width) +# self.bn1 = builder.batchnorm(width) +# self.conv2 = builder.conv3x3(width, width, stride=stride) +# self.bn2 = builder.batchnorm(width) +# self.conv3 = builder.conv1x1(width, planes * self.expansion) +# self.bn3 = builder.batchnorm(planes * self.expansion, last_bn=True) +# self.relu = builder.activation() +# self.downsample = downsample +# self.stride = stride + +# def forward(self, x): +# residual = x + +# out = self.conv1(x) +# out = self.bn1(out) +# out = self.relu(out) + +# out = self.conv2(out) +# out = self.bn2(out) +# out = self.relu(out) + +# out = self.conv3(out) +# out = self.bn3(out) + +# if self.downsample is not None: +# residual = self.downsample(x) + +# out += residual + +# out = self.relu(out) + +# return out + + +# # Bottleneck }}} + +# # ResNet {{{ +# class ResNet(nn.Module): +# def __init__(self, builder, block, layers, num_classes=1000, base_width=64): +# self.args_first_layer_dense = False +# self.args_last_layer_dense = False +# self.args_bias = False + +# self.inplanes = 64 +# super(ResNet, self).__init__() + +# self.base_width = base_width +# if self.base_width // 64 > 1: +# print(f"==> Using {self.base_width // 64}x wide model") + +# if self.args_first_layer_dense: +# self.conv1 = nn.Conv2d( +# 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False +# ) +# else: +# self.conv1 = builder.conv7x7(3, 64, stride=2, first_layer=True) + +# self.bn1 = builder.batchnorm(64) +# self.relu = builder.activation() +# self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) +# self.layer1 = self._make_layer(builder, block, 64, layers[0]) +# self.layer2 = self._make_layer(builder, block, 128, layers[1], stride=2) +# self.layer3 = self._make_layer(builder, block, 256, layers[2], stride=2) +# self.layer4 = self._make_layer(builder, block, 512, layers[3], stride=2) +# self.avgpool = nn.AdaptiveAvgPool2d(1) + +# # self.fc = nn.Linear(512 * block.expansion, num_classes) +# if self.args_last_layer_dense: +# self.fc = nn.Conv2d(512 * block.expansion, num_classes, 1) +# else: +# # self.fc = builder.conv1x1(512 * block.expansion, num_classes) +# self.fc = builder.linear(512 * block.expansion, num_classes) + +# self.prunable_layer_names, self.prunable_biases = self.get_prunable_param_names() + +# def _make_layer(self, builder, block, planes, blocks, stride=1): +# downsample = None +# if stride != 1 or self.inplanes != planes * block.expansion: +# dconv = builder.conv1x1( +# self.inplanes, planes * block.expansion, stride=stride +# ) +# dbn = builder.batchnorm(planes * block.expansion) +# if dbn is not None: +# downsample = nn.Sequential(dconv, dbn) +# else: +# downsample = dconv + +# layers = [] +# layers.append(block(builder, self.inplanes, planes, stride, downsample, base_width=self.base_width)) +# self.inplanes = planes * block.expansion +# for i in range(1, blocks): +# layers.append(block(builder, self.inplanes, planes, base_width=self.base_width)) + +# return nn.Sequential(*layers) + +# def get_prunable_param_names(self): +# prunable_weights = [name + '.weight' for name, module in self.named_modules() if +# isinstance(module, nn.modules.conv.Conv2d) or +# isinstance(module, nn.modules.linear.Linear)] +# if self.args_bias: +# prunable_biases = [name + '.bias' for name, module in self.named_modules() if +# isinstance(module, nn.modules.conv.Conv2d) or +# isinstance(module, nn.modules.linear.Linear)] +# else: +# prunable_biases = [""] + +# return prunable_weights, prunable_biases + +# def forward(self, x): +# # update score thresholds for global ep +# # TODO: Don't need this for now +# # if parser_args.algo in ['global_ep', 'global_ep_iter'] or parser_args.bottom_k_on_forward: +# # prune(self, update_thresholds_only=True) +# x = self.conv1(x) + +# if self.bn1 is not None: +# x = self.bn1(x) +# x = self.relu(x) +# x = self.maxpool(x) + +# x = self.layer1(x) +# x = self.layer2(x) +# x = self.layer3(x) +# x = self.layer4(x) + +# x = self.avgpool(x) +# x = torch.flatten(x, 1) +# x = self.fc(x) +# x = x.view(x.size(0), -1) + +# return x + + +# # ResNet }}} +# def ResNet18(pretrained=False): +# return ResNet(get_builder(), BasicBlock, [2, 2, 2, 2], 1000) + + +# def ResNet50(pretrained=False): +# return ResNet(get_builder(), Bottleneck, [3, 4, 6, 3], 1000) + + +# def ResNet101(pretrained=False): +# return ResNet(get_builder(), Bottleneck, [3, 4, 23, 3], 200) +# #return ResNet(get_builder(), Bottleneck, [3, 4, 23, 3], 1000) + + +# def WideResNet50_2(pretrained=False): +# return ResNet( +# get_builder(), Bottleneck, [3, 4, 6, 3], num_classes=1000, base_width=64 * 2 +# ) + + +# def WideResNet101_2(pretrained=False): +# return ResNet( +# get_builder(), Bottleneck, [3, 4, 23, 3], num_classes=1000, base_width=64 * 2 +# ) + + +class Builder(object): + def __init__(self, conv_layer, bn_layer, first_layer=None, weight_init="signed_constant"): + self.conv_layer = conv_layer + self.bn_layer = bn_layer + # self.first_layer = first_layer or conv_layer + self.first_layer = conv_layer + self.weight_init = weight_init + + def conv(self, kernel_size, in_planes, out_planes, stride=1, first_layer=False, groups=1): + # conv_layer = self.first_layer if first_layer else self.conv_layer + conv_layer = self.conv_layer + + if first_layer: + print(f"==> Building first layer") + + if kernel_size == 3: + conv = conv_layer( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=True, + groups=groups + ) + elif kernel_size == 1: + conv = conv_layer( + in_planes, out_planes, + kernel_size=1, + stride=stride, + bias=True, + ) + elif kernel_size == 5: + conv = conv_layer( + in_planes, + out_planes, + kernel_size=5, + stride=stride, + padding=2, + bias=True, + ) + elif kernel_size == 7: + conv = conv_layer( + in_planes, + out_planes, + kernel_size=7, + stride=stride, + padding=3, + bias=True, + ) + else: + return None + + self._init_conv(conv) + + return conv + + def conv3x3(self, in_planes, out_planes, stride=1, first_layer=False, groups=1): + """3x3 convolution with padding""" + c = self.conv(3, in_planes, out_planes, stride=stride, first_layer=first_layer, groups=groups) + return c + + def conv1x1(self, in_planes, out_planes, stride=1, first_layer=False): + """1x1 convolution with padding""" + c = self.conv(1, in_planes, out_planes, stride=stride, first_layer=first_layer) + return c + + def linear(self, in_planes, out_planes): + l = SubnetLinear(in_planes, out_planes, bias=True) + self._init_conv(l) + return l + + def conv7x7(self, in_planes, out_planes, stride=1, first_layer=False): + """7x7 convolution with padding""" + c = self.conv(7, in_planes, out_planes, stride=stride, first_layer=first_layer) + return c + + def conv5x5(self, in_planes, out_planes, stride=1, first_layer=False): + """5x5 convolution with padding""" + c = self.conv(5, in_planes, out_planes, stride=stride, first_layer=first_layer) + return c + + def batchnorm(self, planes, last_bn=False, first_layer=False): + return self.bn_layer(planes) + + def activation(self): + # always ReLU + return (lambda: nn.ReLU(inplace=True))() + + def _init_conv(self, conv): + if self.weight_init == "signed_constant": + fan = nn.init._calculate_correct_fan(conv.weight, "fan_in") + # scale_fan = False always because this isn't EP + # if scale_fan: + # fan = fan * (1 - parser_args.prune_rate) + gain = nn.init.calculate_gain("relu") + std = gain / math.sqrt(fan) + conv.weight.data = conv.weight.data.sign() * std + + elif self.weight_init == "unsigned_constant": + fan = nn.init._calculate_correct_fan(conv.weight, "fan_in") + # scale_fan = False always because this isn't EP + # if parser_args.scale_fan: + # fan = fan * (1 - parser_args.prune_rate) + gain = nn.init.calculate_gain("relu") + std = gain / math.sqrt(fan) + conv.weight.data = torch.ones_like(conv.weight.data) * std + + elif self.weight_init == "kaiming_normal": + # scale_fan = False always because this isn't EP + # if parser_args.scale_fan: + # fan = nn.init._calculate_correct_fan(conv.weight, parser_args.mode) + # fan = fan * (1 - parser_args.prune_rate) + # gain = nn.init.calculate_gain(parser_args.nonlinearity) + # std = gain / math.sqrt(fan) + # with torch.no_grad(): + # conv.weight.data.normal_(0, std) + # else: + # nn.init.kaiming_normal_( + # conv.weight, mode=parser_args.mode, nonlinearity=parser_args.nonlinearity + # ) + nn.init.kaiming_normal_( + conv.weight, mode="fan_in", nonlinearity="relu" + ) + + elif self.weight_init == "kaiming_uniform": + nn.init.kaiming_uniform_( + conv.weight, mode="fan_in", nonlinearity="relu" + ) + + elif self.weight_init == "xavier_normal": + nn.init.xavier_normal_(conv.weight) + + elif self.weight_init == "xavier_constant": + fan_in, fan_out = nn.init._calculate_fan_in_and_fan_out(conv.weight) + std = math.sqrt(2.0 / float(fan_in + fan_out)) + conv.weight.data = conv.weight.data.sign() * std + + elif self.weight_init == "standard": + nn.init.kaiming_uniform_(conv.weight, a=math.sqrt(5)) + + else: + raise ValueError(f"{weight_init} is not an initialization option!") + + +def get_builder(): + conv_type = "SubnetConv" + bn_type = "NonAffineBatchNorm" # TODO: might change this if it causes problems later + first_layer_type = None # TODO: I think + weight_init = "signed_constant" + + print("==> Conv Type: {}".format(conv_type)) + print("==> BN Type: {}".format(bn_type)) + + # need to fix this + # conv_layer = getattr(utils.conv_type, parser_args.conv_type) + # bn_layer = getattr(utils.bn_type, parser_args.bn_type) + conv_layer = SubnetConv + bn_layer = NonAffineBatchNorm + + if first_layer_type is not None: + first_layer = getattr(utils.conv_type, first_layer_type) + print(f"==> First Layer Type: {first_layer_type}") + else: + first_layer = None + + builder = Builder(conv_layer=conv_layer, bn_layer=bn_layer, first_layer=first_layer, weight_init=weight_init) + + return builder + +class GetSubnet(autograd.Function): + @staticmethod + def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_scores_prune_threshold=-np.inf): + algo = 'hc_iter' + quantize_threshold = 0.5 + if algo == 'ep': + # Get the supermask by sorting the scores and using the top k% + out = scores.clone() + _, idx = scores.flatten().sort() + j = int((1 - k) * scores.numel()) + # flat_out and out access the same memory. + flat_out = out.flatten() + flat_out[idx[:j]] = 0 + flat_out[idx[j:]] = 1 + + # repeat for bias + # Get the supermask by sorting the scores and using the top k% + bias_out = bias_scores.clone() + _, idx = bias_scores.flatten().sort() + j = int((1 - k) * bias_scores.numel()) + + # flat_out and out access the same memory. + bias_flat_out = bias_out.flatten() + bias_flat_out[idx[:j]] = 0 + bias_flat_out[idx[j:]] = 1 + + elif algo in ['global_ep', 'global_ep_iter']: + # define out, bias_out based on the layer's prune_threshold, bias_threshold + out = torch.gt(scores, torch.ones_like(scores)*scores_prune_threshold).float() + bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float() + + elif algo in ['hc', 'hc_iter']: + # round scores to {0, 1} + out = torch.gt(scores, torch.ones_like(scores)*quantize_threshold).float() + bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*quantize_threshold).float() + + else: + print("INVALID PRUNING ALGO") + print("EXITING") + exit() + + return out, bias_out + + @staticmethod + def backward(ctx, g_1, g_2): + # send the gradient g straight-through on the backward pass. + return g_1, g_2, None, None, None + + +# Not learning weights, finding subnet +class SubnetConv(nn.Conv2d): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.args_bias = False + self.algo = 'hc' + self.prune_rate = 0.5 + # resnet50 has bias=False because of BN layers + self.bias = None + + # initialize flag (representing the pruned weights) + self.flag = nn.Parameter(torch.ones(self.weight.size())) + if self.args_bias: + self.bias_flag = nn.Parameter(torch.ones(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_flag = nn.Parameter(torch.Tensor(1)) + + # initialize the scores + self.scores = nn.Parameter(torch.Tensor(self.weight.size())) + if self.args_bias: + self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_scores = nn.Parameter(torch.Tensor(1)) + + # prune scores below this for global EP in bottom-k + self.scores_prune_threshold = -np.inf + self.bias_scores_prune_threshold = -np.inf + + if self.algo in ['hc', 'hc_iter']: + # score init is always uniform + nn.init.uniform_(self.scores, a=0.0, b=1.0) + nn.init.uniform_(self.bias_scores, a=0.0, b=1.0) + else: + nn.init.kaiming_uniform_(self.scores, a=math.sqrt(5)) + nn.init.uniform_(self.bias_scores, a=-1.0, b=1.0) # can't do kaiming here. picking U[-1, 1] for no real reason + + # NOTE: turn the gradient on the weights off + self.weight.requires_grad = False + self.flag.requires_grad = False + self.bias_flag.requires_grad = False + + def set_prune_rate(self, prune_rate): + self.prune_rate = prune_rate + + @property + def clamped_scores(self): + return self.scores.abs() + + def forward(self, x): + if self.algo in ['hc', 'hc_iter', 'transformer']: + subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, self.prune_rate) + subnet = subnet * self.flag.data.float() + bias_subnet = subnet * self.bias_flag.data.float() + elif self.algo in ['imp']: + # no STE, no subnet. Mask is handled outside + pass + elif self.algo in ['global_ep', 'global_ep_iter']: + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) + else: + # ep, global_ep, global_ep_iter, pt etc + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), self.prune_rate) + + if self.algo in ['imp']: + # no STE, no subnet. Mask is handled outside + w = self.weight + b = self.bias + else: + w = self.weight * subnet + if self.args_bias: + b = self.bias * bias_subnet + else: + b = self.bias + + x = F.conv2d( + x, w, b, self.stride, self.padding, self.dilation, self.groups + ) + + return x + + +class SubnetLinear(nn.Linear): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # TODO: hacky. trying to mimic frankle in having biases but not pruning them + self.args_bias = False + self.algo = 'hc' + self.prune_rate = 0.5 + # resnet50 has bias=True only for the FC layer + + # initialize flag (representing the pruned weights) + self.flag = nn.Parameter(torch.ones(self.weight.size())) + if self.args_bias: + self.bias_flag = nn.Parameter(torch.ones(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_flag = nn.Parameter(torch.Tensor(1)) + + # initialize the scores + self.scores = nn.Parameter(torch.Tensor(self.weight.size())) + if self.args_bias: + self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_scores = nn.Parameter(torch.Tensor(1)) + + # prune scores below this for global EP in bottom-k + self.scores_prune_threshold = -np.inf + self.bias_scores_prune_threshold = -np.inf + + if self.algo in ['hc', 'hc_iter']: + # score init is always uniform + nn.init.uniform_(self.scores, a=0.0, b=1.0) + nn.init.uniform_(self.bias_scores, a=0.0, b=1.0) + else: + nn.init.kaiming_uniform_(self.scores, a=math.sqrt(5)) + nn.init.uniform_(self.bias_scores, a=-1.0, b=1.0) # can't do kaiming here. picking U[-1, 1] for no real reason + + # NOTE: turn the gradient on the weights off + self.weight.requires_grad = False + self.flag.requires_grad = False + self.bias_flag.requires_grad = False + # TODO: Hacky. I'm trying to mimic frankle etc in that we have biases, but we don't prune them + self.bias.requires_grad = False + + def set_prune_rate(self, prune_rate): + self.prune_rate = prune_rate + + @property + def clamped_scores(self): + return self.scores.abs() + + def forward(self, x): + if self.algo in ['hc', 'hc_iter']: + # don't need a mask here. the scores are directly multiplied with weights + subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, self.prune_rate) + subnet = subnet * self.flag.data.float() + bias_subnet = subnet * self.bias_flag.data.float() + elif self.algo in ['imp']: + # no STE, no subnet. Mask is handled outside + pass + elif parser_args.algo in ['global_ep', 'global_ep_iter']: + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) + else: + # ep, global_ep, global_ep_iter, pt etc + subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), self.prune_rate) + + if self.algo in ['imp']: + # no STE, no subnet. Mask is handled outside + w = self.weight + b = self.bias + else: + w = self.weight * subnet + if self.args_bias: + b = self.bias * bias_subnet + else: + b = self.bias + + x = F.linear(x, w, b) + return x + + +class NonAffineBatchNorm(nn.BatchNorm2d): + def __init__(self, dim): + super(NonAffineBatchNorm, self).__init__(dim, affine=False) + + +class AffineBatchNorm(nn.BatchNorm2d): + def __init__(self, dim): + super(AffineBatchNorm, self).__init__(dim, affine=True) From 7a933c9fc5f0849a03fd50d2edb9ae4f571c0932 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 3 Jun 2022 16:03:58 -0500 Subject: [PATCH 105/113] changing model to resnet20 --- imagenet/imagenet_main_cifar.py | 4 ++- imagenet/run_cifar.sh | 53 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 imagenet/run_cifar.sh diff --git a/imagenet/imagenet_main_cifar.py b/imagenet/imagenet_main_cifar.py index c5ea22dd..c984da89 100644 --- a/imagenet/imagenet_main_cifar.py +++ b/imagenet/imagenet_main_cifar.py @@ -27,6 +27,7 @@ import torch.nn.functional as F import torchvision.models as torchvision_models import models +import models_cifar model_names = sorted(name for name in torchvision_models.__dict__ if name.islower() and not name.startswith("__") @@ -184,7 +185,8 @@ def main_worker(gpu, ngpus_per_node, args): # args.arch = 'resnet50' print("==> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() - model = models.ResNet50() + # model = models.ResNet50() + model = models_cifar.resnet20() if not args.finetune: model = switch_to_prune(model) if args.finetune: diff --git a/imagenet/run_cifar.sh b/imagenet/run_cifar.sh new file mode 100644 index 00000000..78fd05d6 --- /dev/null +++ b/imagenet/run_cifar.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +#:< Date: Fri, 3 Jun 2022 16:06:21 -0500 Subject: [PATCH 106/113] adding get_layers for resnet20 --- imagenet/imagenet_main_cifar.py | 14 ++++++++++++++ imagenet/run_cifar.sh | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/imagenet/imagenet_main_cifar.py b/imagenet/imagenet_main_cifar.py index c984da89..c6063d84 100644 --- a/imagenet/imagenet_main_cifar.py +++ b/imagenet/imagenet_main_cifar.py @@ -689,6 +689,20 @@ def get_layers(arch='ResNet50', dist_model=None): # conv_layers.append(layer[basic_block_id].shortcut[0]) linear_layers = [model.fc] + elif arch == 'ResNet20': + conv_layers = [model.conv1] + for layer in [model.layer1, model.layer2, model.layer3]: + for basic_block_id in [0, 1, 2]: + conv_layers.append(layer[basic_block_id].conv1) + conv_layers.append(layer[basic_block_id].conv2) + ''' + # handle shortcut + if len(layer[basic_block_id].shortcut) > 0: + conv_layers.append(layer[basic_block_id].shortcut[0]) + ''' + linear_layers = [model.fc] + + return (conv_layers, linear_layers) diff --git a/imagenet/run_cifar.sh b/imagenet/run_cifar.sh index 78fd05d6..97d1d3ee 100644 --- a/imagenet/run_cifar.sh +++ b/imagenet/run_cifar.sh @@ -2,7 +2,7 @@ #:< Date: Fri, 3 Jun 2022 16:11:26 -0500 Subject: [PATCH 107/113] minor typo for cifar10 ddp, looks like it works in that case, the issue might just be tuning in imagenet --- imagenet/imagenet_main_cifar.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/imagenet/imagenet_main_cifar.py b/imagenet/imagenet_main_cifar.py index c6063d84..ed30586a 100644 --- a/imagenet/imagenet_main_cifar.py +++ b/imagenet/imagenet_main_cifar.py @@ -310,13 +310,13 @@ def main_worker(gpu, ngpus_per_node, args): use_cuda = torch.cuda.is_available() # Data loading code - kwargs = {"num_workers": args.num_workers, "pin_memory": True} if use_cuda else {} + kwargs = {"num_workers": args.workers, "pin_memory": True} if use_cuda else {} normalize = transforms.Normalize( mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262] ) - train_dataset = torchvision.datasets.CIFAR10( + train_dataset = datasets.CIFAR10( root=data_root, train=True, download=True, @@ -330,14 +330,14 @@ def main_worker(gpu, ngpus_per_node, args): ), ) - test_dataset = torchvision.datasets.CIFAR10( + test_dataset = datasets.CIFAR10( root=data_root, train=False, download=True, transform=transforms.Compose([transforms.ToTensor(), normalize]), ) - if parser_args.multiprocessing_distributed: + if args.multiprocessing_distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None From 782a85fa869cec2abdbf99899c945524a131510d Mon Sep 17 00:00:00 2001 From: root Date: Fri, 3 Jun 2022 16:15:34 -0500 Subject: [PATCH 108/113] print reg_loss only for pruning --- imagenet/imagenet_main.py | 3 ++- imagenet/imagenet_main_cifar.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 60eaae59..168919a0 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -445,7 +445,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): end = time.time() if i % args.print_freq == 0: - print("Regularization Loss={} | Total Loss={}".format(regularization_loss, loss)) + if not args.finetune: + print("Regularization Loss={} | Total Loss={}".format(regularization_loss, loss)) progress.display(i) if not args.finetune: diff --git a/imagenet/imagenet_main_cifar.py b/imagenet/imagenet_main_cifar.py index ed30586a..7811553a 100644 --- a/imagenet/imagenet_main_cifar.py +++ b/imagenet/imagenet_main_cifar.py @@ -498,7 +498,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): end = time.time() if i % args.print_freq == 0: - print("Regularization Loss={} | Total Loss={}".format(regularization_loss, loss)) + if not args.finetune: + print("Regularization Loss={} | Total Loss={}".format(regularization_loss, loss)) progress.display(i) if not args.finetune: From b139623d33043de4cf0cbe5770b6b515896d2166 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sun, 5 Jun 2022 22:36:53 -0500 Subject: [PATCH 109/113] adding optimizer and results subdir for hyperparam opt --- imagenet/imagenet_main.py | 27 +++++++++++-- imagenet/imagenet_main_cifar.py | 7 +++- imagenet/run.sh | 69 ++++++++++++++++++++++++++++++--- 3 files changed, 92 insertions(+), 11 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 168919a0..08b830d6 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -119,6 +119,16 @@ action="store_true", default=False, help="Enable this to allow pruning biases") +parser.add_argument('--subfolder', + default='results', + type=str, + metavar='PATH', + help='path to subfolder where we will store results and ckpts') +parser.add_argument('--optimizer', + default='sgd', + type=str, + metavar='OPT', + help='sgd|adam') best_acc1 = 0 @@ -231,7 +241,14 @@ def main_worker(gpu, ngpus_per_node, args): # define loss function (criterion), optimizer, and learning rate scheduler criterion = nn.CrossEntropyLoss().cuda(args.gpu) - optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.lr, + if args.optimizer == "adam": + print("Creating optimizer: Adam") + optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, + weight_decay=args.weight_decay) + + else args.optimizer == "sgd": + print("Creating optimizer: SGD") + optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) @@ -358,16 +375,16 @@ def main_worker(gpu, ngpus_per_node, args): best_acc1 = max(acc1, best_acc1) if not args.finetune: - results_filename = "acc_and_sparsity.csv" + results_filename = "{}/acc_and_sparsity.csv".format(args.subfolder) else: - results_filename = "acc_and_sparsity_finetune.csv" + results_filename = "{}/acc_and_sparsity_finetune.csv".format(args.subfolder) if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0): results_df.to_csv(results_filename, index=False) save_flag = ((epoch+1)%10 == 0) or (epoch > 85) or (epoch == args.epochs-1) if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0)): - torch.save(model.module.state_dict(), 'model_before_finetune_epoch_{}.pth'.format(epoch)) + torch.save(model.module.state_dict(), '{}/model_before_finetune_epoch_{}.pth'.format(args.subfolder, epoch)) # save_checkpoint({ # 'epoch': epoch + 1, # 'arch': args.arch, @@ -376,6 +393,8 @@ def main_worker(gpu, ngpus_per_node, args): # 'optimizer' : optimizer.state_dict(), # 'scheduler' : scheduler.state_dict() # }, is_best) + if args.finetune: + torch.save(model.module.state_dict(), '{}/model_after_finetune_epoch_{}.pth'.format(args.subfolder, epoch)) def train(train_loader, model, criterion, optimizer, epoch, args, scaler=None): diff --git a/imagenet/imagenet_main_cifar.py b/imagenet/imagenet_main_cifar.py index 7811553a..4495038c 100644 --- a/imagenet/imagenet_main_cifar.py +++ b/imagenet/imagenet_main_cifar.py @@ -120,6 +120,11 @@ action="store_true", default=False, help="Enable this to allow pruning biases") +parser.add_argument('--subfolder', + default='results', + type=str, + metavar='PATH', + help='path to subfolder where we will store results and ckpts') best_acc1 = 0 @@ -420,7 +425,7 @@ def main_worker(gpu, ngpus_per_node, args): save_flag = ((epoch+1)%10 == 0) or (epoch > 85) or (epoch == args.epochs-1) if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0)): - torch.save(model.module.state_dict(), 'model_before_finetune_epoch_{}.pth'.format(epoch)) + torch.save(model.module.state_dict(), '{}/model_before_finetune_epoch_{}.pth'.format(args.subfolder, epoch)) # save_checkpoint({ # 'epoch': epoch + 1, # 'arch': args.arch, diff --git a/imagenet/run.sh b/imagenet/run.sh index e1948d48..19994dfe 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -4,20 +4,77 @@ python imagenet_main.py \ --arch ResNet50 \ --rank 0 \ - --batch-size 1024 \ + --batch-size 512 \ --workers 8 \ --mixed-precision \ - --epochs 88 \ - --lr 0.04 \ - --target-sparsity 5 \ + --epochs 10 \ + --lr 0.02 \ + --target-sparsity 30 \ + --iter-period 5 \ + --lmbda 0.0000000001 \ + --subfolder results_reg_1e-10_lr_0.02 \ + --data '/data/imagenet/' \ + --dist-url 'tcp://127.0.0.1:2500' \ + --dist-backend 'nccl' \ + --multiprocessing-distributed \ + --world-size 1 + +python imagenet_main.py \ + --arch ResNet50 \ + --rank 0 \ + --batch-size 512 \ + --workers 8 \ + --mixed-precision \ + --epochs 10 \ + --lr 0.01 \ + --target-sparsity 30 \ --iter-period 5 \ --lmbda 0.0000000000 \ - --data '/home/ubuntu/ILSVRC2012' \ + --subfolder results_reg_0_lr_0.01 \ + --data '/data/imagenet/' \ --dist-url 'tcp://127.0.0.1:2500' \ --dist-backend 'nccl' \ --multiprocessing-distributed \ --world-size 1 +python imagenet_main.py \ + --arch ResNet50 \ + --rank 0 \ + --batch-size 512 \ + --workers 8 \ + --mixed-precision \ + --epochs 10 \ + --lr 0.01 \ + --target-sparsity 30 \ + --iter-period 5 \ + --lmbda 0.0000000001 \ + --subfolder results_reg_1e-10_lr_0.01 \ + --data '/data/imagenet/' \ + --dist-url 'tcp://127.0.0.1:2500' \ + --dist-backend 'nccl' \ + --multiprocessing-distributed \ + --world-size 1 + +python imagenet_main.py \ + --arch ResNet50 \ + --rank 0 \ + --batch-size 512 \ + --workers 8 \ + --mixed-precision \ + --epochs 10 \ + --lr 0.005 \ + --target-sparsity 30 \ + --iter-period 5 \ + --lmbda 0.0000000001 \ + --optimizer adam \ + --subfolder results_reg_1e-10_lr_0.02_adam \ + --data '/data/imagenet/' \ + --dist-url 'tcp://127.0.0.1:2500' \ + --dist-backend 'nccl' \ + --multiprocessing-distributed \ + --world-size 1 + +:< Date: Sun, 5 Jun 2022 22:38:12 -0500 Subject: [PATCH 110/113] minor typo with opt --- imagenet/imagenet_main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 08b830d6..474f41a3 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -245,8 +245,7 @@ def main_worker(gpu, ngpus_per_node, args): print("Creating optimizer: Adam") optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.weight_decay) - - else args.optimizer == "sgd": + else: print("Creating optimizer: SGD") optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.lr, momentum=args.momentum, From f6116fe18704f42196ae25e52297c688d7ae621a Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 6 Jun 2022 01:25:51 -0500 Subject: [PATCH 111/113] adding mkdir for subfolder --- imagenet/imagenet_main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/imagenet/imagenet_main.py b/imagenet/imagenet_main.py index 474f41a3..d7f25ea7 100644 --- a/imagenet/imagenet_main.py +++ b/imagenet/imagenet_main.py @@ -156,6 +156,9 @@ def main(): args.distributed = args.world_size > 1 or args.multiprocessing_distributed + if not os.path.isdir(args.subfolder): + os.mkdir(args.subfolder) + ngpus_per_node = torch.cuda.device_count() if args.multiprocessing_distributed: # Since we have ngpus_per_node processes per node, the total world_size From e1c68d3a3f7851be22708db409a6111c3e40c5d5 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 6 Jun 2022 22:54:22 -0500 Subject: [PATCH 112/113] updating run script --- imagenet/run.sh | 71 +++++-------------------------------------------- 1 file changed, 7 insertions(+), 64 deletions(-) diff --git a/imagenet/run.sh b/imagenet/run.sh index 19994dfe..ea1e0770 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -7,74 +7,18 @@ python imagenet_main.py \ --batch-size 512 \ --workers 8 \ --mixed-precision \ - --epochs 10 \ + --epochs 88 \ --lr 0.02 \ - --target-sparsity 30 \ - --iter-period 5 \ + --target-sparsity 5 \ + --iter-period 8 \ --lmbda 0.0000000001 \ - --subfolder results_reg_1e-10_lr_0.02 \ - --data '/data/imagenet/' \ - --dist-url 'tcp://127.0.0.1:2500' \ - --dist-backend 'nccl' \ - --multiprocessing-distributed \ - --world-size 1 - -python imagenet_main.py \ - --arch ResNet50 \ - --rank 0 \ - --batch-size 512 \ - --workers 8 \ - --mixed-precision \ - --epochs 10 \ - --lr 0.01 \ - --target-sparsity 30 \ - --iter-period 5 \ - --lmbda 0.0000000000 \ - --subfolder results_reg_0_lr_0.01 \ + --subfolder results_reg_1e-10_lr_0.02_iter_8_fullrun \ --data '/data/imagenet/' \ --dist-url 'tcp://127.0.0.1:2500' \ --dist-backend 'nccl' \ --multiprocessing-distributed \ --world-size 1 -python imagenet_main.py \ - --arch ResNet50 \ - --rank 0 \ - --batch-size 512 \ - --workers 8 \ - --mixed-precision \ - --epochs 10 \ - --lr 0.01 \ - --target-sparsity 30 \ - --iter-period 5 \ - --lmbda 0.0000000001 \ - --subfolder results_reg_1e-10_lr_0.01 \ - --data '/data/imagenet/' \ - --dist-url 'tcp://127.0.0.1:2500' \ - --dist-backend 'nccl' \ - --multiprocessing-distributed \ - --world-size 1 - -python imagenet_main.py \ - --arch ResNet50 \ - --rank 0 \ - --batch-size 512 \ - --workers 8 \ - --mixed-precision \ - --epochs 10 \ - --lr 0.005 \ - --target-sparsity 30 \ - --iter-period 5 \ - --lmbda 0.0000000001 \ - --optimizer adam \ - --subfolder results_reg_1e-10_lr_0.02_adam \ - --data '/data/imagenet/' \ - --dist-url 'tcp://127.0.0.1:2500' \ - --dist-backend 'nccl' \ - --multiprocessing-distributed \ - --world-size 1 - -:< Date: Fri, 17 Jun 2022 23:42:19 -0500 Subject: [PATCH 113/113] minor changes. for cifar10 testing i think --- imagenet/imagenet_main_cifar.py | 6 +-- imagenet/run.sh | 77 +++++---------------------------- imagenet/run_cifar.sh | 13 +++--- 3 files changed, 22 insertions(+), 74 deletions(-) diff --git a/imagenet/imagenet_main_cifar.py b/imagenet/imagenet_main_cifar.py index 4495038c..e167803b 100644 --- a/imagenet/imagenet_main_cifar.py +++ b/imagenet/imagenet_main_cifar.py @@ -416,14 +416,14 @@ def main_worker(gpu, ngpus_per_node, args): best_acc1 = max(acc1, best_acc1) if not args.finetune: - results_filename = "acc_and_sparsity.csv" + results_filename = "{}/acc_and_sparsity.csv".format(args.subfolder) else: - results_filename = "acc_and_sparsity_finetune.csv" + results_filename = "{}/acc_and_sparsity_finetune.csv".format(args.subfolder) if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0): results_df.to_csv(results_filename, index=False) - save_flag = ((epoch+1)%10 == 0) or (epoch > 85) or (epoch == args.epochs-1) + save_flag = ((epoch+1)%10 == 0) or (epoch > 140) or (epoch == args.epochs-1) if save_flag and (not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank == 0)): torch.save(model.module.state_dict(), '{}/model_before_finetune_epoch_{}.pth'.format(args.subfolder, epoch)) # save_checkpoint({ diff --git a/imagenet/run.sh b/imagenet/run.sh index 19994dfe..a914f484 100644 --- a/imagenet/run.sh +++ b/imagenet/run.sh @@ -1,80 +1,25 @@ #!/bin/bash -#:<