From 40933443a0fba5fb19e622dab0b00e08df62c8b9 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 13 Dec 2021 04:19:27 +0000 Subject: [PATCH 01/16] config file for adding finetune_loss --- config1.yml | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 config1.yml diff --git a/config1.yml b/config1.yml new file mode 100644 index 00000000..9d69c844 --- /dev/null +++ b/config1.yml @@ -0,0 +1,62 @@ +subfolder: quantize_0_5_finetune_loss_test +lam_finetune_loss: 100 +num_step_finetune: 10 + +# Hypercube optimization +algo: 'hc_iter' +# effectively prune once in the entire process +iter_period: 100 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +prune_rate: 0.982 #0.995 +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0 #0.00005 # 0.01 #0.0001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: False From adee4a211acc48fafd722796f0fd440a0e5837e4 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 17 Dec 2021 06:18:07 +0000 Subject: [PATCH 02/16] debugging --- cifar_exec_GD.sh | 4 ++-- config1.yml | 2 +- trainers/default.py | 30 ++++++++++++++++++------------ 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index b87c32b6..e08d7f78 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -14,11 +14,11 @@ ### ResNet-20 #python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml -python main.py --config config_current_best.yml --run_idx 1 +#python main.py --config config_current_best.yml --run_idx 1 ## testing adding finetune loss -#python main.py --config config1.yml --run_idx 1 # +python main.py --config config1.yml --run_idx 1 # #python main.py --config config2.yml --run_idx 2 diff --git a/config1.yml b/config1.yml index 9d69c844..1bc13079 100644 --- a/config1.yml +++ b/config1.yml @@ -53,7 +53,7 @@ lmbda: 0 #0.00005 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # workers: 4 -gpu: 3 +gpu: 1 # ===== Checkpointing ===== # checkpoint_at_prune: True diff --git a/trainers/default.py b/trainers/default.py index c0848ff9..e550933e 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -8,6 +8,8 @@ from utils.logging import AverageMeter, ProgressMeter from utils.net_utils import get_regularization_loss, prune, get_layers +from main_utils import get_model + from torch import optim __all__ = ["train", "validate", "modifier"] @@ -62,7 +64,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): if args.lam_finetune_loss > 0: - model2 = copy.deepcopy(model) + #model2 = copy.deepcopy(model) + model2 = get_model(args) #delta_list = {} conv, linear = get_layers(arch=args.arch, model=model) conv2, linear2 = get_layers(arch=args.arch, model=model2) @@ -73,11 +76,12 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): # clone parameters from model for m_from, m_to in zip(layer_list, layer_list2):#model.modules(), model2.modules()): m_to.scores.data = m_from.scores.data.clone() - m_to.flag.data = m_from.flag.data.clone() + m_to.flag.data = m_from.flag.data + m_to.weight.data = m_from.weight.data # turn on gradient for weight, turn off gradient for mask for name, params in model2.named_parameters(): - print(name) + #print(name) if "weight" in name: params.requires_grad=True elif "score" in name: @@ -92,8 +96,10 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): loss_updated_model.backward()#retain_graph=True) meta_optimizer.step() + + # go back to original setting - for name, params in model.named_parameters(): + for name, params in model2.named_parameters(): if "weight" in name: params.requires_grad=False elif "score" in name: @@ -116,10 +122,10 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): ''' finetune_loss = args.lam_finetune_loss * loss_updated_model - print('original loss: ', loss) - print('finetune loss: ', finetune_loss) + #print('original loss: ', loss) + #print('finetune loss: ', finetune_loss) - + #''' print('For model2') for name, params in model2.named_parameters(): if params.requires_grad: @@ -131,14 +137,14 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): for name, params in model.named_parameters(): if params.requires_grad: #pdb.set_trace() - print(name) - #grad =torch.autograd.grad(finetune_loss, params, retain_graph=True, allow_unused=True)[0].data - #print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) - + #print(name) + grad =torch.autograd.grad(loss, params, retain_graph=True)[0].data + print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) + #''' loss += finetune_loss - pdb.set_trace() + #pdb.set_trace() regularization_loss = torch.tensor(0) if args.regularization: From 4c8c01ec36318a1be8df714e7a8b3a5e8e1cda98 Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Fri, 17 Dec 2021 16:19:31 -0600 Subject: [PATCH 03/16] add weight_ft to use finetune_loss --- trainers/default.py | 386 ++++++++++++++++++++------------------------ utils/conv_type.py | 16 +- 2 files changed, 188 insertions(+), 214 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index 50ada91a..d6396cb1 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -14,230 +14,194 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): - batch_time = AverageMeter("Time", ":6.3f") - data_time = AverageMeter("Data", ":6.3f") - losses = AverageMeter("Loss", ":.3f") - top1 = AverageMeter("Acc@1", ":6.2f") - top5 = AverageMeter("Acc@5", ":6.2f") - top10 = AverageMeter("Acc@10", ":6.2f") - progress = ProgressMeter( - len(train_loader), - [batch_time, data_time, losses, top1, top5], - prefix=f"Epoch: [{epoch}]", - ) - - # switch to train mode - model.train() - - batch_size = train_loader.batch_size - num_batches = len(train_loader) - end = time.time() - for i, (images, target) in tqdm.tqdm( - enumerate(train_loader), ascii=True, total=len(train_loader) - ): - # measure data loading time - data_time.update(time.time() - end) - - if args.gpu is not None: - images = images.cuda(args.gpu, non_blocking=True) - - target = target.cuda(args.gpu, non_blocking=True) - - # update score thresholds for global ep - if args.algo in ['global_ep', 'global_ep_iter']: - prune(model, update_thresholds_only=True) - - # compute output - output = model(images) - - if args.algo in ['hc', 'hc_iter', 'pt'] and i % args.project_freq == 0 and not args.differentiate_clamp: - for name, params in model.named_parameters(): - if "score" in name: - scores = params - with torch.no_grad(): - scores.data = torch.clamp(scores.data, 0.0, 1.0) - - loss = criterion(output, target) - - - if args.lam_finetune_loss > 0: - - model2 = copy.deepcopy(model) - #delta_list = {} - conv, linear = get_layers(arch=args.arch, model=model) - conv2, linear2 = get_layers(arch=args.arch, model=model2) - layer_list = [*conv, *linear] - layer_list2 = [*conv2, *linear2] - - #pdb.set_trace() - # clone parameters from model - for m_from, m_to in zip(layer_list, layer_list2):#model.modules(), model2.modules()): - m_to.scores.data = m_from.scores.data.clone() - m_to.flag.data = m_from.flag.data.clone() - - # turn on gradient for weight, turn off gradient for mask - for name, params in model2.named_parameters(): - print(name) - if "weight" in name: - params.requires_grad=True - elif "score" in name: - params.requires_grad=False - - meta_optimizer = optim.SGD(model2.parameters(), lr=0.1, momentum=0.9) - # update for several steps - for i in range(args.num_step_finetune): - # forward and backward to update net_pi grad. - loss_updated_model = criterion(model2(images), target) - meta_optimizer.zero_grad() - loss_updated_model.backward()#retain_graph=True) - meta_optimizer.step() - - # go back to original setting - for name, params in model.named_parameters(): - if "weight" in name: - params.requires_grad=False - elif "score" in name: - params.requires_grad=True - - loss_updated_model = criterion(model2(images), target) - - - ''' - # define updates - for i, (name, params) in enumerate(model2.named_parameters()): - if "weight" in name: - delta_list[i] = torch.autograd.grad(loss, params) - - model3 = clone(model) - for i, (name, params) in enumerate(model3.named_parameters()): - if "weight" in name: - params -= alpha * delta_list[i] - output3 = model3(images) - ''' - - finetune_loss = args.lam_finetune_loss * loss_updated_model - print('original loss: ', loss) - print('finetune loss: ', finetune_loss) - - - print('For model2') - for name, params in model2.named_parameters(): - if params.requires_grad: - #pdb.set_trace() - grad =torch.autograd.grad(finetune_loss, params, retain_graph=True)[0].data - print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) - - print('For model') - for name, params in model.named_parameters(): - if params.requires_grad: - #pdb.set_trace() - print(name) - #grad =torch.autograd.grad(finetune_loss, params, retain_graph=True, allow_unused=True)[0].data - #print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) - - - loss += finetune_loss - - pdb.set_trace() - - regularization_loss = torch.tensor(0) - if args.regularization: - regularization_loss =\ - get_regularization_loss(model, regularizer=args.regularization, - lmbda=args.lmbda, alpha=args.alpha, - alpha_prime=args.alpha_prime) - - #print('regularization_loss: ', regularization_loss) - loss += regularization_loss - - # measure accuracy and record loss - acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - losses.update(loss.item(), images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - top10.update(acc10.item(), images.size(0)) - - # compute gradient and do SGD step - optimizer.zero_grad() - loss.backward() - optimizer.step() - - pdb.set_trace() - # TODO: print the updated score - # TODO: print the weight - - # measure elapsed time - batch_time.update(time.time() - end) - end = time.time() - - if i % args.print_freq == 0: - t = (num_batches * epoch + i) * batch_size - progress.display(i) - progress.write_to_tensorboard(writer, prefix="train", global_step=t) - - # before completing training, clean up model based on latest scores - # update score thresholds for global ep - if args.algo in ['global_ep', 'global_ep_iter']: - prune(model, update_thresholds_only=True) - if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: - for name, params in model.named_parameters(): - if "score" in name: - scores = params - with torch.no_grad(): - scores.data = torch.clamp(scores.data, 0.0, 1.0) - - return top1.avg, top5.avg, top10.avg, regularization_loss.item() + batch_time = AverageMeter("Time", ":6.3f") + data_time = AverageMeter("Data", ":6.3f") + losses = AverageMeter("Loss", ":.3f") + top1 = AverageMeter("Acc@1", ":6.2f") + top5 = AverageMeter("Acc@5", ":6.2f") + top10 = AverageMeter("Acc@10", ":6.2f") + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix=f"Epoch: [{epoch}]", + ) + + # switch to train mode + model.train() + + batch_size = train_loader.batch_size + num_batches = len(train_loader) + end = time.time() + for i, (images, target) in tqdm.tqdm( + enumerate(train_loader), ascii=True, total=len(train_loader) + ): + # measure data loading time + data_time.update(time.time() - end) + + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + + target = target.cuda(args.gpu, non_blocking=True) + + # update score thresholds for global ep + if args.algo in ['global_ep', 'global_ep_iter']: + prune(model, update_thresholds_only=True) + + # compute output + output = model(images) + + if args.algo in ['hc', 'hc_iter', 'pt'] and i % args.project_freq == 0 and not args.differentiate_clamp: + for name, params in model.named_parameters(): + if "score" in name: + scores = params + with torch.no_grad(): + scores.data = torch.clamp(scores.data, 0.0, 1.0) + + loss = criterion(output, target) + + + if args.lam_finetune_loss > 0: + + model2 = copy.deepcopy(model) + # turn on gradient for weight, turn off gradient for mask + for name, params in model2.named_parameters(): + print(name) + if "weight" in name: + params.requires_grad=True + elif "score" in name: + params.requires_grad=False + + meta_optimizer = optim.SGD(model2.parameters(), lr=0.1, momentum=0.9) + # update for several steps + for i in range(args.num_step_finetune): + loss_updated_model = criterion(model2(images), target) + meta_optimizer.zero_grad() + loss_updated_model.backward()#retain_graph=True) + meta_optimizer.step() + + + conv, linear = get_layers(arch=args.arch, model=model) + conv2, linear2 = get_layers(arch=args.arch, model=model2) + layer_list = [*conv, *linear] + layer_list2 = [*conv2, *linear2] + + # load updated weight from model2 to model + for m_from, m_to in zip(layer_list2, layer_list): + m_to.weight_ft.data = m_from.weight.data + + # compute loss on the finetuned weights + finetune_loss = args.lam_finetune_loss * criterion(model(images, finetune=True), data) + print('original loss: ', loss) + print('finetune loss: ', finetune_loss) + + # print('For model') + # for name, params in model.named_parameters(): + # if params.requires_grad: + # print(name) + # grad =torch.autograd.grad(finetune_loss, params, retain_graph=True)[0].data + # print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) + + + loss += finetune_loss + + + regularization_loss = torch.tensor(0) + if args.regularization: + regularization_loss =\ + get_regularization_loss(model, regularizer=args.regularization, + lmbda=args.lmbda, alpha=args.alpha, + alpha_prime=args.alpha_prime) + + loss += regularization_loss + + # measure accuracy and record loss + acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + top10.update(acc10.item(), images.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + loss.backward() + optimizer.step() + + #pdb.set_trace() + # TODO: print the updated score + # TODO: print the weight + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + t = (num_batches * epoch + i) * batch_size + progress.display(i) + progress.write_to_tensorboard(writer, prefix="train", global_step=t) + + # before completing training, clean up model based on latest scores + # update score thresholds for global ep + if args.algo in ['global_ep', 'global_ep_iter']: + prune(model, update_thresholds_only=True) + if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: + for name, params in model.named_parameters(): + if "score" in name: + scores = params + with torch.no_grad(): + scores.data = torch.clamp(scores.data, 0.0, 1.0) + + return top1.avg, top5.avg, top10.avg, regularization_loss.item() def validate(val_loader, model, criterion, args, writer, epoch): - batch_time = AverageMeter("Time", ":6.3f", write_val=False) - losses = AverageMeter("Loss", ":.3f", write_val=False) - top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) - top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) - top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) - progress = ProgressMeter( - len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " - ) + batch_time = AverageMeter("Time", ":6.3f", write_val=False) + losses = AverageMeter("Loss", ":.3f", write_val=False) + top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) + top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) + top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) + progress = ProgressMeter( + len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " + ) - # switch to evaluate mode - model.eval() + # switch to evaluate mode + model.eval() - with torch.no_grad(): - end = time.time() - for i, (images, target) in tqdm.tqdm( - enumerate(val_loader), ascii=True, total=len(val_loader) - ): - if args.gpu is not None: - images = images.cuda(args.gpu, non_blocking=True) + with torch.no_grad(): + end = time.time() + for i, (images, target) in tqdm.tqdm( + enumerate(val_loader), ascii=True, total=len(val_loader) + ): + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) - target = target.cuda(args.gpu, non_blocking=True) + target = target.cuda(args.gpu, non_blocking=True) - # compute output - output = model(images) + # compute output + output = model(images) - loss = criterion(output, target) + loss = criterion(output, target) - # measure accuracy and record loss - acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - losses.update(loss.item(), images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - top10.update(acc10.item(), images.size(0)) + # measure accuracy and record loss + acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + top10.update(acc10.item(), images.size(0)) - # measure elapsed time - batch_time.update(time.time() - end) - end = time.time() + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() - if i % args.print_freq == 0: - progress.display(i) + if i % args.print_freq == 0: + progress.display(i) - progress.display(len(val_loader)) + progress.display(len(val_loader)) - if writer is not None: - progress.write_to_tensorboard(writer, prefix="test", global_step=epoch) + if writer is not None: + progress.write_to_tensorboard(writer, prefix="test", global_step=epoch) - print("Model top1 Accuracy: {}".format(top1.avg)) - return top1.avg, top5.avg, top10.avg + print("Model top1 Accuracy: {}".format(top1.avg)) + return top1.avg, top5.avg, top10.avg def modifier(args, epoch, model): - return + return diff --git a/utils/conv_type.py b/utils/conv_type.py index d58b67fc..ca668e93 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -115,6 +115,9 @@ def __init__(self, *args, **kwargs): # dummy variable just so other things don't break self.bias_scores = nn.Parameter(torch.Tensor(1)) + # storage for finetuned weights + self.weight_ft = nn.Parameter(torch.zeros(self.weight.size())) + # prune scores below this for global EP in bottom-k self.scores_prune_threshold = -np.inf self.bias_scores_prune_threshold = -np.inf @@ -150,6 +153,7 @@ def __init__(self, *args, **kwargs): if parser_args.freeze_weights: # NOTE: turn the gradient on the weights off self.weight.requires_grad = False + self.weight_ft.requires_grad = False self.flag.requires_grad = False self.bias_flag.requires_grad = False if parser_args.bias: @@ -166,7 +170,7 @@ def set_prune_rate(self, prune_rate): def clamped_scores(self): return self.scores.abs() - def forward(self, x): + def forward(self, x, finetuned=False): if parser_args.algo in ['hc', 'hc_iter']: # don't need a mask here. the scores are directly multiplied with weights if parser_args.differentiate_clamp: @@ -191,10 +195,16 @@ def forward(self, x): if parser_args.algo in ['imp']: # no STE, no subnet. Mask is handled outside - w = self.weight + if finetuned: + w = self.weight + else: + w = self.weight b = self.bias else: - w = self.weight * subnet + if finetuned: + w = self.weight_ft * subnet + else: + w = self.weight * subnet if parser_args.bias: b = self.bias * bias_subnet else: From dbb23f4f449de08c4d19b2846a4d72099061641a Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Fri, 17 Dec 2021 16:28:51 -0600 Subject: [PATCH 04/16] re-indent --- trainers/default.py | 188 ++++++++++++++++++++++---------------------- 1 file changed, 93 insertions(+), 95 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index fad62abb..08bf0d05 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -17,52 +17,50 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): + batch_time = AverageMeter("Time", ":6.3f") + data_time = AverageMeter("Data", ":6.3f") + losses = AverageMeter("Loss", ":.3f") + top1 = AverageMeter("Acc@1", ":6.2f") + top5 = AverageMeter("Acc@5", ":6.2f") + top10 = AverageMeter("Acc@10", ":6.2f") + progress = ProgressMeter( + len(train_loader), + [batch_time, data_time, losses, top1, top5], + prefix=f"Epoch: [{epoch}]", + ) + + # switch to train mode + model.train() + + batch_size = train_loader.batch_size + num_batches = len(train_loader) + end = time.time() + for i, (images, target) in tqdm.tqdm( + enumerate(train_loader), ascii=True, total=len(train_loader) + ): + # measure data loading time + data_time.update(time.time() - end) + + if args.gpu is not None: + images = images.cuda(args.gpu, non_blocking=True) + + target = target.cuda(args.gpu, non_blocking=True) + # update score thresholds for global ep + if args.algo in ['global_ep', 'global_ep_iter']: + prune(model, update_thresholds_only=True) - batch_time = AverageMeter("Time", ":6.3f") - data_time = AverageMeter("Data", ":6.3f") - losses = AverageMeter("Loss", ":.3f") - top1 = AverageMeter("Acc@1", ":6.2f") - top5 = AverageMeter("Acc@5", ":6.2f") - top10 = AverageMeter("Acc@10", ":6.2f") - progress = ProgressMeter( - len(train_loader), - [batch_time, data_time, losses, top1, top5], - prefix=f"Epoch: [{epoch}]", - ) - - # switch to train mode - model.train() - - batch_size = train_loader.batch_size - num_batches = len(train_loader) - end = time.time() - for i, (images, target) in tqdm.tqdm( - enumerate(train_loader), ascii=True, total=len(train_loader) - ): - # measure data loading time - data_time.update(time.time() - end) - - if args.gpu is not None: - images = images.cuda(args.gpu, non_blocking=True) - - target = target.cuda(args.gpu, non_blocking=True) - - # update score thresholds for global ep - if args.algo in ['global_ep', 'global_ep_iter']: - prune(model, update_thresholds_only=True) - - # compute output - output = model(images) - - if args.algo in ['hc', 'hc_iter', 'pt'] and i % args.project_freq == 0 and not args.differentiate_clamp: - for name, params in model.named_parameters(): - if "score" in name: - scores = params - with torch.no_grad(): - scores.data = torch.clamp(scores.data, 0.0, 1.0) - - loss = criterion(output, target) + # compute output + output = model(images) + + if args.algo in ['hc', 'hc_iter', 'pt'] and i % args.project_freq == 0 and not args.differentiate_clamp: + for name, params in model.named_parameters(): + if "score" in name: + scores = params + with torch.no_grad(): + scores.data = torch.clamp(scores.data, 0.0, 1.0) + + loss = criterion(output, target) if args.lam_finetune_loss > 0: @@ -101,61 +99,61 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): # print('For model') # for name, params in model.named_parameters(): - # if params.requires_grad: - # print(name) - # grad =torch.autograd.grad(finetune_loss, params, retain_graph=True)[0].data - # print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) + # if params.requires_grad: + # print(name) + # grad =torch.autograd.grad(finetune_loss, params, retain_graph=True)[0].data + # print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) loss += finetune_loss - regularization_loss = torch.tensor(0) - if args.regularization: - regularization_loss =\ - get_regularization_loss(model, regularizer=args.regularization, - lmbda=args.lmbda, alpha=args.alpha, - alpha_prime=args.alpha_prime) - - #print('regularization_loss: ', regularization_loss) - loss += regularization_loss - - # measure accuracy and record loss - acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - losses.update(loss.item(), images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - top10.update(acc10.item(), images.size(0)) - - # compute gradient and do SGD step - optimizer.zero_grad() - loss.backward() - optimizer.step() - - #pdb.set_trace() - # TODO: print the updated score - # TODO: print the weight - - # measure elapsed time - batch_time.update(time.time() - end) - end = time.time() - - if i % args.print_freq == 0: - t = (num_batches * epoch + i) * batch_size - progress.display(i) - progress.write_to_tensorboard(writer, prefix="train", global_step=t) - - # before completing training, clean up model based on latest scores - # update score thresholds for global ep - if args.algo in ['global_ep', 'global_ep_iter']: - prune(model, update_thresholds_only=True) - if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: - for name, params in model.named_parameters(): - if "score" in name: - scores = params - with torch.no_grad(): - scores.data = torch.clamp(scores.data, 0.0, 1.0) - - return top1.avg, top5.avg, top10.avg, regularization_loss.item() + regularization_loss = torch.tensor(0) + if args.regularization: + regularization_loss =\ + get_regularization_loss(model, regularizer=args.regularization, + lmbda=args.lmbda, alpha=args.alpha, + alpha_prime=args.alpha_prime) + + #print('regularization_loss: ', regularization_loss) + loss += regularization_loss + + # measure accuracy and record loss + acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + top10.update(acc10.item(), images.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + loss.backward() + optimizer.step() + + #pdb.set_trace() + # TODO: print the updated score + # TODO: print the weight + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + t = (num_batches * epoch + i) * batch_size + progress.display(i) + progress.write_to_tensorboard(writer, prefix="train", global_step=t) + + # before completing training, clean up model based on latest scores + # update score thresholds for global ep + if args.algo in ['global_ep', 'global_ep_iter']: + prune(model, update_thresholds_only=True) + if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: + for name, params in model.named_parameters(): + if "score" in name: + scores = params + with torch.no_grad(): + scores.data = torch.clamp(scores.data, 0.0, 1.0) + + return top1.avg, top5.avg, top10.avg, regularization_loss.item() def validate(val_loader, model, criterion, args, writer, epoch): From 7eebe3470f6c343f75a3f432309fda4d60db40d0 Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Fri, 17 Dec 2021 16:33:20 -0600 Subject: [PATCH 05/16] change finetuned as parser_args --- trainers/default.py | 4 +++- utils/conv_type.py | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index 08bf0d05..22ff18be 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -93,7 +93,9 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): m_to.weight_ft.data = m_from.weight.data # compute loss on the finetuned weights - finetune_loss = args.lam_finetune_loss * criterion(model(images, finetune=True), data) + parser_args.finetuned = True + finetune_loss = args.lam_finetune_loss * criterion(model(images), data) + parser_args.finetuned = False print('original loss: ', loss) print('finetune loss: ', finetune_loss) diff --git a/utils/conv_type.py b/utils/conv_type.py index ca668e93..4b931fab 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -170,7 +170,7 @@ def set_prune_rate(self, prune_rate): def clamped_scores(self): return self.scores.abs() - def forward(self, x, finetuned=False): + def forward(self, x): if parser_args.algo in ['hc', 'hc_iter']: # don't need a mask here. the scores are directly multiplied with weights if parser_args.differentiate_clamp: @@ -195,13 +195,14 @@ def forward(self, x, finetuned=False): if parser_args.algo in ['imp']: # no STE, no subnet. Mask is handled outside - if finetuned: - w = self.weight + if parser_args.finetuned: + print("Are we sure we are using finetuned loss for imp?") + w = self.weight_ft else: w = self.weight b = self.bias else: - if finetuned: + if parser_args.finetuned: w = self.weight_ft * subnet else: w = self.weight * subnet From 7548ef68e696d4fdf45415cdb281645d90f6f045 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 19 Dec 2021 15:21:32 +0000 Subject: [PATCH 06/16] check the accuracy for 1.4% sparsity after adding finetune loss --- args_helper.py | 6 ++++++ cifar_exec_GD.sh | 16 +++++++++++++++- trainers/default.py | 8 ++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/args_helper.py b/args_helper.py index f5b0a487..c054fdbf 100644 --- a/args_helper.py +++ b/args_helper.py @@ -720,6 +720,12 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="chg weights before sanity check" ) + parser.add_argument( + "--finetuned", + action="store_true", + default=False, + help="used for finetuned loss (please set it false in the command line. The code will automatically turn on/off it" + ) parser.add_argument( "--fine-tune-optimizer", type=str, diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index e08d7f78..a45ff0a9 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -18,8 +18,22 @@ ## testing adding finetune loss -python main.py --config config1.yml --run_idx 1 # +#python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5.yml + + +#python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune_without_unflag.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_without_unflag.yml +python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5_without_unflag.yml + + + +#python main.py --config config1.yml --run_idx 1 # +#python main.py --config config1.yml --run_idx 0 # #python main.py --config config2.yml --run_idx 2 +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_10.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_10.yml diff --git a/trainers/default.py b/trainers/default.py index 22ff18be..788a937d 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -68,7 +68,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): model2 = copy.deepcopy(model) # turn on gradient for weight, turn off gradient for mask for name, params in model2.named_parameters(): - print(name) + #print(name) if "weight" in name: params.requires_grad=True elif "score" in name: @@ -93,9 +93,9 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): m_to.weight_ft.data = m_from.weight.data # compute loss on the finetuned weights - parser_args.finetuned = True - finetune_loss = args.lam_finetune_loss * criterion(model(images), data) - parser_args.finetuned = False + args.finetuned = True + finetune_loss = args.lam_finetune_loss * criterion(model(images), target) + args.finetuned = False print('original loss: ', loss) print('finetune loss: ', finetune_loss) From 3324724889053269fa3d38b6d2aae44deac44c4d Mon Sep 17 00:00:00 2001 From: root Date: Sun, 19 Dec 2021 15:52:35 +0000 Subject: [PATCH 07/16] minor change --- cifar_exec_GD.sh | 4 ++-- config1.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index a45ff0a9..9bff9c43 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -18,14 +18,14 @@ ## testing adding finetune loss -#python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml +python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml #python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5.yml #python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5.yml #python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune_without_unflag.yml #python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_without_unflag.yml -python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5_without_unflag.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5_without_unflag.yml diff --git a/config1.yml b/config1.yml index 1bc13079..05cee927 100644 --- a/config1.yml +++ b/config1.yml @@ -1,5 +1,5 @@ subfolder: quantize_0_5_finetune_loss_test -lam_finetune_loss: 100 +lam_finetune_loss: 0 #100 num_step_finetune: 10 # Hypercube optimization From 31e9980312c73c8809721bdd792361cde834b84f Mon Sep 17 00:00:00 2001 From: root Date: Sun, 19 Dec 2021 15:52:57 +0000 Subject: [PATCH 08/16] minor --- config1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config1.yml b/config1.yml index 05cee927..bd0c975f 100644 --- a/config1.yml +++ b/config1.yml @@ -53,7 +53,7 @@ lmbda: 0 #0.00005 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # workers: 4 -gpu: 1 +gpu: 2 # ===== Checkpointing ===== # checkpoint_at_prune: True From 48f7636a5fd5e3400ebd369dfdce3dea1810bbca Mon Sep 17 00:00:00 2001 From: root Date: Sun, 19 Dec 2021 16:15:13 +0000 Subject: [PATCH 09/16] reproduce --- cifar_exec_GD.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 9bff9c43..777545e4 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -18,7 +18,7 @@ ## testing adding finetune loss -python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml +python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml > log_no_finetune_5e-5 2>&1 #python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5.yml #python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5.yml From 17de71c1ff808107f49816dd295f9f971f708970 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 19 Dec 2021 20:49:18 +0000 Subject: [PATCH 10/16] adding finetune loss only at last 10 epochs --- args_helper.py | 6 ++++++ cifar_exec_GD.sh | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/args_helper.py b/args_helper.py index c054fdbf..36ec690e 100644 --- a/args_helper.py +++ b/args_helper.py @@ -720,6 +720,12 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="chg weights before sanity check" ) + parser.add_argument( + "--finetune_last_epochs", + action="store_true", + default=False, + help="used for adding finetune loss at last 10 epochs of HC" + ) parser.add_argument( "--finetuned", action="store_true", diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 777545e4..4f1ee839 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -18,7 +18,8 @@ ## testing adding finetune loss -python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml > log_no_finetune_5e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml > log_no_finetune_5e-5 2>&1 +python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs.yml #python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5.yml #python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5.yml From 9e17f72ac1b1f25b32385ff3c49721f82e1dad55 Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Sun, 19 Dec 2021 15:01:37 -0600 Subject: [PATCH 11/16] add finetune loss only for last 10 epochs; done --- trainers/default.py | 82 +++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index 788a937d..ea8a6ab3 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -64,50 +64,44 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): if args.lam_finetune_loss > 0: - - model2 = copy.deepcopy(model) - # turn on gradient for weight, turn off gradient for mask - for name, params in model2.named_parameters(): - #print(name) - if "weight" in name: - params.requires_grad=True - elif "score" in name: - params.requires_grad=False - - meta_optimizer = optim.SGD(model2.parameters(), lr=0.1, momentum=0.9) - # update for several steps - for i in range(args.num_step_finetune): - loss_updated_model = criterion(model2(images), target) - meta_optimizer.zero_grad() - loss_updated_model.backward()#retain_graph=True) - meta_optimizer.step() - - - conv, linear = get_layers(arch=args.arch, model=model) - conv2, linear2 = get_layers(arch=args.arch, model=model2) - layer_list = [*conv, *linear] - layer_list2 = [*conv2, *linear2] - - # load updated weight from model2 to model - for m_from, m_to in zip(layer_list2, layer_list): - m_to.weight_ft.data = m_from.weight.data - - # compute loss on the finetuned weights - args.finetuned = True - finetune_loss = args.lam_finetune_loss * criterion(model(images), target) - args.finetuned = False - print('original loss: ', loss) - print('finetune loss: ', finetune_loss) - - # print('For model') - # for name, params in model.named_parameters(): - # if params.requires_grad: - # print(name) - # grad =torch.autograd.grad(finetune_loss, params, retain_graph=True)[0].data - # print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) - - - loss += finetune_loss + if args.finetune_last_epochs and epoch < args.epochs - 10: + pass + else: + model2 = copy.deepcopy(model) + # turn on gradient for weight, turn off gradient for mask + for name, params in model2.named_parameters(): + #print(name) + if "weight" in name: + params.requires_grad=True + elif "score" in name: + params.requires_grad=False + + meta_optimizer = optim.SGD(model2.parameters(), lr=0.1, momentum=0.9) + # update for several steps + for i in range(args.num_step_finetune): + loss_updated_model = criterion(model2(images), target) + meta_optimizer.zero_grad() + loss_updated_model.backward()#retain_graph=True) + meta_optimizer.step() + + + conv, linear = get_layers(arch=args.arch, model=model) + conv2, linear2 = get_layers(arch=args.arch, model=model2) + layer_list = [*conv, *linear] + layer_list2 = [*conv2, *linear2] + + # load updated weight from model2 to model + for m_from, m_to in zip(layer_list2, layer_list): + m_to.weight_ft.data = m_from.weight.data + + # compute loss on the finetuned weights + args.finetuned = True + finetune_loss = args.lam_finetune_loss * criterion(model(images), target) + args.finetuned = False + print('original loss: ', loss) + print('finetune loss: ', finetune_loss) + + loss += finetune_loss regularization_loss = torch.tensor(0) if args.regularization: From 8e68e9703e189c0222da510f990f1528d204742c Mon Sep 17 00:00:00 2001 From: root Date: Mon, 20 Dec 2021 16:52:49 +0000 Subject: [PATCH 12/16] tested adding finetune loss only for the last 10 epochds --- cifar_exec_GD.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 4f1ee839..03a9f1a2 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -19,9 +19,10 @@ ## testing adding finetune loss #python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml > log_no_finetune_5e-5 2>&1 -python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs.yml -#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5.yml -#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs_without_unflag.yml > log_finetune_1e-4_lam_1_num_5_last_10_epochs_without_unflag 2>&1 +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs.yml > log_finetune_1e-4_lam_1_num_5_last_10_epochs 2>&1 +python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs_without_unflag.yml > log_finetune_5e-5_lam_1_num_5_last_10_epochs_without_unflag 2>&1 +python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs.yml > log_finetune_5e-5_lam_1_num_5_last_10_epochs 2>&1 #python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune_without_unflag.yml From 9213f59ebc2cf34187b74ed9acef471ea903048b Mon Sep 17 00:00:00 2001 From: root Date: Thu, 23 Dec 2021 05:15:53 +0000 Subject: [PATCH 13/16] checking finetune loss for dense model --- cifar_exec_GD.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 5c4c6a01..a33ce15d 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -12,7 +12,7 @@ ### ResNet-20 -#python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml > log_EP_sparsity_50 2>&1 +python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #> log_EP_sparsity_50 2>&1 #python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml #python main.py --config config_current_best.yml --run_idx 1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_1_4_highreg.yml @@ -32,6 +32,14 @@ #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag.yml > log_target_sparsity_50_without_flag_lam_0 2>&1 +## adding finetune for denser models +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag_with_finetune.yml > log_target_sparsity_5_without_unflag_with_finetune_lam_3e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune.yml > log_target_sparsity_20_without_unflag_with_finetune_lam_1e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag_with_finetune.yml > log_target_sparsity_50_without_unflag_with_finetune_lam_0 2>&1 + +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_5_without_unflag_with_finetune_last_10_epochs_lam_3e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_20_without_unflag_with_finetune_last_10_epochs_lam_1e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_50_without_unflag_with_finetune_last_10_epochs_lam_0 2>&1 ## testing adding finetune loss @@ -57,6 +65,11 @@ + + + + + #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1.yml --run_idx 1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1e-2.yml --run_idx 1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1e-4.yml --run_idx 2 From 67257a0d74ce1f562350f230e699a98bbcf194a7 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 30 Dec 2021 18:23:41 +0000 Subject: [PATCH 14/16] run for 20% target sparsity, with three options: no finetune loss, add finetune loss for all epochs, add finetune loss for last 10 epochs --- cifar_exec_GD.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index a33ce15d..d9c0b2d0 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -12,7 +12,7 @@ ### ResNet-20 -python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #> log_EP_sparsity_50 2>&1 +#python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #> log_EP_sparsity_50 2>&1 #python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml #python main.py --config config_current_best.yml --run_idx 1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_1_4_highreg.yml @@ -28,17 +28,17 @@ python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #> log_EP_sparsit #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20.yml > log_target_sparsity_20_lam_1e-5 2>&1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag.yml > log_target_sparsity_5_without_unflag_lam_3e-5 2>&1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag.yml > log_target_sparsity_20_without_unflag_lam_1e-5 2>&1 +python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag.yml > log_target_sparsity_20_without_unflag_lam_1e-5 2>&1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag.yml > log_target_sparsity_50_without_flag_lam_0 2>&1 ## adding finetune for denser models #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag_with_finetune.yml > log_target_sparsity_5_without_unflag_with_finetune_lam_3e-5 2>&1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune.yml > log_target_sparsity_20_without_unflag_with_finetune_lam_1e-5 2>&1 +python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune.yml > log_target_sparsity_20_without_unflag_with_finetune_lam_1e-5 2>&1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag_with_finetune.yml > log_target_sparsity_50_without_unflag_with_finetune_lam_0 2>&1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_5_without_unflag_with_finetune_last_10_epochs_lam_3e-5 2>&1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_20_without_unflag_with_finetune_last_10_epochs_lam_1e-5 2>&1 +python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_20_without_unflag_with_finetune_last_10_epochs_lam_1e-5 2>&1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_50_without_unflag_with_finetune_last_10_epochs_lam_0 2>&1 From fecb1684962854fa28e235c30a483cb283ec1a7d Mon Sep 17 00:00:00 2001 From: root Date: Sun, 2 Jan 2022 01:10:08 +0000 Subject: [PATCH 15/16] minor --- cifar_exec_GD.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 2272c867..7fbf82fa 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -32,10 +32,10 @@ python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0 BLOCK ## ran in finetune_loss repo -python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t1.yml > log_hc_sparsity_1_35_t1 2>&1 +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t1.yml > log_hc_sparsity_1_35_t1 2>&1 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t2.yml > log_hc_sparsity_1_35_t2 2>&1 -python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t3.yml > log_hc_sparsity_1_35_t3 2>&1 +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t3.yml > log_hc_sparsity_1_35_t3 2>&1 python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t4.yml > log_hc_sparsity_1_35_t4 2>&1 python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t5.yml > log_hc_sparsity_1_35_t5 2>&1 From 44f95e76e897f8d7052b540bb42c98fad448d5f0 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 2 Jan 2022 15:24:56 +0000 Subject: [PATCH 16/16] average for 5 trials --- cifar_exec_GD.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 7fbf82fa..9196afc8 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -36,8 +36,8 @@ BLOCK #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t2.yml > log_hc_sparsity_1_35_t2 2>&1 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t3.yml > log_hc_sparsity_1_35_t3 2>&1 -python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t4.yml > log_hc_sparsity_1_35_t4 2>&1 -python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t5.yml > log_hc_sparsity_1_35_t5 2>&1 +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t4.yml > log_hc_sparsity_1_35_t4 2>&1 +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t5.yml > log_hc_sparsity_1_35_t5 2>&1