diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..46d0e652 Binary files /dev/null and b/.DS_Store differ diff --git a/args_helper.py b/args_helper.py index f5b0a487..c9b166ec 100644 --- a/args_helper.py +++ b/args_helper.py @@ -245,6 +245,12 @@ def parse_arguments(self, jupyter_mode=False): type=int, help="warmup epochs for hypercube" ) + parser.add_argument( + "--toggle-warmup", + action='store_true', + default=False, + help='flag that decides if we add noise to the rounded p_i' + ) parser.add_argument( "--hc-period", default=1, diff --git a/cifar_exec.sh b/cifar_exec.sh index 51418739..166bd75e 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -25,7 +25,7 @@ BLOCK # target sparsity 0.5 python main.py \ ---config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_0_5.yml > cifar_log_target_0_5 2>&1 +--config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml #> cifar_log 2>&1 # EP diff --git a/configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov22_3pm.yml b/configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov22_3pm.yml index 485e62d1..ee3c24f0 100644 --- a/configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov22_3pm.yml +++ b/configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov22_3pm.yml @@ -40,7 +40,7 @@ hc_quantized: True quantize_threshold: 0.5 # ===== Regularization ===== # -regularization: L2 +regularization: L1 #lmbda: 0.000001 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # diff --git a/configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml b/configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml index f4273e10..d20b6731 100644 --- a/configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml +++ b/configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml @@ -1,6 +1,6 @@ # Hypercube optimization algo: 'hc_iter' -#iter_period: 8 #15 +iter_period: 8 #15 # Architecture arch: resnet20 @@ -12,10 +12,15 @@ name: resnet20_cifar10_hypercube_bottom_K_SGD # ===== Learning Rate Policy ======== # optimizer: sgd lr: 0.1 #0.01 -#lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr +lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr # ===== Network training config ===== # -epochs: 50 #150 +epochs: 30 #150 +unif_prune: False +PRs: .60,.50,.50,.05,.40,.40 +epoch_pr: 1,5,10,15,20,30,40 wd: 0.0 momentum: 0.9 batch_size: 128 @@ -25,9 +30,9 @@ conv_type: SubnetConv bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK -#prune_rate: 0.2 +prune_rate: 0.2 init: signed_constant -#score_init: skew #half #bimodal #skew # bern +score_init: unif #half #bimodal #skew # bern scale_fan: False #True # ===== Rounding ===== # @@ -40,8 +45,8 @@ hc_quantized: True quantize_threshold: 0.5 # ===== Regularization ===== # -#regularization: L2 -#lmbda: 0.000001 # 0.01 #0.0001 #0.000001 +regularization: L1 +lmbda: 0.00005 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # workers: 4 @@ -53,3 +58,8 @@ checkpoint_at_prune: True # ==== sanity check ==== # skip_sanity_checks: True +# ==== get sparsity quickly and train much ==== # +fast_sparse: True + +# ==== ==== # +finetune_standard: False \ No newline at end of file diff --git a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml index f94f3295..20495f7d 100644 --- a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml +++ b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml @@ -1,4 +1,4 @@ -subfolder: quantize_0_75 +subfolder: quantize_0_75_checking # Hypercube optimization algo: 'hc_iter' @@ -14,7 +14,7 @@ name: resnet20_quantized_iter_hc # ===== Learning Rate Policy ======== # optimizer: sgd -lr: 0.1 #0.01 +lr: 0.1 lr_policy: cosine_lr #constant_lr #multistep_lr fine_tune_lr: 0.01 fine_tune_lr_policy: multistep_lr @@ -31,7 +31,7 @@ bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK # enter target sparsity here -prune_rate: 0.995 +prune_rate: 0.982 init: signed_constant score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True @@ -51,10 +51,14 @@ lmbda: 0.00005 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # workers: 4 -gpu: 0 +gpu: 3 # ===== Checkpointing ===== # checkpoint_at_prune: True # ==== sanity check ==== # -skip_sanity_checks: False +skip_sanity_checks: True + +# ==== HC with warm-up ==== # +hc_warmup: 3 +toggle_warmup: True diff --git a/configs/hypercube/resnet20/resnet20_quantized_nonunif_iter_hc_0_5.yml b/configs/hypercube/resnet20/resnet20_quantized_nonunif_iter_hc_0_5.yml new file mode 100644 index 00000000..6c79734b --- /dev/null +++ b/configs/hypercube/resnet20/resnet20_quantized_nonunif_iter_hc_0_5.yml @@ -0,0 +1,67 @@ +subfolder: quantize_0_75_checking + +# Hypercube optimization +algo: 'hc_iter' +# effectively prune once in the entire process +iter_period: 100 +unif_prune: False + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +prune_rate: 0.982 +PRs: 0.5,0.5,0.5,0.5,0.288 +epoch_pr: 20,40,60,80,100,200 +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 0.01 #0.0001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True + +# ==== HC with warm-up ==== # +hc_warmup: 3 +toggle_warmup: False diff --git a/configs/hypercube/resnet20/test_warmup.yml b/configs/hypercube/resnet20/test_warmup.yml new file mode 100644 index 00000000..fea479c0 --- /dev/null +++ b/configs/hypercube/resnet20/test_warmup.yml @@ -0,0 +1,64 @@ +subfolder: quantize_0_75 + +# Hypercube optimization +algo: 'hc_iter' +# effectively prune once in the entire process +iter_period: 100 + +# Architecture +arch: TwoLayerFC + +# ===== Dataset ===== # +dataset: MNIST +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +prune_rate: 0.995 +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 0.01 #0.0001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: False + +# ==== HC with warm-up ==== # +hc_warmup: 3 +toggle_warmup: True diff --git a/main.py b/main.py index d685f90c..cd019e34 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +from yaml import parse from main_utils import * @@ -41,6 +42,12 @@ def main_worker(gpu, ngpus_per_node): if not os.path.isdir(result_root): os.mkdir(result_root) + + if not parser_args.unif_prune: + prune_rates = list(map(float, parser_args.PRs.split(','))) + prune_epochs = list(map(float, parser_args.epoch_pr.split(','))) + parser_args.prune_rate = prune_rates[0] + model = get_model(parser_args) ''' @@ -147,10 +154,19 @@ def main_worker(gpu, ngpus_per_node): validation_time.update((time.time() - start_validation) / 60) # prune the model every T_{prune} epochs - if parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: + if parser_args.algo in ['hc_iter', 'global_ep_iter'] and parser_args.unif_prune and epoch % (parser_args.iter_period) == 0 and epoch != 0: prune(model) if parser_args.checkpoint_at_prune: save_checkpoint_at_prune(model, parser_args) + + #prune model non-uniformly + if not parser_args.unif_prune: + if epoch == prune_epochs[0]-1: + print('PRUNING NON-UNIFORMLY ######################################********') + prune_epochs.pop(0) + parser_args.prune_rate = prune_rates.pop(0) + set_model_prune_rate(model, parser_args.prune_rate) + prune(model) # get model sparsity if not parser_args.weight_training: diff --git a/main_utils.py b/main_utils.py index 8f856960..e5e6e1b4 100644 --- a/main_utils.py +++ b/main_utils.py @@ -213,6 +213,13 @@ def eval_and_print(validate, data_loader, model, criterion, parser_args, writer= return acc1 def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_reg_loss_list, old_model_sparsity_list, result_root, shuffle=False, reinit=False, invert=False, chg_mask=False, chg_weight=False): + #this order is important because the second will always override the first!!! + if parser_args.fast_sparse: + finetune_epochs = 300 - parser_args.epochs + if parser_args.finetune_standard: + finetune_epochs = 150 + else: + finetune_epochs = 300 - parser_args.epochs epoch_list = copy.deepcopy(old_epoch_list) test_acc_before_round_list = copy.deepcopy(old_test_acc_before_round_list) test_acc_list = copy.deepcopy(old_test_acc_list) @@ -269,7 +276,7 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b model_sparsity_list.append(avg_sparsity) end_epoch = time.time() - for epoch in range(parser_args.epochs, parser_args.epochs*2): + for epoch in range(parser_args.epochs, parser_args.epochs+finetune_epochs): if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) @@ -335,7 +342,10 @@ def get_idty_str(parser_args): model_str = parser_args.arch algo_str = parser_args.algo rate_str = parser_args.prune_rate + #rate_str = parser_args.PRs period_str = parser_args.iter_period + #period_str = parser_args.epoch_pr + epoch_str = parser_args.epochs reg_str = 'reg_{}'.format(parser_args.regularization) reg_lmbda = parser_args.lmbda if parser_args.regularization else '' opt_str = parser_args.optimizer @@ -356,7 +366,10 @@ def get_idty_str(parser_args): format(train_mode_str, dataset_str, model_str, algo_str, rate_str, period_str, reg_str, reg_lmbda, opt_str, policy_str, lr_str, lr_gamma, lr_adj, finetune_lr_str, lam_ft_str, n_step_ft_str, fan_str, w_str, s_str, width_str, seed_str, run_idx_str).replace(".", "_") - + # idty_str = "{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_finetune_{}_fan_{}_{}_{}_width_{}_seed_{}_idx_{}".\ + # format(train_mode_str, dataset_str, model_str, algo_str, rate_str, period_str, reg_str, reg_lmbda, + # opt_str, policy_str, lr_str, lr_gamma, lr_adj, finetune_lr_str, fan_str, w_str, s_str, + # width_str, seed_str, run_idx_str).replace(".", "_") return idty_str @@ -415,10 +428,31 @@ def compare_rounding(validate, data_loader, model, criterion, parser_args, resul # switches off gradients for scores and flags and switches it on for weights and biases -def switch_to_wt(model): +def switch_to_pruning(model, reinit_scores=False): print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') parser_args.lam_finetune_loss = 0 # this is for the case considering finetune loss + for name, params in model.named_parameters(): + # make sure param_name ends with .weight or .bias + if re.match('.*\.weight', name): + params.requires_grad = False + elif parser_args.bias and re.match('.*\.bias$', name): + params.requires_grad = False + elif "score" in name: + params.requires_grad = True + if reinit_scores: + nn.init.uniform_(params, a=0.0, b=1.0) + # NOTE: this does not take parser_args.score_init into consideration + # HACK: does not work for bias, yet + else: + # flags and everything else + params.requires_grad = False + return model + + +# switches off gradients for scores and flags and switches it on for weights and biases +def switch_to_wt(model, set_scores_to_one=False): + print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') for name, params in model.named_parameters(): # make sure param_name ends with .weight or .bias if re.match('.*\.weight', name): @@ -427,6 +461,10 @@ def switch_to_wt(model): params.requires_grad = True elif "score" in name: params.requires_grad = False + # set all scores to one if you want to do vanilla weight training + if set_scores_to_one: + params.data = torch.ones_like(params.data) + # NOTE: Hack | does not work for bias, yet else: # flags and everything else params.requires_grad = False diff --git a/matt.sh b/matt.sh new file mode 100644 index 00000000..4f91744f --- /dev/null +++ b/matt.sh @@ -0,0 +1,5 @@ +### ResNet20 + +do + python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml +done diff --git a/utils/conv_type.py b/utils/conv_type.py index d58b67fc..cd7f23ea 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -199,6 +199,7 @@ def forward(self, x): b = self.bias * bias_subnet else: b = self.bias + #b = self.bias x = F.conv2d( x, w, b, self.stride, self.padding, self.dilation, self.groups )