From e6c0a483f892320b4481a8982dbadcf1de820da8 Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Sat, 4 Dec 2021 19:12:33 -0600 Subject: [PATCH 01/20] added --fast=sparse --- args_helper.py | 6 ++++++ ...resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml | 6 ++++-- main_utils.py | 2 ++ matt.sh | 15 +++++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 matt.sh diff --git a/args_helper.py b/args_helper.py index 5d7dde70..0ee9fd58 100644 --- a/args_helper.py +++ b/args_helper.py @@ -786,6 +786,12 @@ def parse_arguments(self, jupyter_mode=False): default=None, help="subfolder within the location for saving the results" ) + parser.add_argument( + "--fast-sparse", + action="store_true", + default=False, + help="Enable this to check for sparsity" + ) if jupyter_mode: args = parser.parse_args("") diff --git a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml index f4273e10..b4ee81f5 100644 --- a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml +++ b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml @@ -15,7 +15,7 @@ lr: 0.1 #0.01 #lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr # ===== Network training config ===== # -epochs: 50 #150 +epochs: 30 #150 wd: 0.0 momentum: 0.9 batch_size: 128 @@ -40,7 +40,7 @@ hc_quantized: True quantize_threshold: 0.5 # ===== Regularization ===== # -#regularization: L2 +regularization: L2 #lmbda: 0.000001 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # @@ -53,3 +53,5 @@ checkpoint_at_prune: True # ==== sanity check ==== # skip_sanity_checks: True +# ==== sanity check ==== # +fast_sparse: True \ No newline at end of file diff --git a/main_utils.py b/main_utils.py index ff736fd6..727bcddc 100644 --- a/main_utils.py +++ b/main_utils.py @@ -212,6 +212,8 @@ def eval_and_print(validate, data_loader, model, criterion, parser_args, writer= return acc1 def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_reg_loss_list, old_model_sparsity_list, result_root, shuffle=False, reinit=False, invert=False, chg_mask=False, chg_weight=False): + if parser_args.fast_sparse: + parser_args.epochs = 300 - parser_args.epochs epoch_list = copy.deepcopy(old_epoch_list) test_acc_before_round_list = copy.deepcopy(old_test_acc_before_round_list) test_acc_list = copy.deepcopy(old_test_acc_list) diff --git a/matt.sh b/matt.sh new file mode 100644 index 00000000..820a49d2 --- /dev/null +++ b/matt.sh @@ -0,0 +1,15 @@ + +### ResNet-20 +init_list=(unif) +#init_list=(skew unif) +for init in ${init_list[@]} +do +# python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov20_v2.yml \ +# --iter_period 25 --prune-rate 0.55 --score-init $init --gpu 1 --lmbda 0.0001 +# python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov20_v2.yml \ +# --iter_period 25 --prune-rate 0.55 --score-init $init --gpu 2 --lmbda 0.0001 --fine-tune-lr 0.1 +# python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov20_v2.yml \ +# --iter_period 25 --prune-rate 0.55 --score-init $init --gpu 2 --lmbda 0 --fine-tune-lr 0.1 + python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov22_3pm.yml \ + --iter_period 5 --prune-rate 0.5 --score-init $init --gpu 2 --lmbda 0.00005 --fine-tune-lr 0.01 +done \ No newline at end of file From 8e16b55c55182e1384824707819b8b0b0f2bfcc4 Mon Sep 17 00:00:00 2001 From: Matthew Grinde Date: Sun, 5 Dec 2021 19:53:15 +0000 Subject: [PATCH 02/20] updates from running --- .../resnet20/resnet20_hypercube_bottom_K_Nov22_3pm.yml | 2 +- .../resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml | 8 ++++---- matt.sh | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov22_3pm.yml b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov22_3pm.yml index 485e62d1..ee3c24f0 100644 --- a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov22_3pm.yml +++ b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov22_3pm.yml @@ -40,7 +40,7 @@ hc_quantized: True quantize_threshold: 0.5 # ===== Regularization ===== # -regularization: L2 +regularization: L1 #lmbda: 0.000001 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # diff --git a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml index b4ee81f5..b75ca922 100644 --- a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml +++ b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml @@ -12,10 +12,10 @@ name: resnet20_cifar10_hypercube_bottom_K_SGD # ===== Learning Rate Policy ======== # optimizer: sgd lr: 0.1 #0.01 -#lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr +lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr # ===== Network training config ===== # -epochs: 30 #150 +epochs: 50 #150 wd: 0.0 momentum: 0.9 batch_size: 128 @@ -40,7 +40,7 @@ hc_quantized: True quantize_threshold: 0.5 # ===== Regularization ===== # -regularization: L2 +regularization: L1 #lmbda: 0.000001 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # @@ -54,4 +54,4 @@ checkpoint_at_prune: True skip_sanity_checks: True # ==== sanity check ==== # -fast_sparse: True \ No newline at end of file +fast_sparse: True diff --git a/matt.sh b/matt.sh index 820a49d2..cdf41d55 100644 --- a/matt.sh +++ b/matt.sh @@ -10,6 +10,6 @@ do # --iter_period 25 --prune-rate 0.55 --score-init $init --gpu 2 --lmbda 0.0001 --fine-tune-lr 0.1 # python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov20_v2.yml \ # --iter_period 25 --prune-rate 0.55 --score-init $init --gpu 2 --lmbda 0 --fine-tune-lr 0.1 - python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov22_3pm.yml \ - --iter_period 5 --prune-rate 0.5 --score-init $init --gpu 2 --lmbda 0.00005 --fine-tune-lr 0.01 -done \ No newline at end of file + python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml \ + --iter_period 6 --prune-rate 0.4 --score-init $init --gpu 2 --lmbda 0.00005 --fine-tune-lr 0.01 +done From 9fee1c1c35d3a3e4d664fd8bb8cb3ce2e7a29aa3 Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Sun, 5 Dec 2021 17:17:07 -0600 Subject: [PATCH 03/20] non-uniform pruning --- args_helper.py | 20 ++++++++++++++++++- ...et20_hypercube_bottom_K_Nov23_11am_SGD.yml | 6 +++++- main.py | 17 +++++++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/args_helper.py b/args_helper.py index 0ee9fd58..8cbb5d04 100644 --- a/args_helper.py +++ b/args_helper.py @@ -790,7 +790,25 @@ def parse_arguments(self, jupyter_mode=False): "--fast-sparse", action="store_true", default=False, - help="Enable this to check for sparsity" + help="Enable this for 300-epochs for fine-tuning" + ) + parser.add_argument( + "--unif-prune", + action="store_true", + default=True, + help="disable for non uniform pruning rates" + ) + parser.add_argument( + '--PRs', #pruning rates + default=None, + type=str, + help='for each number in PR it will start at the corresponding epoch-pr' + ) + parser.add_argument( + '--epoch-pr', #pruning rates + default=None, + type=str, + help='this goes in tandum with PRs, for each number in epoch that is when the corresponding PR starts' ) if jupyter_mode: diff --git a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml index b75ca922..97b93ba6 100644 --- a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml +++ b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml @@ -16,6 +16,9 @@ lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr # ===== Network training config ===== # epochs: 50 #150 +unif_prune: False +PRs: .50,.50,.50,.25 +epoch_pr: 2,4,6,10,20 wd: 0.0 momentum: 0.9 batch_size: 128 @@ -26,6 +29,7 @@ bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK #prune_rate: 0.2 + init: signed_constant #score_init: skew #half #bimodal #skew # bern scale_fan: False #True @@ -53,5 +57,5 @@ checkpoint_at_prune: True # ==== sanity check ==== # skip_sanity_checks: True -# ==== sanity check ==== # +# ==== get sparsity quickly and train much ==== # fast_sparse: True diff --git a/main.py b/main.py index df3d83bf..663cf9d4 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +from yaml import parse from main_utils import * @@ -41,6 +42,12 @@ def main_worker(gpu, ngpus_per_node): if not os.path.isdir(result_root): os.mkdir(result_root) + + if not parser_args.unif_prune: + pruning_rates = list(map(int, parser_args.PRs.split(','))) + prune_epochs = list(map(int, parser_args.epoch_pr.split(','))) + parser_args.pruning_rate = pruning_rates[0] + model = get_model(parser_args) ''' @@ -140,10 +147,18 @@ def main_worker(gpu, ngpus_per_node): validation_time.update((time.time() - start_validation) / 60) # prune the model every T_{prune} epochs - if parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: + if parser_args.algo in ['hc_iter', 'global_ep_iter'] and parser_args.unif_prune and epoch % (parser_args.iter_period) == 0 and epoch != 0: prune(model) if parser_args.checkpoint_at_prune: save_checkpoint_at_prune(model, parser_args) + + #prune model non-uniformly + if not parser_args.unif_prune: + if epoch == prune_epochs[0]: + prune_epochs.pop(0) + parser_args.pruning_rate = pruning_rates.pop(0) + set_model_prune_rate(model, parser_args.pruning_rate) + prune(model) # get model sparsity if not parser_args.weight_training: From f6427d0d1ce81e295ba72900b23ce0121a3893f0 Mon Sep 17 00:00:00 2001 From: Matthew Grinde Date: Mon, 6 Dec 2021 15:41:16 +0000 Subject: [PATCH 04/20] nonuniform bug fixes and output change --- .../resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml | 4 ++-- main.py | 13 +++++++------ main_utils.py | 14 ++++++++++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml index 97b93ba6..7b889bb4 100644 --- a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml +++ b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml @@ -17,8 +17,8 @@ lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr # ===== Network training config ===== # epochs: 50 #150 unif_prune: False -PRs: .50,.50,.50,.25 -epoch_pr: 2,4,6,10,20 +PRs: .50,.50,.40,.35,.30,.30 +epoch_pr: 5,10,15,20,25,35,60 wd: 0.0 momentum: 0.9 batch_size: 128 diff --git a/main.py b/main.py index 663cf9d4..0d1d360a 100644 --- a/main.py +++ b/main.py @@ -44,9 +44,9 @@ def main_worker(gpu, ngpus_per_node): os.mkdir(result_root) if not parser_args.unif_prune: - pruning_rates = list(map(int, parser_args.PRs.split(','))) - prune_epochs = list(map(int, parser_args.epoch_pr.split(','))) - parser_args.pruning_rate = pruning_rates[0] + prune_rates = list(map(float, parser_args.PRs.split(','))) + prune_epochs = list(map(float, parser_args.epoch_pr.split(','))) + parser_args.prune_rate = prune_rates[0] model = get_model(parser_args) @@ -154,10 +154,11 @@ def main_worker(gpu, ngpus_per_node): #prune model non-uniformly if not parser_args.unif_prune: - if epoch == prune_epochs[0]: + if epoch == prune_epochs[0]-1: + print('PRUNING NON-UNIFORMLY ######################################********') prune_epochs.pop(0) - parser_args.pruning_rate = pruning_rates.pop(0) - set_model_prune_rate(model, parser_args.pruning_rate) + parser_args.prune_rate = prune_rates.pop(0) + set_model_prune_rate(model, parser_args.prune_rate) prune(model) # get model sparsity diff --git a/main_utils.py b/main_utils.py index 727bcddc..acfdc566 100644 --- a/main_utils.py +++ b/main_utils.py @@ -323,8 +323,11 @@ def get_idty_str(parser_args): dataset_str = parser_args.dataset model_str = parser_args.arch algo_str = parser_args.algo - rate_str = parser_args.prune_rate - period_str = parser_args.iter_period + #rate_str = parser_args.prune_rate + rate_str = parser_args.PRs + #period_str = parser_args.iter_period + period_str = parser_args.epoch_pr + epoch_str = parser_args.epochs reg_str = 'reg_{}'.format(parser_args.regularization) reg_lmbda = parser_args.lmbda if parser_args.regularization else '' opt_str = parser_args.optimizer @@ -340,10 +343,13 @@ def get_idty_str(parser_args): seed_str = parser_args.seed + parser_args.trial_num - 1 run_idx_str = parser_args.run_idx idty_str = "{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_finetune_{}_fan_{}_{}_{}_width_{}_seed_{}_idx_{}".\ - format(train_mode_str, dataset_str, model_str, algo_str, rate_str, period_str, reg_str, reg_lmbda, + format(train_mode_str, dataset_str, model_str, algo_str, rate_str, period_str, epoch_str, reg_str, reg_lmbda, opt_str, policy_str, lr_str, lr_gamma, lr_adj, finetune_lr_str, fan_str, w_str, s_str, width_str, seed_str, run_idx_str).replace(".", "_") - + # idty_str = "{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_finetune_{}_fan_{}_{}_{}_width_{}_seed_{}_idx_{}".\ + # format(train_mode_str, dataset_str, model_str, algo_str, rate_str, period_str, reg_str, reg_lmbda, + # opt_str, policy_str, lr_str, lr_gamma, lr_adj, finetune_lr_str, fan_str, w_str, s_str, + # width_str, seed_str, run_idx_str).replace(".", "_") return idty_str From 8786219cd799cd7041d687e7d9b031a5f49878af Mon Sep 17 00:00:00 2001 From: Matthew Grinde Date: Tue, 7 Dec 2021 20:02:46 +0000 Subject: [PATCH 05/20] changes from docker --- .../resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml index 7b889bb4..752a6ccc 100644 --- a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml +++ b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml @@ -15,10 +15,10 @@ lr: 0.1 #0.01 lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr # ===== Network training config ===== # -epochs: 50 #150 +epochs: 30 #150 unif_prune: False -PRs: .50,.50,.40,.35,.30,.30 -epoch_pr: 5,10,15,20,25,35,60 +PRs: .50,.50,.50,.50,.50 +epoch_pr: 1,5,10,15,20,30,40 wd: 0.0 momentum: 0.9 batch_size: 128 From be840854bcf5375eb8d886eda903b3752fd74243 Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Tue, 7 Dec 2021 14:10:52 -0600 Subject: [PATCH 06/20] updated config file --- .../resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml index 752a6ccc..18d3dde1 100644 --- a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml +++ b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml @@ -13,6 +13,8 @@ name: resnet20_cifar10_hypercube_bottom_K_SGD optimizer: sgd lr: 0.1 #0.01 lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr # ===== Network training config ===== # epochs: 30 #150 @@ -29,9 +31,8 @@ bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK #prune_rate: 0.2 - init: signed_constant -#score_init: skew #half #bimodal #skew # bern +score_init: unif #half #bimodal #skew # bern scale_fan: False #True # ===== Rounding ===== # @@ -45,7 +46,7 @@ quantize_threshold: 0.5 # ===== Regularization ===== # regularization: L1 -#lmbda: 0.000001 # 0.01 #0.0001 #0.000001 +lmbda: 0.00005 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # workers: 4 From b7e65687be940eb57530bb75536c161ac49c55aa Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Tue, 7 Dec 2021 14:53:38 -0600 Subject: [PATCH 07/20] changes to configs and added standard finetuning length param --- args_helper.py | 6 ++++++ .../resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml | 9 ++++++--- main_utils.py | 7 +++++-- matt.sh | 12 ++---------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/args_helper.py b/args_helper.py index 8cbb5d04..e62372b3 100644 --- a/args_helper.py +++ b/args_helper.py @@ -798,6 +798,12 @@ def parse_arguments(self, jupyter_mode=False): default=True, help="disable for non uniform pruning rates" ) + parser.add_argument( + "--finetune-standard", + action="store_true", + default=False, + help="True mean 150 epochs finetuning nomatter how many epochs for pruning" + ) parser.add_argument( '--PRs', #pruning rates default=None, diff --git a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml index 18d3dde1..d20b6731 100644 --- a/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml +++ b/configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml @@ -1,6 +1,6 @@ # Hypercube optimization algo: 'hc_iter' -#iter_period: 8 #15 +iter_period: 8 #15 # Architecture arch: resnet20 @@ -19,7 +19,7 @@ fine_tune_lr_policy: multistep_lr # ===== Network training config ===== # epochs: 30 #150 unif_prune: False -PRs: .50,.50,.50,.50,.50 +PRs: .60,.50,.50,.05,.40,.40 epoch_pr: 1,5,10,15,20,30,40 wd: 0.0 momentum: 0.9 @@ -30,7 +30,7 @@ conv_type: SubnetConv bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK -#prune_rate: 0.2 +prune_rate: 0.2 init: signed_constant score_init: unif #half #bimodal #skew # bern scale_fan: False #True @@ -60,3 +60,6 @@ skip_sanity_checks: True # ==== get sparsity quickly and train much ==== # fast_sparse: True + +# ==== ==== # +finetune_standard: False \ No newline at end of file diff --git a/main_utils.py b/main_utils.py index acfdc566..2ebb3671 100644 --- a/main_utils.py +++ b/main_utils.py @@ -212,8 +212,11 @@ def eval_and_print(validate, data_loader, model, criterion, parser_args, writer= return acc1 def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_reg_loss_list, old_model_sparsity_list, result_root, shuffle=False, reinit=False, invert=False, chg_mask=False, chg_weight=False): + #this order is important because the second will always override the first!!! if parser_args.fast_sparse: - parser_args.epochs = 300 - parser_args.epochs + finetune_epochs = 300 - parser_args.epochs + if parser_args.finetune_standard: + finetune_epochs = 150 epoch_list = copy.deepcopy(old_epoch_list) test_acc_before_round_list = copy.deepcopy(old_test_acc_before_round_list) test_acc_list = copy.deepcopy(old_test_acc_list) @@ -258,7 +261,7 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b model_sparsity_list.append(avg_sparsity) end_epoch = time.time() - for epoch in range(parser_args.epochs, parser_args.epochs*2): + for epoch in range(parser_args.epochs, parser_args.epochs+finetune_epochs): if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) diff --git a/matt.sh b/matt.sh index cdf41d55..8140ae96 100644 --- a/matt.sh +++ b/matt.sh @@ -1,15 +1,7 @@ ### ResNet-20 -init_list=(unif) -#init_list=(skew unif) -for init in ${init_list[@]} + do -# python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov20_v2.yml \ -# --iter_period 25 --prune-rate 0.55 --score-init $init --gpu 1 --lmbda 0.0001 -# python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov20_v2.yml \ -# --iter_period 25 --prune-rate 0.55 --score-init $init --gpu 2 --lmbda 0.0001 --fine-tune-lr 0.1 -# python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov20_v2.yml \ -# --iter_period 25 --prune-rate 0.55 --score-init $init --gpu 2 --lmbda 0 --fine-tune-lr 0.1 python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml \ - --iter_period 6 --prune-rate 0.4 --score-init $init --gpu 2 --lmbda 0.00005 --fine-tune-lr 0.01 + --gpu 2 done From 62bd02b2471b5f60a7d4fa3c2ab012339e8cc4f9 Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 09:51:16 -0600 Subject: [PATCH 08/20] added back arguments for nonuniform pruning and first updates for warm HC --- .DS_Store | Bin 0 -> 6148 bytes args_helper.py | 39 ++++++++++++++++++++++++++++++++++++++- main.py | 12 ++++++++++++ main_utils.py | 25 +++++++++++++++++++++---- matt.sh | 4 ++++ 5 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..46d0e6527f13af5ba9deefa2b9fdae1c01117d61 GIT binary patch literal 6148 zcmeHK%}&BV5S|5;1!Lr3BFA35aR4+X9!wgj-i6q|fEMUz%E4tud0$I$^P zOS06dRMWK4IILOwd(COhN)MZLvJa-ylHA#C9G!OW9v;)D>GO-`R^V@_WXs?TUeWln z%zI}P22ps6ky+I&n~@n{2AF{rXTWTGa%08U$;)E~n1SCjK>LG*N@yF*G^(uw8@fJH zyg*2THoYYXErYhfOe026gf2zYrNTThgf2(FW%6u;nMPd>Ld}fhn3;unp$IiQ`Yn|X z!q&(wGr$ZiGLYBZ3Z4HaKfnJMi@3)OFas;afGD(`b_wIm`g`-6;%oPEiw>ogPBJ3pzuXN(ZCHe@TUxX0H?51GXMYp literal 0 HcmV?d00001 diff --git a/args_helper.py b/args_helper.py index d4d8cd69..163c4263 100644 --- a/args_helper.py +++ b/args_helper.py @@ -245,6 +245,13 @@ def parse_arguments(self, jupyter_mode=False): type=int, help="warmup epochs for hypercube" ) + parser.add_argument( + "--toggle-warmup", + default=9999, + action='store_true', + default=False, + help='flag that decides if we add noise to the rounded p_i' + ) parser.add_argument( "--hc-period", default=1, @@ -813,7 +820,37 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="Enable this to use bottomK on forward for HC" ) - + parser.add_argument( + "--fast-sparse", + action="store_true", + default=False, + help="Enable this for 300-epochs for fine-tuning" + ) + parser.add_argument( + "--unif-prune", + action="store_true", + default=True, + help="disable for non uniform pruning rates" + ) + parser.add_argument( + "--finetune-standard", + action="store_true", + default=False, + help="True mean 150 epochs finetuning nomatter how many epochs for pruning" + ) + parser.add_argument( + '--PRs', #pruning rates + default=None, + type=str, + help='for each number in PR it will start at the corresponding epoch-pr' + ) + parser.add_argument( + '--epoch-pr', #pruning rates + default=None, + type=str, + help='this goes in tandum with PRs, for each number in epoch that is when the corresponding PR starts' + ) + if jupyter_mode: args = parser.parse_args("") else: diff --git a/main.py b/main.py index adf4b2d5..b9375ee7 100644 --- a/main.py +++ b/main.py @@ -103,6 +103,18 @@ def main_worker(gpu, ngpus_per_node): # Save the initial model torch.save(model.state_dict(), result_root + 'init_model.pth') + warmup_epochs = parser_args.hc_warmup + if parser_args.toggle_warmup: + reg = parser_args.regularization + parser_args.regularization = None + model = switch_to_wt(model) + for warmup in range(0,warmup_epochs): + train_acc1, train_acc5, train_acc10, reg_loss = train( + data.train_loader, model, criterion, optimizer, warmup, parser_args, writer=writer + ) + parser_args.regularization = reg + model = switch_to_pruning(model) + # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): if parser_args.multiprocessing_distributed: diff --git a/main_utils.py b/main_utils.py index e3962e04..9fa00c18 100644 --- a/main_utils.py +++ b/main_utils.py @@ -328,10 +328,10 @@ def get_idty_str(parser_args): dataset_str = parser_args.dataset model_str = parser_args.arch algo_str = parser_args.algo - #rate_str = parser_args.prune_rate - rate_str = parser_args.PRs - #period_str = parser_args.iter_period - period_str = parser_args.epoch_pr + rate_str = parser_args.prune_rate + #rate_str = parser_args.PRs + period_str = parser_args.iter_period + #period_str = parser_args.epoch_pr epoch_str = parser_args.epochs reg_str = 'reg_{}'.format(parser_args.regularization) reg_lmbda = parser_args.lmbda if parser_args.regularization else '' @@ -412,6 +412,23 @@ def compare_rounding(validate, data_loader, model, criterion, parser_args, resul return +# switches off gradients for scores and flags and switches it on for weights and biases +def switch_to_pruning(model): + print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') + + for name, params in model.named_parameters(): + # make sure param_name ends with .weight or .bias + if re.match('.*\.weight', name): + params.requires_grad = False + elif parser_args.bias and re.match('.*\.bias$', name): + params.requires_grad = False + elif "score" in name: + params.requires_grad = True + else: + # flags and everything else + params.requires_grad = False + + # switches off gradients for scores and flags and switches it on for weights and biases def switch_to_wt(model): print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') diff --git a/matt.sh b/matt.sh index 8140ae96..5a152d59 100644 --- a/matt.sh +++ b/matt.sh @@ -5,3 +5,7 @@ do python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml \ --gpu 2 done + + +# hc-warmup +#--toggle-warmup \ No newline at end of file From c8bea660374015f3d037c98b5af3fb62d40fe8f3 Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 11:26:13 -0600 Subject: [PATCH 09/20] updated the config and bash files to run warm HC in with current yml --- .../resnet20/resnet20_quantized_iter_hc_0_5.yml | 4 ++++ matt.sh | 10 ++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml index f94f3295..983d34a9 100644 --- a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml +++ b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml @@ -58,3 +58,7 @@ checkpoint_at_prune: True # ==== sanity check ==== # skip_sanity_checks: False + +# ==== HC with warm-up ==== # +hc-warmup: 3 +toggle_warmup: True diff --git a/matt.sh b/matt.sh index 5a152d59..4d1c72e3 100644 --- a/matt.sh +++ b/matt.sh @@ -1,11 +1,5 @@ - -### ResNet-20 +### ResNet20 do - python main.py --config configs/hypercube/resnet20/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml \ - --gpu 2 + python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml \ done - - -# hc-warmup -#--toggle-warmup \ No newline at end of file From 64e4d700c48395f0cc06064519fa6eebbe5de33d Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 14:23:11 -0600 Subject: [PATCH 10/20] fleshed out the warmup process - ready for initial run --- main.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/main.py b/main.py index b9375ee7..c1987a81 100644 --- a/main.py +++ b/main.py @@ -109,9 +109,32 @@ def main_worker(gpu, ngpus_per_node): parser_args.regularization = None model = switch_to_wt(model) for warmup in range(0,warmup_epochs): + print('warm-up epoch', warmup) train_acc1, train_acc5, train_acc10, reg_loss = train( data.train_loader, model, criterion, optimizer, warmup, parser_args, writer=writer ) + # add stuff here to write the results to the CSV so I can tell if it works + acc1, acc5, acc10 = validate(data.val_loader, model, criterion, parser_args, writer, warmup) + + epoch_list.append(warmup) + test_acc_before_round_list.append(-1) + test_acc_list.append(acc1) + reg_loss_list.append(reg_loss) + cp_model = round_model(model, parser_args.round, noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) + avg_sparsity = get_model_sparsity(cp_model) + model_sparsity_list.append(avg_sparsity) + + results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list,'test_acc': test_acc_list, + 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) + + if parser_args.results_filename: + results_filename = parser_args.results_filename + else: + results_filename = result_root + 'acc_and_sparsity.csv' + + + print("Writing results into: {}".format(results_filename)) + results_df.to_csv(results_filename, index=False) parser_args.regularization = reg model = switch_to_pruning(model) From 000dc74952d5abd05dc7b832f331599221effa82 Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 14:30:52 -0600 Subject: [PATCH 11/20] bug fix --- args_helper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/args_helper.py b/args_helper.py index 163c4263..1aea1147 100644 --- a/args_helper.py +++ b/args_helper.py @@ -247,7 +247,6 @@ def parse_arguments(self, jupyter_mode=False): ) parser.add_argument( "--toggle-warmup", - default=9999, action='store_true', default=False, help='flag that decides if we add noise to the rounded p_i' @@ -850,7 +849,7 @@ def parse_arguments(self, jupyter_mode=False): type=str, help='this goes in tandum with PRs, for each number in epoch that is when the corresponding PR starts' ) - + if jupyter_mode: args = parser.parse_args("") else: From 9489ae3c22a4977171062292e5231cc63e49908a Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 14:46:07 -0600 Subject: [PATCH 12/20] bug fix --- configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml index 983d34a9..ce323509 100644 --- a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml +++ b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml @@ -60,5 +60,5 @@ checkpoint_at_prune: True skip_sanity_checks: False # ==== HC with warm-up ==== # -hc-warmup: 3 +hc_warmup: 3 toggle_warmup: True From 48909c3952be6017e89e609a99f6d3d221018ca0 Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 14:54:45 -0600 Subject: [PATCH 13/20] more bug fixes --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index c1987a81..d4739132 100644 --- a/main.py +++ b/main.py @@ -137,7 +137,8 @@ def main_worker(gpu, ngpus_per_node): results_df.to_csv(results_filename, index=False) parser_args.regularization = reg model = switch_to_pruning(model) - + train, validate, modifier = get_trainer(parser_args) + # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): if parser_args.multiprocessing_distributed: From 83f0fdd9d1cc921811d77c94266fad90d3909e7e Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 15:01:12 -0600 Subject: [PATCH 14/20] more bug fixes --- main.py | 1 - main_utils.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index d4739132..4778ae93 100644 --- a/main.py +++ b/main.py @@ -137,7 +137,6 @@ def main_worker(gpu, ngpus_per_node): results_df.to_csv(results_filename, index=False) parser_args.regularization = reg model = switch_to_pruning(model) - train, validate, modifier = get_trainer(parser_args) # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): diff --git a/main_utils.py b/main_utils.py index 9fa00c18..9ba5e599 100644 --- a/main_utils.py +++ b/main_utils.py @@ -427,6 +427,7 @@ def switch_to_pruning(model): else: # flags and everything else params.requires_grad = False + return model # switches off gradients for scores and flags and switches it on for weights and biases From ebe9bd7bbb73a65095f470566ac32f741d4bf8e6 Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 15:15:27 -0600 Subject: [PATCH 15/20] new yml for testing warm up on 2 layer FC network --- configs/hypercube/resnet20/test_warmup.yml | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 configs/hypercube/resnet20/test_warmup.yml diff --git a/configs/hypercube/resnet20/test_warmup.yml b/configs/hypercube/resnet20/test_warmup.yml new file mode 100644 index 00000000..fea479c0 --- /dev/null +++ b/configs/hypercube/resnet20/test_warmup.yml @@ -0,0 +1,64 @@ +subfolder: quantize_0_75 + +# Hypercube optimization +algo: 'hc_iter' +# effectively prune once in the entire process +iter_period: 100 + +# Architecture +arch: TwoLayerFC + +# ===== Dataset ===== # +dataset: MNIST +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +prune_rate: 0.995 +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 0.01 #0.0001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: False + +# ==== HC with warm-up ==== # +hc_warmup: 3 +toggle_warmup: True From c9ea0d6a9ad666e9b6ff32d895120714fe23ba9c Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Fri, 10 Dec 2021 16:22:57 -0600 Subject: [PATCH 16/20] comments --- main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 4778ae93..cf56c00b 100644 --- a/main.py +++ b/main.py @@ -103,8 +103,11 @@ def main_worker(gpu, ngpus_per_node): # Save the initial model torch.save(model.state_dict(), result_root + 'init_model.pth') + # TODO: once this works, move it to the main_utils.py! + # maybe I should just hack it to use the finetune method for 3 epochs? warmup_epochs = parser_args.hc_warmup if parser_args.toggle_warmup: + # save the regularizer and disable it for warmup then swtich to weight training reg = parser_args.regularization parser_args.regularization = None model = switch_to_wt(model) @@ -113,13 +116,13 @@ def main_worker(gpu, ngpus_per_node): train_acc1, train_acc5, train_acc10, reg_loss = train( data.train_loader, model, criterion, optimizer, warmup, parser_args, writer=writer ) - # add stuff here to write the results to the CSV so I can tell if it works acc1, acc5, acc10 = validate(data.val_loader, model, criterion, parser_args, writer, warmup) - + # for reporting accuracy in the CSV for debug purpose epoch_list.append(warmup) test_acc_before_round_list.append(-1) test_acc_list.append(acc1) reg_loss_list.append(reg_loss) + # calcualte sparsity for each epoch -- do we expect this to be 100 or 50? currently 50 cp_model = round_model(model, parser_args.round, noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) avg_sparsity = get_model_sparsity(cp_model) model_sparsity_list.append(avg_sparsity) @@ -135,6 +138,7 @@ def main_worker(gpu, ngpus_per_node): print("Writing results into: {}".format(results_filename)) results_df.to_csv(results_filename, index=False) + # enable regularization for HC and switch back to pruning parser_args.regularization = reg model = switch_to_pruning(model) From 93a9e38f03a3b9c12447e6335142a5a77cad846c Mon Sep 17 00:00:00 2001 From: Kartik Sreenivasan Date: Fri, 10 Dec 2021 22:59:36 +0000 Subject: [PATCH 17/20] adding some lines of code to debug weight_training --- cifar_exec.sh | 2 +- .../resnet20/resnet20_quantized_iter_hc_0_5.yml | 4 ++-- main.py | 5 +++-- main_utils.py | 13 ++++++++++--- trainers/default.py | 1 - utils/conv_type.py | 3 +++ 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 51418739..166bd75e 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -25,7 +25,7 @@ BLOCK # target sparsity 0.5 python main.py \ ---config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_0_5.yml > cifar_log_target_0_5 2>&1 +--config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml #> cifar_log 2>&1 # EP diff --git a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml index ce323509..84a1796d 100644 --- a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml +++ b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml @@ -14,7 +14,7 @@ name: resnet20_quantized_iter_hc # ===== Learning Rate Policy ======== # optimizer: sgd -lr: 0.1 #0.01 +lr: 0.01 lr_policy: cosine_lr #constant_lr #multistep_lr fine_tune_lr: 0.01 fine_tune_lr_policy: multistep_lr @@ -60,5 +60,5 @@ checkpoint_at_prune: True skip_sanity_checks: False # ==== HC with warm-up ==== # -hc_warmup: 3 +hc_warmup: 20 toggle_warmup: True diff --git a/main.py b/main.py index cf56c00b..21f1ad96 100644 --- a/main.py +++ b/main.py @@ -110,7 +110,7 @@ def main_worker(gpu, ngpus_per_node): # save the regularizer and disable it for warmup then swtich to weight training reg = parser_args.regularization parser_args.regularization = None - model = switch_to_wt(model) + model = switch_to_wt(model, set_scores_to_one=True) for warmup in range(0,warmup_epochs): print('warm-up epoch', warmup) train_acc1, train_acc5, train_acc10, reg_loss = train( @@ -140,7 +140,8 @@ def main_worker(gpu, ngpus_per_node): results_df.to_csv(results_filename, index=False) # enable regularization for HC and switch back to pruning parser_args.regularization = reg - model = switch_to_pruning(model) + model = switch_to_pruning(model, reinit_scores=True) + import ipdb; ipdb.set_trace() # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): diff --git a/main_utils.py b/main_utils.py index 9ba5e599..917700e3 100644 --- a/main_utils.py +++ b/main_utils.py @@ -413,7 +413,7 @@ def compare_rounding(validate, data_loader, model, criterion, parser_args, resul # switches off gradients for scores and flags and switches it on for weights and biases -def switch_to_pruning(model): +def switch_to_pruning(model, reinit_scores=False): print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') for name, params in model.named_parameters(): @@ -424,6 +424,10 @@ def switch_to_pruning(model): params.requires_grad = False elif "score" in name: params.requires_grad = True + if reinit_scores: + nn.init.uniform_(params, a=0.0, b=1.0) + # NOTE: this does not take parser_args.score_init into consideration + # HACK: does not work for bias, yet else: # flags and everything else params.requires_grad = False @@ -431,9 +435,8 @@ def switch_to_pruning(model): # switches off gradients for scores and flags and switches it on for weights and biases -def switch_to_wt(model): +def switch_to_wt(model, set_scores_to_one=False): print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.') - for name, params in model.named_parameters(): # make sure param_name ends with .weight or .bias if re.match('.*\.weight', name): @@ -442,6 +445,10 @@ def switch_to_wt(model): params.requires_grad = True elif "score" in name: params.requires_grad = False + # set all scores to one if you want to do vanilla weight training + if set_scores_to_one: + params.data = torch.ones_like(params.data) + # NOTE: Hack | does not work for bias, yet else: # flags and everything else params.requires_grad = False diff --git a/trainers/default.py b/trainers/default.py index b74bd8d3..9172743a 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -76,7 +76,6 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): optimizer.zero_grad() loss.backward() optimizer.step() - # measure elapsed time batch_time.update(time.time() - end) end = time.time() diff --git a/utils/conv_type.py b/utils/conv_type.py index d58b67fc..cdb253fe 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -199,6 +199,9 @@ def forward(self, x): b = self.bias * bias_subnet else: b = self.bias + # NOTE @mgrinde: changing this for ease of debug + w = self.weight + b = self.bias x = F.conv2d( x, w, b, self.stride, self.padding, self.dilation, self.groups ) From f510c9cdc1b2cf5d17786e3f39b92e07e94dca66 Mon Sep 17 00:00:00 2001 From: Kartik Sreenivasan Date: Sat, 11 Dec 2021 01:08:22 +0000 Subject: [PATCH 18/20] bugfix: optimizer was not tied to the weight parameters --- .../resnet20/resnet20_quantized_iter_hc_0_5.yml | 2 +- main.py | 12 ++++++++---- utils/conv_type.py | 2 -- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml index 84a1796d..ad512918 100644 --- a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml +++ b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml @@ -60,5 +60,5 @@ checkpoint_at_prune: True skip_sanity_checks: False # ==== HC with warm-up ==== # -hc_warmup: 20 +hc_warmup: 3 toggle_warmup: True diff --git a/main.py b/main.py index 21f1ad96..a876f8ef 100644 --- a/main.py +++ b/main.py @@ -111,6 +111,9 @@ def main_worker(gpu, ngpus_per_node): reg = parser_args.regularization parser_args.regularization = None model = switch_to_wt(model, set_scores_to_one=True) + # optimizer needs to be tied to the correct parameters + optimizer = get_optimizer(parser_args, model) + scheduler = get_scheduler(optimizer, parser_args.lr_policy) for warmup in range(0,warmup_epochs): print('warm-up epoch', warmup) train_acc1, train_acc5, train_acc10, reg_loss = train( @@ -126,7 +129,6 @@ def main_worker(gpu, ngpus_per_node): cp_model = round_model(model, parser_args.round, noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) avg_sparsity = get_model_sparsity(cp_model) model_sparsity_list.append(avg_sparsity) - results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list,'test_acc': test_acc_list, 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) @@ -135,14 +137,16 @@ def main_worker(gpu, ngpus_per_node): else: results_filename = result_root + 'acc_and_sparsity.csv' - print("Writing results into: {}".format(results_filename)) results_df.to_csv(results_filename, index=False) # enable regularization for HC and switch back to pruning parser_args.regularization = reg model = switch_to_pruning(model, reinit_scores=True) - import ipdb; ipdb.set_trace() - + # optimizer needs to be tied to the correct parameters + optimizer = get_optimizer(parser_args, model) + scheduler = get_scheduler(optimizer, parser_args.lr_policy) + + # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): if parser_args.multiprocessing_distributed: diff --git a/utils/conv_type.py b/utils/conv_type.py index cdb253fe..e1dcd8d3 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -199,8 +199,6 @@ def forward(self, x): b = self.bias * bias_subnet else: b = self.bias - # NOTE @mgrinde: changing this for ease of debug - w = self.weight b = self.bias x = F.conv2d( x, w, b, self.stride, self.padding, self.dilation, self.groups From f79e443d3616cb40460f24e312e8dcf104bdd059 Mon Sep 17 00:00:00 2001 From: Matthew Grinde Date: Mon, 13 Dec 2021 16:19:46 +0000 Subject: [PATCH 19/20] mild changes and bug fix in finetune --- .../resnet20/resnet20_quantized_iter_hc_0_5.yml | 10 +++++----- main_utils.py | 2 ++ matt.sh | 2 +- utils/conv_type.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml index ad512918..20495f7d 100644 --- a/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml +++ b/configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml @@ -1,4 +1,4 @@ -subfolder: quantize_0_75 +subfolder: quantize_0_75_checking # Hypercube optimization algo: 'hc_iter' @@ -14,7 +14,7 @@ name: resnet20_quantized_iter_hc # ===== Learning Rate Policy ======== # optimizer: sgd -lr: 0.01 +lr: 0.1 lr_policy: cosine_lr #constant_lr #multistep_lr fine_tune_lr: 0.01 fine_tune_lr_policy: multistep_lr @@ -31,7 +31,7 @@ bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK # enter target sparsity here -prune_rate: 0.995 +prune_rate: 0.982 init: signed_constant score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True @@ -51,13 +51,13 @@ lmbda: 0.00005 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # workers: 4 -gpu: 0 +gpu: 3 # ===== Checkpointing ===== # checkpoint_at_prune: True # ==== sanity check ==== # -skip_sanity_checks: False +skip_sanity_checks: True # ==== HC with warm-up ==== # hc_warmup: 3 diff --git a/main_utils.py b/main_utils.py index 917700e3..a7c40376 100644 --- a/main_utils.py +++ b/main_utils.py @@ -217,6 +217,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b finetune_epochs = 300 - parser_args.epochs if parser_args.finetune_standard: finetune_epochs = 150 + else: + finetune_epochs = 300 - parser_args.epochs epoch_list = copy.deepcopy(old_epoch_list) test_acc_before_round_list = copy.deepcopy(old_test_acc_before_round_list) test_acc_list = copy.deepcopy(old_test_acc_list) diff --git a/matt.sh b/matt.sh index 4d1c72e3..4f91744f 100644 --- a/matt.sh +++ b/matt.sh @@ -1,5 +1,5 @@ ### ResNet20 do - python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml \ + python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml done diff --git a/utils/conv_type.py b/utils/conv_type.py index e1dcd8d3..cd7f23ea 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -199,7 +199,7 @@ def forward(self, x): b = self.bias * bias_subnet else: b = self.bias - b = self.bias + #b = self.bias x = F.conv2d( x, w, b, self.stride, self.padding, self.dilation, self.groups ) From 4010ce419791886c884c8802b7a1b3980ec0a7ec Mon Sep 17 00:00:00 2001 From: matthewGrinde Date: Mon, 13 Dec 2021 10:29:06 -0600 Subject: [PATCH 20/20] new yml for nonunifom pruning --- ...resnet20_quantized_nonunif_iter_hc_0_5.yml | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 configs/hypercube/resnet20/resnet20_quantized_nonunif_iter_hc_0_5.yml diff --git a/configs/hypercube/resnet20/resnet20_quantized_nonunif_iter_hc_0_5.yml b/configs/hypercube/resnet20/resnet20_quantized_nonunif_iter_hc_0_5.yml new file mode 100644 index 00000000..6c79734b --- /dev/null +++ b/configs/hypercube/resnet20/resnet20_quantized_nonunif_iter_hc_0_5.yml @@ -0,0 +1,67 @@ +subfolder: quantize_0_75_checking + +# Hypercube optimization +algo: 'hc_iter' +# effectively prune once in the entire process +iter_period: 100 +unif_prune: False + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +prune_rate: 0.982 +PRs: 0.5,0.5,0.5,0.5,0.288 +epoch_pr: 20,40,60,80,100,200 +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 0.01 #0.0001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True + +# ==== HC with warm-up ==== # +hc_warmup: 3 +toggle_warmup: False