From c9e7f0c370a7147e84cd288aaa408ff3681557f0 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 7 Apr 2022 10:53:48 +0000 Subject: [PATCH 01/72] delete default model saving code (for saving memory) --- main.py | 7 +++---- main_utils.py | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 42dc3829..87840cea 100644 --- a/main.py +++ b/main.py @@ -96,7 +96,7 @@ def main_worker(gpu, ngpus_per_node): epoch_list, test_acc_before_round_list, test_acc_list, reg_loss_list, model_sparsity_list, val_acc_list, train_acc_list = [], [], [], [], [], [], [] # Save the initial model - torch.save(model.state_dict(), result_root + 'init_model.pth') + #torch.save(model.state_dict(), result_root + 'init_model.pth') # compute prune_rate to reach target_sparsity if not parser_args.override_prune_rate: @@ -280,7 +280,7 @@ def main_worker(gpu, ngpus_per_node): results_df.to_csv(results_filename, index=False) # save checkpoint before fine-tuning - torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') + #torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') # finetune weights cp_model = copy.deepcopy(model) @@ -292,8 +292,7 @@ def main_worker(gpu, ngpus_per_node): eval_and_print(validate, data.val_loader, cp_model, criterion, parser_args, writer=None, description='final model after finetuning') # save checkpoint after fine-tuning - torch.save(cp_model.state_dict(), result_root + - 'model_after_finetune.pth') + #torch.save(cp_model.state_dict(), result_root + 'model_after_finetune.pth') else: print("Skipping finetuning!!!") diff --git a/main_utils.py b/main_utils.py index 0ab9a80b..d9656c33 100644 --- a/main_utils.py +++ b/main_utils.py @@ -208,16 +208,16 @@ def test_random_subnet(model, data, criterion, parser_args, result_root, smart_r model = redraw(model, shuffle=parser_args.shuffle, reinit=parser_args.reinit, chg_mask=parser_args.chg_mask, chg_weight=parser_args.chg_weight) model_filename = result_root + 'model_before_finetune.pth' - print("Writing init model to {}".format(model_filename)) - torch.save(model.state_dict(), model_filename) + #print("Writing init model to {}".format(model_filename)) + #torch.save(model.state_dict(), model_filename) old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_reg_loss_list, old_model_sparsity_list = [], [], [], [], [] model = finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_reg_loss_list, old_model_sparsity_list, result_root, shuffle=False, reinit=False, invert=False, chg_mask=False, chg_weight=False) # save checkpoint for later debug model_filename = result_root + 'model_after_finetune.pth' - print("Writing final model to {}".format(model_filename)) - torch.save(model.state_dict(), model_filename) + #print("Writing final model to {}".format(model_filename)) + #torch.save(model.state_dict(), model_filename) def eval_and_print(validate, data_loader, model, criterion, parser_args, writer=None, epoch=parser_args.start_epoch, description='model'): From 53da0ddaf664e00cb1c795824137d098afadda82 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 7 Apr 2022 18:47:02 -0500 Subject: [PATCH 02/72] adding exec file for full runs --- cifar_exec.sh | 31 +++++++++++++++++-- .../param_tuning/resnet20_059_KS/conf4.yml | 2 +- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 3005e4b5..8ce39dea 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -61,8 +61,8 @@ python main.py \ --config configs/hypercube/wideresnet28/wideresnet28_weight_training.yml > wideresnet_wt_log 2>&1 BLOCK - -# Running trials in parallel +:< "$log_root$trial$log_end" 2>&1 & #python main.py \ @@ -83,3 +82,29 @@ do #--invert-sanity-check \ #--subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & done +BLOCK + +# Final run on full data +# NOTE: make sure to delete/comment subfolder from the config file or else it may not work +conf_file="configs/param_tuning/resnet20_059_KS/conf4" +conf_end=".yml" +log_root="resnet20_059_" +log_end="_log" +subfolder_root="resnet20_059_" + +for trial in 1 2 3 +do + python main.py \ + --config "$conf_file$conf_end" \ + --trial-num $trial \ + --use-full-data \ + --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 & + + python main.py \ + --config "$conf_file" \ + --trial-num $trial \ + --invert-sanity-check \ + --use-full-data \ + --skip-sanity-checks \ + --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +done \ No newline at end of file diff --git a/configs/param_tuning/resnet20_059_KS/conf4.yml b/configs/param_tuning/resnet20_059_KS/conf4.yml index d4a7591a..ec3a4174 100644 --- a/configs/param_tuning/resnet20_059_KS/conf4.yml +++ b/configs/param_tuning/resnet20_059_KS/conf4.yml @@ -60,4 +60,4 @@ gpu: 1 checkpoint_at_prune: False # ==== sanity check ==== # -skip_sanity_checks: True +# skip_sanity_checks: True From f07f25674f78d88f38f481b25e948687ad2fe11b Mon Sep 17 00:00:00 2001 From: root Date: Thu, 7 Apr 2022 23:50:03 +0000 Subject: [PATCH 03/72] config for cifar10 --- configs/param_tuning/mobile_1_4/conf1.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_1_4/conf2.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_1_4/conf3.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_1_4/conf4.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_1_4/conf5.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_1_4/conf6.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_20/conf1.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_20/conf2.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_20/conf3.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_20/conf4.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_20/conf5.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_20/conf6.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_5/conf1.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_5/conf2.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_5/conf3.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_5/conf4.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_5/conf5.yml | 61 +++++++++++++++++++ configs/param_tuning/mobile_5/conf6.yml | 61 +++++++++++++++++++ configs/param_tuning/vgg_0_5/conf1.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_0_5/conf2.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_0_5/conf3.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_0_5/conf4.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_0_5/conf5.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_0_5/conf6.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_1_4/conf1.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_1_4/conf2.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_1_4/conf3.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_1_4/conf4.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_1_4/conf5.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_1_4/conf6.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_2_5/conf1.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_2_5/conf2.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_2_5/conf3.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_2_5/conf4.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_2_5/conf5.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_2_5/conf6.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_5/conf1.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_5/conf2.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_5/conf3.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_5/conf4.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_5/conf5.yml | 59 ++++++++++++++++++ configs/param_tuning/vgg_5/conf6.yml | 59 ++++++++++++++++++ .../param_tuning/wideresnet28_1_4/conf1.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_1_4/conf2.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_1_4/conf3.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_1_4/conf4.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_1_4/conf5.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_1_4/conf6.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_20/conf1.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_20/conf2.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_20/conf3.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_20/conf4.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_20/conf5.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_20/conf6.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_2_3/conf1.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_2_3/conf2.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_2_3/conf3.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_2_3/conf4.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_2_3/conf5.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_2_3/conf6.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_5_5/conf1.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_5_5/conf2.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_5_5/conf3.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_5_5/conf4.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_5_5/conf5.yml | 61 +++++++++++++++++++ .../param_tuning/wideresnet28_5_5/conf6.yml | 61 +++++++++++++++++++ 66 files changed, 3978 insertions(+) create mode 100644 configs/param_tuning/mobile_1_4/conf1.yml create mode 100644 configs/param_tuning/mobile_1_4/conf2.yml create mode 100644 configs/param_tuning/mobile_1_4/conf3.yml create mode 100644 configs/param_tuning/mobile_1_4/conf4.yml create mode 100644 configs/param_tuning/mobile_1_4/conf5.yml create mode 100644 configs/param_tuning/mobile_1_4/conf6.yml create mode 100644 configs/param_tuning/mobile_20/conf1.yml create mode 100644 configs/param_tuning/mobile_20/conf2.yml create mode 100644 configs/param_tuning/mobile_20/conf3.yml create mode 100644 configs/param_tuning/mobile_20/conf4.yml create mode 100644 configs/param_tuning/mobile_20/conf5.yml create mode 100644 configs/param_tuning/mobile_20/conf6.yml create mode 100644 configs/param_tuning/mobile_5/conf1.yml create mode 100644 configs/param_tuning/mobile_5/conf2.yml create mode 100644 configs/param_tuning/mobile_5/conf3.yml create mode 100644 configs/param_tuning/mobile_5/conf4.yml create mode 100644 configs/param_tuning/mobile_5/conf5.yml create mode 100644 configs/param_tuning/mobile_5/conf6.yml create mode 100644 configs/param_tuning/vgg_0_5/conf1.yml create mode 100644 configs/param_tuning/vgg_0_5/conf2.yml create mode 100644 configs/param_tuning/vgg_0_5/conf3.yml create mode 100644 configs/param_tuning/vgg_0_5/conf4.yml create mode 100644 configs/param_tuning/vgg_0_5/conf5.yml create mode 100644 configs/param_tuning/vgg_0_5/conf6.yml create mode 100644 configs/param_tuning/vgg_1_4/conf1.yml create mode 100644 configs/param_tuning/vgg_1_4/conf2.yml create mode 100644 configs/param_tuning/vgg_1_4/conf3.yml create mode 100644 configs/param_tuning/vgg_1_4/conf4.yml create mode 100644 configs/param_tuning/vgg_1_4/conf5.yml create mode 100644 configs/param_tuning/vgg_1_4/conf6.yml create mode 100644 configs/param_tuning/vgg_2_5/conf1.yml create mode 100644 configs/param_tuning/vgg_2_5/conf2.yml create mode 100644 configs/param_tuning/vgg_2_5/conf3.yml create mode 100644 configs/param_tuning/vgg_2_5/conf4.yml create mode 100644 configs/param_tuning/vgg_2_5/conf5.yml create mode 100644 configs/param_tuning/vgg_2_5/conf6.yml create mode 100644 configs/param_tuning/vgg_5/conf1.yml create mode 100644 configs/param_tuning/vgg_5/conf2.yml create mode 100644 configs/param_tuning/vgg_5/conf3.yml create mode 100644 configs/param_tuning/vgg_5/conf4.yml create mode 100644 configs/param_tuning/vgg_5/conf5.yml create mode 100644 configs/param_tuning/vgg_5/conf6.yml create mode 100644 configs/param_tuning/wideresnet28_1_4/conf1.yml create mode 100644 configs/param_tuning/wideresnet28_1_4/conf2.yml create mode 100644 configs/param_tuning/wideresnet28_1_4/conf3.yml create mode 100644 configs/param_tuning/wideresnet28_1_4/conf4.yml create mode 100644 configs/param_tuning/wideresnet28_1_4/conf5.yml create mode 100644 configs/param_tuning/wideresnet28_1_4/conf6.yml create mode 100644 configs/param_tuning/wideresnet28_20/conf1.yml create mode 100644 configs/param_tuning/wideresnet28_20/conf2.yml create mode 100644 configs/param_tuning/wideresnet28_20/conf3.yml create mode 100644 configs/param_tuning/wideresnet28_20/conf4.yml create mode 100644 configs/param_tuning/wideresnet28_20/conf5.yml create mode 100644 configs/param_tuning/wideresnet28_20/conf6.yml create mode 100644 configs/param_tuning/wideresnet28_2_3/conf1.yml create mode 100644 configs/param_tuning/wideresnet28_2_3/conf2.yml create mode 100644 configs/param_tuning/wideresnet28_2_3/conf3.yml create mode 100644 configs/param_tuning/wideresnet28_2_3/conf4.yml create mode 100644 configs/param_tuning/wideresnet28_2_3/conf5.yml create mode 100644 configs/param_tuning/wideresnet28_2_3/conf6.yml create mode 100644 configs/param_tuning/wideresnet28_5_5/conf1.yml create mode 100644 configs/param_tuning/wideresnet28_5_5/conf2.yml create mode 100644 configs/param_tuning/wideresnet28_5_5/conf3.yml create mode 100644 configs/param_tuning/wideresnet28_5_5/conf4.yml create mode 100644 configs/param_tuning/wideresnet28_5_5/conf5.yml create mode 100644 configs/param_tuning/wideresnet28_5_5/conf6.yml diff --git a/configs/param_tuning/mobile_1_4/conf1.yml b/configs/param_tuning/mobile_1_4/conf1.yml new file mode 100644 index 00000000..b71b472b --- /dev/null +++ b/configs/param_tuning/mobile_1_4/conf1.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_1_4_re +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000015 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_1_4/conf2.yml b/configs/param_tuning/mobile_1_4/conf2.yml new file mode 100644 index 00000000..8e36841e --- /dev/null +++ b/configs/param_tuning/mobile_1_4/conf2.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_1_4_re +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_1_4/conf3.yml b/configs/param_tuning/mobile_1_4/conf3.yml new file mode 100644 index 00000000..b71d26a5 --- /dev/null +++ b/configs/param_tuning/mobile_1_4/conf3.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_1_4_re +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_1_4/conf4.yml b/configs/param_tuning/mobile_1_4/conf4.yml new file mode 100644 index 00000000..3edbe7fc --- /dev/null +++ b/configs/param_tuning/mobile_1_4/conf4.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_1_4_re +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000015 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_1_4/conf5.yml b/configs/param_tuning/mobile_1_4/conf5.yml new file mode 100644 index 00000000..ff3562ab --- /dev/null +++ b/configs/param_tuning/mobile_1_4/conf5.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_1_4_re +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_1_4/conf6.yml b/configs/param_tuning/mobile_1_4/conf6.yml new file mode 100644 index 00000000..5ee5fd15 --- /dev/null +++ b/configs/param_tuning/mobile_1_4/conf6.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_1_4_re +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_20/conf1.yml b/configs/param_tuning/mobile_20/conf1.yml new file mode 100644 index 00000000..556c4945 --- /dev/null +++ b/configs/param_tuning/mobile_20/conf1.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_20_3lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000003 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_20/conf2.yml b/configs/param_tuning/mobile_20/conf2.yml new file mode 100644 index 00000000..f2ed9201 --- /dev/null +++ b/configs/param_tuning/mobile_20/conf2.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_20_3lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_20/conf3.yml b/configs/param_tuning/mobile_20/conf3.yml new file mode 100644 index 00000000..922441a9 --- /dev/null +++ b/configs/param_tuning/mobile_20/conf3.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_20_3lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_20/conf4.yml b/configs/param_tuning/mobile_20/conf4.yml new file mode 100644 index 00000000..b4d756e8 --- /dev/null +++ b/configs/param_tuning/mobile_20/conf4.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_20_3lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000003 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_20/conf5.yml b/configs/param_tuning/mobile_20/conf5.yml new file mode 100644 index 00000000..4bc4884c --- /dev/null +++ b/configs/param_tuning/mobile_20/conf5.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_20_3lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_20/conf6.yml b/configs/param_tuning/mobile_20/conf6.yml new file mode 100644 index 00000000..a3e88bee --- /dev/null +++ b/configs/param_tuning/mobile_20/conf6.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_20_3lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_5/conf1.yml b/configs/param_tuning/mobile_5/conf1.yml new file mode 100644 index 00000000..896cbf6b --- /dev/null +++ b/configs/param_tuning/mobile_5/conf1.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_5_7lam6_unflag_True +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000007 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_5/conf2.yml b/configs/param_tuning/mobile_5/conf2.yml new file mode 100644 index 00000000..1b13a073 --- /dev/null +++ b/configs/param_tuning/mobile_5/conf2.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_5_7lam6_unflag_True +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_5/conf3.yml b/configs/param_tuning/mobile_5/conf3.yml new file mode 100644 index 00000000..f4b082e2 --- /dev/null +++ b/configs/param_tuning/mobile_5/conf3.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_5_7lam6_unflag_True +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_5/conf4.yml b/configs/param_tuning/mobile_5/conf4.yml new file mode 100644 index 00000000..3316402d --- /dev/null +++ b/configs/param_tuning/mobile_5/conf4.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_5_7lam6_unflag_True +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000007 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_5/conf5.yml b/configs/param_tuning/mobile_5/conf5.yml new file mode 100644 index 00000000..d03481c4 --- /dev/null +++ b/configs/param_tuning/mobile_5/conf5.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_5_7lam6_unflag_True +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/mobile_5/conf6.yml b/configs/param_tuning/mobile_5/conf6.yml new file mode 100644 index 00000000..33f414d5 --- /dev/null +++ b/configs/param_tuning/mobile_5/conf6.yml @@ -0,0 +1,61 @@ +#subfolder: mobilenetV2_hc_sparsity_5_7lam6_unflag_True +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: MobileNetV2 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_0_5/conf1.yml b/configs/param_tuning/vgg_0_5/conf1.yml new file mode 100644 index 00000000..a7da84e9 --- /dev/null +++ b/configs/param_tuning/vgg_0_5/conf1.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_0_5/conf2.yml b/configs/param_tuning/vgg_0_5/conf2.yml new file mode 100644 index 00000000..5e721d2b --- /dev/null +++ b/configs/param_tuning/vgg_0_5/conf2.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_0_5/conf3.yml b/configs/param_tuning/vgg_0_5/conf3.yml new file mode 100644 index 00000000..969c0621 --- /dev/null +++ b/configs/param_tuning/vgg_0_5/conf3.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_0_5/conf4.yml b/configs/param_tuning/vgg_0_5/conf4.yml new file mode 100644 index 00000000..6d13f97e --- /dev/null +++ b/configs/param_tuning/vgg_0_5/conf4.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_0_5/conf5.yml b/configs/param_tuning/vgg_0_5/conf5.yml new file mode 100644 index 00000000..08676a1b --- /dev/null +++ b/configs/param_tuning/vgg_0_5/conf5.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_0_5/conf6.yml b/configs/param_tuning/vgg_0_5/conf6.yml new file mode 100644 index 00000000..4219db64 --- /dev/null +++ b/configs/param_tuning/vgg_0_5/conf6.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_1_4/conf1.yml b/configs/param_tuning/vgg_1_4/conf1.yml new file mode 100644 index 00000000..39e27185 --- /dev/null +++ b/configs/param_tuning/vgg_1_4/conf1.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_1_4/conf2.yml b/configs/param_tuning/vgg_1_4/conf2.yml new file mode 100644 index 00000000..50f1304c --- /dev/null +++ b/configs/param_tuning/vgg_1_4/conf2.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_1_4/conf3.yml b/configs/param_tuning/vgg_1_4/conf3.yml new file mode 100644 index 00000000..37cfb187 --- /dev/null +++ b/configs/param_tuning/vgg_1_4/conf3.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_1_4/conf4.yml b/configs/param_tuning/vgg_1_4/conf4.yml new file mode 100644 index 00000000..86135e84 --- /dev/null +++ b/configs/param_tuning/vgg_1_4/conf4.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_1_4/conf5.yml b/configs/param_tuning/vgg_1_4/conf5.yml new file mode 100644 index 00000000..db3662d1 --- /dev/null +++ b/configs/param_tuning/vgg_1_4/conf5.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_1_4/conf6.yml b/configs/param_tuning/vgg_1_4/conf6.yml new file mode 100644 index 00000000..81d0e08a --- /dev/null +++ b/configs/param_tuning/vgg_1_4/conf6.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_2_5/conf1.yml b/configs/param_tuning/vgg_2_5/conf1.yml new file mode 100644 index 00000000..bf6c70e0 --- /dev/null +++ b/configs/param_tuning/vgg_2_5/conf1.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_2_5/conf2.yml b/configs/param_tuning/vgg_2_5/conf2.yml new file mode 100644 index 00000000..f75f651d --- /dev/null +++ b/configs/param_tuning/vgg_2_5/conf2.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_2_5/conf3.yml b/configs/param_tuning/vgg_2_5/conf3.yml new file mode 100644 index 00000000..de2d2820 --- /dev/null +++ b/configs/param_tuning/vgg_2_5/conf3.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_2_5/conf4.yml b/configs/param_tuning/vgg_2_5/conf4.yml new file mode 100644 index 00000000..3be8894f --- /dev/null +++ b/configs/param_tuning/vgg_2_5/conf4.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_2_5/conf5.yml b/configs/param_tuning/vgg_2_5/conf5.yml new file mode 100644 index 00000000..92f715a3 --- /dev/null +++ b/configs/param_tuning/vgg_2_5/conf5.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_2_5/conf6.yml b/configs/param_tuning/vgg_2_5/conf6.yml new file mode 100644 index 00000000..57a49967 --- /dev/null +++ b/configs/param_tuning/vgg_2_5/conf6.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_5/conf1.yml b/configs/param_tuning/vgg_5/conf1.yml new file mode 100644 index 00000000..e2fabea8 --- /dev/null +++ b/configs/param_tuning/vgg_5/conf1.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_5/conf2.yml b/configs/param_tuning/vgg_5/conf2.yml new file mode 100644 index 00000000..cd4d4521 --- /dev/null +++ b/configs/param_tuning/vgg_5/conf2.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_5/conf3.yml b/configs/param_tuning/vgg_5/conf3.yml new file mode 100644 index 00000000..d4bd90fa --- /dev/null +++ b/configs/param_tuning/vgg_5/conf3.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_5/conf4.yml b/configs/param_tuning/vgg_5/conf4.yml new file mode 100644 index 00000000..5b5389d7 --- /dev/null +++ b/configs/param_tuning/vgg_5/conf4.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_5/conf5.yml b/configs/param_tuning/vgg_5/conf5.yml new file mode 100644 index 00000000..b360ce19 --- /dev/null +++ b/configs/param_tuning/vgg_5/conf5.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/vgg_5/conf6.yml b/configs/param_tuning/vgg_5/conf6.yml new file mode 100644 index 00000000..747567aa --- /dev/null +++ b/configs/param_tuning/vgg_5/conf6.yml @@ -0,0 +1,59 @@ +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: vgg16 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: vgg16_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.05 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant #signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_1_4/conf1.yml b/configs/param_tuning/wideresnet28_1_4/conf1.yml new file mode 100644 index 00000000..69cad0fb --- /dev/null +++ b/configs/param_tuning/wideresnet28_1_4/conf1.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_1_4/conf2.yml b/configs/param_tuning/wideresnet28_1_4/conf2.yml new file mode 100644 index 00000000..53b8be0a --- /dev/null +++ b/configs/param_tuning/wideresnet28_1_4/conf2.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_1_4/conf3.yml b/configs/param_tuning/wideresnet28_1_4/conf3.yml new file mode 100644 index 00000000..8b286dd6 --- /dev/null +++ b/configs/param_tuning/wideresnet28_1_4/conf3.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_1_4/conf4.yml b/configs/param_tuning/wideresnet28_1_4/conf4.yml new file mode 100644 index 00000000..a8681fa3 --- /dev/null +++ b/configs/param_tuning/wideresnet28_1_4/conf4.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_1_4/conf5.yml b/configs/param_tuning/wideresnet28_1_4/conf5.yml new file mode 100644 index 00000000..9d3cd3ef --- /dev/null +++ b/configs/param_tuning/wideresnet28_1_4/conf5.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_1_4/conf6.yml b/configs/param_tuning/wideresnet28_1_4/conf6.yml new file mode 100644 index 00000000..3fcdbeab --- /dev/null +++ b/configs/param_tuning/wideresnet28_1_4/conf6.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_20/conf1.yml b/configs/param_tuning/wideresnet28_20/conf1.yml new file mode 100644 index 00000000..b08a61ac --- /dev/null +++ b/configs/param_tuning/wideresnet28_20/conf1.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.5 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_20/conf2.yml b/configs/param_tuning/wideresnet28_20/conf2.yml new file mode 100644 index 00000000..28f98f4a --- /dev/null +++ b/configs/param_tuning/wideresnet28_20/conf2.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_20/conf3.yml b/configs/param_tuning/wideresnet28_20/conf3.yml new file mode 100644 index 00000000..9462f1d2 --- /dev/null +++ b/configs/param_tuning/wideresnet28_20/conf3.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_20/conf4.yml b/configs/param_tuning/wideresnet28_20/conf4.yml new file mode 100644 index 00000000..6be68095 --- /dev/null +++ b/configs/param_tuning/wideresnet28_20/conf4.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_20/conf5.yml b/configs/param_tuning/wideresnet28_20/conf5.yml new file mode 100644 index 00000000..5b971f08 --- /dev/null +++ b/configs/param_tuning/wideresnet28_20/conf5.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_20/conf6.yml b/configs/param_tuning/wideresnet28_20/conf6.yml new file mode 100644 index 00000000..777fc186 --- /dev/null +++ b/configs/param_tuning/wideresnet28_20/conf6.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_2_3/conf1.yml b/configs/param_tuning/wideresnet28_2_3/conf1.yml new file mode 100644 index 00000000..a27d52ab --- /dev/null +++ b/configs/param_tuning/wideresnet28_2_3/conf1.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.3 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_2_3/conf2.yml b/configs/param_tuning/wideresnet28_2_3/conf2.yml new file mode 100644 index 00000000..572c1844 --- /dev/null +++ b/configs/param_tuning/wideresnet28_2_3/conf2.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.3 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_2_3/conf3.yml b/configs/param_tuning/wideresnet28_2_3/conf3.yml new file mode 100644 index 00000000..d5ae0ba2 --- /dev/null +++ b/configs/param_tuning/wideresnet28_2_3/conf3.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.3 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_2_3/conf4.yml b/configs/param_tuning/wideresnet28_2_3/conf4.yml new file mode 100644 index 00000000..6499210a --- /dev/null +++ b/configs/param_tuning/wideresnet28_2_3/conf4.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.3 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_2_3/conf5.yml b/configs/param_tuning/wideresnet28_2_3/conf5.yml new file mode 100644 index 00000000..81845d38 --- /dev/null +++ b/configs/param_tuning/wideresnet28_2_3/conf5.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.3 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_2_3/conf6.yml b/configs/param_tuning/wideresnet28_2_3/conf6.yml new file mode 100644 index 00000000..633b5d48 --- /dev/null +++ b/configs/param_tuning/wideresnet28_2_3/conf6.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.3 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_5_5/conf1.yml b/configs/param_tuning/wideresnet28_5_5/conf1.yml new file mode 100644 index 00000000..8e4a7311 --- /dev/null +++ b/configs/param_tuning/wideresnet28_5_5/conf1.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_5_5/conf2.yml b/configs/param_tuning/wideresnet28_5_5/conf2.yml new file mode 100644 index 00000000..4ad55b44 --- /dev/null +++ b/configs/param_tuning/wideresnet28_5_5/conf2.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.5 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_5_5/conf3.yml b/configs/param_tuning/wideresnet28_5_5/conf3.yml new file mode 100644 index 00000000..4bcf7186 --- /dev/null +++ b/configs/param_tuning/wideresnet28_5_5/conf3.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.5 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.5 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_5_5/conf4.yml b/configs/param_tuning/wideresnet28_5_5/conf4.yml new file mode 100644 index 00000000..2f932551 --- /dev/null +++ b/configs/param_tuning/wideresnet28_5_5/conf4.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.5 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_5_5/conf5.yml b/configs/param_tuning/wideresnet28_5_5/conf5.yml new file mode 100644 index 00000000..e8ce385a --- /dev/null +++ b/configs/param_tuning/wideresnet28_5_5/conf5.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.5 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file diff --git a/configs/param_tuning/wideresnet28_5_5/conf6.yml b/configs/param_tuning/wideresnet28_5_5/conf6.yml new file mode 100644 index 00000000..41631c85 --- /dev/null +++ b/configs/param_tuning/wideresnet28_5_5/conf6.yml @@ -0,0 +1,61 @@ +#subfolder: wideresnet + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 10 + +# Architecture +arch: WideResNet28 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: wideresnet28_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.5 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True +#invert_sanity_check: True \ No newline at end of file From f9102513978f230b8d8960661dd3bca711a6b101 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 8 Apr 2022 01:24:28 +0000 Subject: [PATCH 04/72] minor fix to exec script --- cifar_exec.sh | 4 ++-- configs/param_tuning/resnet20_059_KS/conf4.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 8ce39dea..370be572 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -101,10 +101,10 @@ do --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 & python main.py \ - --config "$conf_file" \ + --config "$conf_file$conf_end" \ --trial-num $trial \ --invert-sanity-check \ --use-full-data \ --skip-sanity-checks \ --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & -done \ No newline at end of file +done diff --git a/configs/param_tuning/resnet20_059_KS/conf4.yml b/configs/param_tuning/resnet20_059_KS/conf4.yml index ec3a4174..e1d57a4f 100644 --- a/configs/param_tuning/resnet20_059_KS/conf4.yml +++ b/configs/param_tuning/resnet20_059_KS/conf4.yml @@ -54,7 +54,7 @@ lmbda: 0.0001 # 1e-4 # ===== Hardware setup ===== # workers: 4 -gpu: 1 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False From 648e8cd779f16fe8aa7b8af94d4cde8492c5a924 Mon Sep 17 00:00:00 2001 From: jysohn1108 Date: Fri, 8 Apr 2022 02:08:11 +0000 Subject: [PATCH 05/72] run cifar10 hparam selection and final run --- cifar_exec_GD.sh | 127 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 2 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 01ade60b..e3791018 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -1,4 +1,127 @@ +# REBUTTAL +# NOTE: make sure to delete/comment subfolder from the config file or else it may not work +#:< "$log_root$trial$log_end" 2>&1 #& +done + +#BLOCK +# Final run on full data +# # NOTE: make sure to delete/comment subfolder from the config file or else it may not work +# conf_file="configs/param_tuning/resnet20_059_KS/conf4" +# conf_end=".yml" +# log_root="resnet20_059_" +# log_end="_log" +# subfolder_root="resnet20_059_" + +# for trial in 1 2 3 +# do +# python main.py \ +# --config "$conf_file$conf_end" \ +# --trial-num $trial \ +# --use-full-data \ +# --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 & + +# python main.py \ +# --config "$conf_file" \ +# --trial-num $trial \ +# --invert-sanity-check \ +# --use-full-data \ +# --skip-sanity-checks \ +# --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +# done + + + + + + + + + + + + + + + + + +:< "$log_root$trial$log_end" 2>&1 & + + #python main.py \ + #--config "$conf_file" \ + #--trial-num $trial \ + #--invert-sanity-check \ + #--subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +done +BLOCK + +# # Final run on full data +# # NOTE: make sure to delete/comment subfolder from the config file or else it may not work +# conf_file="configs/param_tuning/resnet20_059_KS/conf4" +# conf_end=".yml" +# log_root="resnet20_059_" +# log_end="_log" +# subfolder_root="resnet20_059_" + +# for trial in 1 2 3 +# do +# python main.py \ +# --config "$conf_file$conf_end" \ +# --trial-num $trial \ +# --use-full-data \ +# --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 & + +# python main.py \ +# --config "$conf_file" \ +# --trial-num $trial \ +# --invert-sanity-check \ +# --use-full-data \ +# --skip-sanity-checks \ +# --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +# done + + + + + + + + + + + + + + + # VGG16, 0.5%, bf_ft_acc vs af_ft_acc :< "$subfolder_root$log_end" 2>&1 & -#BLOCK +BLOCK From 4ba4f2196fdfcc3b59b129c5ea63a453cafb74e4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 8 Apr 2022 05:27:20 +0000 Subject: [PATCH 06/72] resnet18 best config --- .../tinyimgnet/resnet18_0_5/conf1.yml | 62 +++++++++++++++++++ .../tinyimgnet/resnet18_1_4/conf1.yml | 62 +++++++++++++++++++ .../tinyimgnet/resnet18_5/conf1.yml | 62 +++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 configs/param_tuning/tinyimgnet/resnet18_0_5/conf1.yml create mode 100644 configs/param_tuning/tinyimgnet/resnet18_1_4/conf1.yml create mode 100644 configs/param_tuning/tinyimgnet/resnet18_5/conf1.yml diff --git a/configs/param_tuning/tinyimgnet/resnet18_0_5/conf1.yml b/configs/param_tuning/tinyimgnet/resnet18_0_5/conf1.yml new file mode 100644 index 00000000..df7596cb --- /dev/null +++ b/configs/param_tuning/tinyimgnet/resnet18_0_5/conf1.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_hc_sparsity_0_5_adam_1lam5 +trial_num: 1 + + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: TinyResNet18 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_resnet18_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0 +momentum: 0.9 +batch_size: 256 # 256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.5 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +#skip_sanity_checks: True diff --git a/configs/param_tuning/tinyimgnet/resnet18_1_4/conf1.yml b/configs/param_tuning/tinyimgnet/resnet18_1_4/conf1.yml new file mode 100644 index 00000000..6bfde19e --- /dev/null +++ b/configs/param_tuning/tinyimgnet/resnet18_1_4/conf1.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_hc_sparsity_1_4_adam_5lam6 +trial_num: 1 + + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: TinyResNet18 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_resnet18_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0 +momentum: 0.9 +batch_size: 256 # 256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +#skip_sanity_checks: True diff --git a/configs/param_tuning/tinyimgnet/resnet18_5/conf1.yml b/configs/param_tuning/tinyimgnet/resnet18_5/conf1.yml new file mode 100644 index 00000000..e4202ea4 --- /dev/null +++ b/configs/param_tuning/tinyimgnet/resnet18_5/conf1.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_hc_sparsity_5_adam_8lam6 +trial_num: 1 + + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: TinyResNet18 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_resnet18_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0 +momentum: 0.9 +batch_size: 256 # 256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +#skip_sanity_checks: True From 070aabcc553436b0ca4614d8114897cbcd71c592 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 8 Apr 2022 05:34:33 +0000 Subject: [PATCH 07/72] resnet50, tinyimgnet, best hparam --- .../tinyimgnet/resnet50_0_5/conf1.yml | 63 +++++++++++++++++++ .../tinyimgnet/resnet50_1_4/conf1.yml | 63 +++++++++++++++++++ .../tinyimgnet/resnet50_5/conf1.yml | 63 +++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 configs/param_tuning/tinyimgnet/resnet50_0_5/conf1.yml create mode 100644 configs/param_tuning/tinyimgnet/resnet50_1_4/conf1.yml create mode 100644 configs/param_tuning/tinyimgnet/resnet50_5/conf1.yml diff --git a/configs/param_tuning/tinyimgnet/resnet50_0_5/conf1.yml b/configs/param_tuning/tinyimgnet/resnet50_0_5/conf1.yml new file mode 100644 index 00000000..18137e11 --- /dev/null +++ b/configs/param_tuning/tinyimgnet/resnet50_0_5/conf1.yml @@ -0,0 +1,63 @@ +#subfolder: resnet50_HC_50 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_resnet50_HC + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr +fine_tune_optimizer: sgd +nesterov: True + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 #0.0001 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK + +target_sparsity: 0.5 + +unflag_before_finetune: True +init: signed_constant +score_init: unif +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 #0.000001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +#skip_sanity_checks: False diff --git a/configs/param_tuning/tinyimgnet/resnet50_1_4/conf1.yml b/configs/param_tuning/tinyimgnet/resnet50_1_4/conf1.yml new file mode 100644 index 00000000..d06edcf4 --- /dev/null +++ b/configs/param_tuning/tinyimgnet/resnet50_1_4/conf1.yml @@ -0,0 +1,63 @@ +#subfolder: resnet50_HC_50 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_resnet50_HC + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr +fine_tune_optimizer: sgd +nesterov: True + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 #0.0001 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK + +target_sparsity: 1.4 + +unflag_before_finetune: True +init: signed_constant +score_init: unif +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 #0.000001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +#skip_sanity_checks: False diff --git a/configs/param_tuning/tinyimgnet/resnet50_5/conf1.yml b/configs/param_tuning/tinyimgnet/resnet50_5/conf1.yml new file mode 100644 index 00000000..427f415f --- /dev/null +++ b/configs/param_tuning/tinyimgnet/resnet50_5/conf1.yml @@ -0,0 +1,63 @@ +#subfolder: resnet50_HC_50 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_resnet50_HC + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr +fine_tune_optimizer: sgd +nesterov: True + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 #0.0001 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK + +target_sparsity: 5 + +unflag_before_finetune: True +init: signed_constant +score_init: unif +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 #0.000001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +#skip_sanity_checks: False From a69ab8c8a6ac19f3fa5364c6843dd639bc90b347 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 8 Apr 2022 05:40:20 +0000 Subject: [PATCH 08/72] caltech101 --- .../caltech101/resnet50_2/conf1.yml | 67 +++++++++++++++++++ .../caltech101/resnet50_5/conf1.yml | 67 +++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 configs/param_tuning/caltech101/resnet50_2/conf1.yml create mode 100644 configs/param_tuning/caltech101/resnet50_5/conf1.yml diff --git a/configs/param_tuning/caltech101/resnet50_2/conf1.yml b/configs/param_tuning/caltech101/resnet50_2/conf1.yml new file mode 100644 index 00000000..63f7e1c9 --- /dev/null +++ b/configs/param_tuning/caltech101/resnet50_2/conf1.yml @@ -0,0 +1,67 @@ + +# algorithm +algo: 'hc_iter' +iter_period: 5 + + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: Caltech101 +name: resnet50_caltech101_HC +transfer_learning: True + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 +lr_policy: cosine_lr #cosine_lr #constant_lr +fine_tune_optimizer: adam +fine_tune_lr: 0.0001 +fine_tune_lr_policy: multistep_lr #cosine_lr #constant_lr + +# ===== Network training config ===== # +epochs: 50 #5 +wd: 0 +momentum: 0.9 +batch_size: 16 + + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: AffineBatchNorm #NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +target_sparsity: 2 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +#skip_sanity_checks: True + + diff --git a/configs/param_tuning/caltech101/resnet50_5/conf1.yml b/configs/param_tuning/caltech101/resnet50_5/conf1.yml new file mode 100644 index 00000000..cfe70376 --- /dev/null +++ b/configs/param_tuning/caltech101/resnet50_5/conf1.yml @@ -0,0 +1,67 @@ + +# algorithm +algo: 'hc_iter' +iter_period: 5 + + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: Caltech101 +name: resnet50_caltech101_HC +transfer_learning: True + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 +lr_policy: cosine_lr #cosine_lr #constant_lr +fine_tune_optimizer: adam +fine_tune_lr: 0.0001 +fine_tune_lr_policy: multistep_lr #cosine_lr #constant_lr + +# ===== Network training config ===== # +epochs: 50 #5 +wd: 0 +momentum: 0.9 +batch_size: 16 + + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: AffineBatchNorm #NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +target_sparsity: 5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000002 + + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +#skip_sanity_checks: True + + From 3b288d32f0fb841406d71431711c93d73641983b Mon Sep 17 00:00:00 2001 From: jysohn1108 Date: Fri, 8 Apr 2022 06:10:03 +0000 Subject: [PATCH 09/72] val/train/test for tinyimgnet --- data/tinyimagenet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/tinyimagenet.py b/data/tinyimagenet.py index 9827ee53..2d59ae82 100644 --- a/data/tinyimagenet.py +++ b/data/tinyimagenet.py @@ -70,7 +70,7 @@ def __init__(self, args): self.val_loader = torch.utils.data.DataLoader( datasets.ImageFolder( - testdir, # valdir, #TODO: change here + testdir, transforms.Compose( [ transforms.ToTensor(), @@ -83,9 +83,9 @@ def __init__(self, args): **kwargs ) - self.test_loader = torch.utils.data.DataLoader( + self.actual_val_loader = torch.utils.data.DataLoader( datasets.ImageFolder( - testdir, + valdir, transforms.Compose( [ transforms.ToTensor(), From c76961ed87d19bc7842c89c79049ecdbd323da06 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 8 Apr 2022 06:25:31 +0000 Subject: [PATCH 10/72] run final for tinyimgnet, resnet18 --- tinyimagenet_exec_GD.sh | 43 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index c32224f4..85c260c9 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -1,4 +1,45 @@ +## REBUTTAL +# Final run on full data +# # NOTE: make sure to delete/comment subfolder from the config file or else it may not work +conf_file="configs/param_tuning/tinyimgnet/resnet18_0_5/conf1" +conf_end=".yml" +log_root="tiny_resnet18_0_5" +log_end="_log" +subfolder_root="tiny_resnet18_0_5_" + +for trial in 1 2 3 +do + python main.py \ + --config "$conf_file$conf_end" \ + --trial-num $trial \ + --use-full-data \ + --subfolder "$subfolder_root$trial" #> "$log_root$trial$log_end" 2>&1 & + + python main.py \ + --config "$conf_file$conf_end" \ + --trial-num $trial \ + --invert-sanity-check \ + --use-full-data \ + --skip-sanity-checks \ + --subfolder "invert_$subfolder_root$trial" #> "invert_$log_root$trial$log_end" 2>&1 & +done + + + + + + + + + + + + + + + + # TinyImageNet, ResNet-50 @@ -6,7 +47,7 @@ #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_001_multi.yml > log_tiny_res50_wt_adam_001_multi 2>&1 # this is current best #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_001_cosine.yml > log_tiny_res50_wt_adam_001_cosine 2>&1 # this is current best #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_0001_cosine.yml > log_tiny_res50_wt_adam_0001_cosine 2>&1 # this is current best -python main.py --config configs/training/resnet50/tiny_resnet50_training_sgd_multi.yml > log_tiny_res50_wt_sgd_multi 2>&1 # this is current best +#python main.py --config configs/training/resnet50/tiny_resnet50_training_sgd_multi.yml > log_tiny_res50_wt_sgd_multi 2>&1 # this is current best From a42b9fd02a54eb423663bf27f53a3373439afd2f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 8 Apr 2022 20:14:56 +0000 Subject: [PATCH 11/72] adding code to print wallclock time at beginning, mid and end of process --- configs/param_tuning/resnet20_1_44/conf2.yml | 2 +- main.py | 8 ++++++++ main_utils.py | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/configs/param_tuning/resnet20_1_44/conf2.yml b/configs/param_tuning/resnet20_1_44/conf2.yml index 2a8c59e2..3e27c13b 100644 --- a/configs/param_tuning/resnet20_1_44/conf2.yml +++ b/configs/param_tuning/resnet20_1_44/conf2.yml @@ -61,4 +61,4 @@ workers: 4 checkpoint_at_prune: False # ==== sanity check ==== # -skip_sanity_checks: True \ No newline at end of file +# skip_sanity_checks: True diff --git a/main.py b/main.py index 87840cea..cc1af83e 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,8 @@ def main(): print(parser_args) + print("\n\nBeginning of process.") + print_time() set_seed(parser_args.seed * parser_args.trial_num) #set_seed(parser_args.seed + parser_args.trial_num - 1) @@ -282,6 +284,9 @@ def main_worker(gpu, ngpus_per_node): # save checkpoint before fine-tuning #torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') + print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") + print_time() + # finetune weights cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: @@ -303,6 +308,9 @@ def main_worker(gpu, ngpus_per_node): else: print("Skipping sanity checks!!!") + print("\n\nEnd of process. Exiting") + print_time() + if parser_args.multiprocessing_distributed: cleanup_distributed() diff --git a/main_utils.py b/main_utils.py index d9656c33..55187b37 100644 --- a/main_utils.py +++ b/main_utils.py @@ -1143,3 +1143,11 @@ def print_num_dataset(data): num_test += label.size()[0] print(num_train, num_val, num_test) + + +def print_time(): + print("\n\n--------------------------------------") + print("TIME: The current time is: {}".format(time.ctime())) + print("TIME: The current time in seconds is: {}".format(time.time())) + print("--------------------------------------\n\n") + From 2acc1bdd11f9c2e0f6d07c8fe85e19369e4ad45e Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 8 Apr 2022 16:55:07 -0500 Subject: [PATCH 12/72] adding print_time for IMP --- imp_main.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/imp_main.py b/imp_main.py index 0e95442a..6ff66002 100644 --- a/imp_main.py +++ b/imp_main.py @@ -21,7 +21,7 @@ # for merge the code into parent directory from args_helper import parser_args -from main_utils import get_model, get_dataset, get_optimizer, switch_to_wt, set_gpu +from main_utils import get_model, get_dataset, get_optimizer, switch_to_wt, set_gpu, print_time from utils.utils import set_seed from utils.schedulers import get_scheduler @@ -233,6 +233,9 @@ def print_nonzeros(model): optimizer = get_optimizer(parser_args, model) scheduler = get_scheduler(optimizer, parser_args.lr_policy, gamma=parser_args.lr_gamma) + print("\n\nFound ticket for sparsity: {}.".format(print_nonzeros(comp1))) + print_time() + # save the model and mask right after prune PATH_model = os.path.join(dest_dir, "round_{}_model.pth".format(idx_round)) torch.save({ @@ -297,6 +300,10 @@ def print_nonzeros(model): 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict() }, PATH_model_after) + + print("\n\nTrained ticket for sparsity: {}.".format(print_nonzeros(comp1))) + print_time() + return @@ -309,6 +316,9 @@ def main(): kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} set_seed(parser_args.seed) + print("\n\nBeginning of process.") + print_time() + if parser_args.arch in ['transformer']: # since the transformer code is not ready, just leave the code piece here, it will never go into this branch data = None @@ -317,6 +327,9 @@ def main(): IMP_train(parser_args, data, device) + print("\n\nEnd of process. Exiting") + print_time() + if __name__ == '__main__': From 86e6f590dbfb179730cd6b69c80f5c78da14e37f Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 8 Apr 2022 22:19:05 +0000 Subject: [PATCH 13/72] setting up imp for logging time --- imp_exec.sh | 9 ++++----- imp_main.py | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/imp_exec.sh b/imp_exec.sh index 10e24500..b7b3ec4f 100644 --- a/imp_exec.sh +++ b/imp_exec.sh @@ -1,14 +1,13 @@ # ===== warm short IMP ===== # -subfd="no_rewind_long_warm_imp_resnet20" +subfd="imp_resnet20_sp0_59" # subfd="cifar_resnet_check_sparse_mask_1_4_at_init" -n_gpu=1 +n_gpu=0 python imp_main.py \ --config configs/imp/resnet20.yml \ ---imp-no-rewind \ ---imp-rounds 20 \ +--imp-rounds 30 \ --gpu $n_gpu \ ---subfolder $subfd +--subfolder $subfd > imp_log 2>&1 diff --git a/imp_main.py b/imp_main.py index 6ff66002..8fc06b8b 100644 --- a/imp_main.py +++ b/imp_main.py @@ -233,7 +233,7 @@ def print_nonzeros(model): optimizer = get_optimizer(parser_args, model) scheduler = get_scheduler(optimizer, parser_args.lr_policy, gamma=parser_args.lr_gamma) - print("\n\nFound ticket for sparsity: {}.".format(print_nonzeros(comp1))) + print("\n\nFound ticket for sparsity: {}%".format(comp1)) print_time() # save the model and mask right after prune @@ -301,7 +301,7 @@ def print_nonzeros(model): 'optimizer_state_dict': optimizer.state_dict() }, PATH_model_after) - print("\n\nTrained ticket for sparsity: {}.".format(print_nonzeros(comp1))) + print("\n\nTrained ticket for sparsity: {}%".format(comp1)) print_time() return From ce05860f5194681bb5fec47cbf690cd078437fd8 Mon Sep 17 00:00:00 2001 From: Kartik Date: Sat, 9 Apr 2022 02:36:14 +0000 Subject: [PATCH 14/72] dropping determinism for performance --- .../tinyimgnet/resnet50_5/conf1.yml | 4 +- tinyimagenet_exec.sh | 133 ++++++++++++++++++ utils/utils.py | 4 +- 3 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 tinyimagenet_exec.sh diff --git a/configs/param_tuning/tinyimgnet/resnet50_5/conf1.yml b/configs/param_tuning/tinyimgnet/resnet50_5/conf1.yml index 427f415f..07730a61 100644 --- a/configs/param_tuning/tinyimgnet/resnet50_5/conf1.yml +++ b/configs/param_tuning/tinyimgnet/resnet50_5/conf1.yml @@ -53,8 +53,8 @@ regularization: L2 lmbda: 0.000001 #0.000001 #0.000001 # ===== Hardware setup ===== # -workers: 4 -#gpu: 0 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False diff --git a/tinyimagenet_exec.sh b/tinyimagenet_exec.sh new file mode 100644 index 00000000..0eb562bc --- /dev/null +++ b/tinyimagenet_exec.sh @@ -0,0 +1,133 @@ + +## REBUTTAL +# Final run on full data +# # NOTE: make sure to delete/comment subfolder from the config file or else it may not work +conf_file="configs/param_tuning/tinyimgnet/resnet50_5/conf1" +conf_end=".yml" +log_root="tinyimagenet_resnet50_5_" +log_end="_log" +subfolder_root="tinyimagenet_resnet50_5_" + +for trial in 1 +do + python main.py \ + --config "$conf_file$conf_end" \ + --trial-num $trial \ + --use-full-data \ + --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 & + + python main.py \ + --config "$conf_file$conf_end" \ + --trial-num $trial \ + --invert-sanity-check \ + --use-full-data \ + --skip-sanity-checks \ + --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +done + + + + + + + + + + + + + + + +# TinyImageNet, ResNet-50 + +# Weight training +#python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_001_multi.yml > log_tiny_res50_wt_adam_001_multi 2>&1 # this is current best +#python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_001_cosine.yml > log_tiny_res50_wt_adam_001_cosine 2>&1 # this is current best +#python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_0001_cosine.yml > log_tiny_res50_wt_adam_0001_cosine 2>&1 # this is current best +#python main.py --config configs/training/resnet50/tiny_resnet50_training_sgd_multi.yml > log_tiny_res50_wt_sgd_multi 2>&1 # this is current best + + + + +#python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_short.yml > log_tiny_res50_wt_adam_short 2>&1 # this is current best +#python main.py --config configs/training/resnet50/tiny_resnet50_training_adam.yml > log_tiny_res50_wt_adam 2>&1 # this is current best + + + + +# TinyImageNet, ResNet-101 + +## Weight training +#python main.py --config configs/training/resnet101/tiny_resnet101_training.yml > log_tiny_res101_wt_adam 2>&1 # this is current best +#python main.py --config configs/training/resnet101/tiny_resnet101_training_300.yml > log_tiny_res101_wt_adam_300 2>&1 # this is current best + +#python main.py --config configs/training/resnet101/tiny_resnet101_training.yml > log_tiny_res101_wt 2>&1 # this is current best + + +## HC +#python main.py --config configs/hypercube/tinyImageNet/resnet101/resnet101_sparsity_5.yml > log_tiny_res101_hc_sparsity_5 2>&1 + + +## EP +#python main.py --config configs/ep/tinyImageNet/resnet101/resnet101_sparsity_5.yml > log_tiny_res101_ep_sparsity_5 2>&1 +#python main.py --config configs/ep/tinyImageNet/resnet101/resnet101_sparsity_50.yml > log_tiny_res101_ep_sparsity_50 2>&1 + + + + + + +# TinyImageNet, ResNet-18 + + +# Weight training +#python main.py --config configs/training/resnet18/tiny_resnet18_training_preproc2_v3.yml #> log_tiny_wt_p2_v3 2>&1 # this is current best: 49.59% + +# HC +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_50_sgd_5.yml > log_tiny_hc_sparsity_50_sgd_5 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_50_sgd_25.yml > log_tiny_hc_sparsity_50_sgd_25 2>&1 + +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_50_adam_5.yml > log_tiny_hc_sparsity_50_adam_5 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_50_adam_25.yml > log_tiny_hc_sparsity_50_adam_25 2>&1 + + +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_5_sgd_lam8.yml #> log_tiny_hc_sparsity_5_sgd_lam8 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_5_sgd_lam8_T10.yml #> log_tiny_hc_sparsity_5_sgd_lam8 2>&1 + +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_50_sgd_lam0_T10.yml > log_tiny_hc_sparsity_50_sgd_lam0_T10 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_20_sgd_lam8_T10.yml > log_tiny_hc_sparsity_20_sgd_lam8_T10 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_1_4_sgd_lam7_T10.yml > log_tiny_hc_sparsity_1_4_sgd_lam7_T10 2>&1 + +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_50_sgd_lam0_unflag_F.yml > log_tiny_hc_sparsity_50_sgd_lam0_unflag_F 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_20_sgd_lam6_unflag_F.yml > log_tiny_hc_sparsity_20_sgd_lam6_unflag_F 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_20_sgd_lam5_unflag_F.yml > log_tiny_hc_sparsity_20_sgd_lam5_unflag_F 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_5_sgd_lam4_unflag_F.yml > log_tiny_hc_sparsity_5_sgd_lam4_unflag_F 2>&1 + +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_5_sgd_5lam6_unflag_F.yml > log_tiny_hc_sparsity_5_sgd_5lam6_unflag_F 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_20_sgd_3lam6_unflag_F.yml > log_tiny_hc_sparsity_20_sgd_3lam6_unflag_F 2>&1 + +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_5_adam_8lam6.yml > log_tiny_hc_sparsity_5_adam_8lam6 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_1_4_adam_9lam6.yml #> log_tiny_hc_sparsity_1_4_adam_9lam6 2>&1 + +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_50_adam.yml > log_tiny_hc_sparsity_50_adam 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_5_adam_1lam6.yml > log_tiny_hc_sparsity_5_adam_1lam6 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_1_4_adam_5lam6.yml > log_tiny_hc_sparsity_1_4_adam_5lam6 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_0_5_adam_1lam5.yml > log_tiny_hc_sparsity_0_5_adam_1lam5 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/resnet18/resnet18_sparsity_0_5_sgd_1_5lam5.yml > log_tiny_hc_sparsity_0_5_sgd_1_5lam5 2>&1 + + + +# EP +#:< log_tiny_ep_sparsity_5 2>&1 +#python main.py --config configs/ep/tinyImageNet/resnet18/sparsity_50.yml > log_tiny_ep_sparsity_50 2>&1 +#python main.py --config configs/ep/tinyImageNet/resnet18/sparsity_100.yml > log_tiny_ep_sparsity_100 2>&1 +#python main.py --config configs/ep/tinyImageNet/resnet18/sparsity_1_4.yml > log_tiny_ep_sparsity_1_4 2>&1 +#python main.py --config configs/ep/tinyImageNet/resnet18/sparsity_20.yml > log_tiny_ep_sparsity_20 2>&1 +#BLOCK +#python main.py --config configs/ep/tinyImageNet/resnet18/sparsity_0_75.yml > log_tiny_ep_sparsity_0_75 2>&1 + + +# testing mixed precision +#python main.py --config configs/training/resnet18/tiny_resnet18_training_test_MP.yml #> log_tiny_wt_p2_v3 2>&1 diff --git a/utils/utils.py b/utils/utils.py index 1db4f656..2ed92dba 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -25,9 +25,9 @@ def set_seed(seed): np.random.seed(seed) os.environ['PYTHONHASHSEED'] = str(seed) # making sure GPU runs are deterministic even if they are slower - torch.backends.cudnn.deterministic = True + torch.backends.cudnn.deterministic = False # this causes the code to vary across runs. I don't want that for now. - # torch.backends.cudnn.benchmark = True + torch.backends.cudnn.benchmark = True print("Seeded everything: {}".format(seed)) From dac22cf0d68a35ea22c41a0a0614ba49348f0bfd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 9 Apr 2022 09:28:12 +0000 Subject: [PATCH 15/72] adding cifar_exec for wideresnet28 --- cifar_exec.sh | 14 ++++++++------ .../param_tuning/tinyimgnet/resnet50_0_5/conf1.yml | 4 ++-- configs/param_tuning/wideresnet28_5_5/conf1.yml | 6 +++--- configs/param_tuning/wideresnet28_5_5/conf2.yml | 6 +++--- configs/param_tuning/wideresnet28_5_5/conf3.yml | 6 +++--- configs/param_tuning/wideresnet28_5_5/conf4.yml | 6 +++--- configs/param_tuning/wideresnet28_5_5/conf5.yml | 6 +++--- configs/param_tuning/wideresnet28_5_5/conf6.yml | 6 +++--- tinyimagenet_exec.sh | 6 +++--- 9 files changed, 31 insertions(+), 29 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 370be572..340f2f7f 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -61,16 +61,16 @@ python main.py \ --config configs/hypercube/wideresnet28/wideresnet28_weight_training.yml > wideresnet_wt_log 2>&1 BLOCK -:< "invert_$log_root$trial$log_end" 2>&1 & done -BLOCK +# BLOCK +:< "invert_$log_root$trial$log_end" 2>&1 & done +BLOCK diff --git a/configs/param_tuning/tinyimgnet/resnet50_0_5/conf1.yml b/configs/param_tuning/tinyimgnet/resnet50_0_5/conf1.yml index 18137e11..fb9a682f 100644 --- a/configs/param_tuning/tinyimgnet/resnet50_0_5/conf1.yml +++ b/configs/param_tuning/tinyimgnet/resnet50_0_5/conf1.yml @@ -53,8 +53,8 @@ regularization: L2 lmbda: 0.000001 #0.000001 #0.000001 # ===== Hardware setup ===== # -workers: 4 -#gpu: 0 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False diff --git a/configs/param_tuning/wideresnet28_5_5/conf1.yml b/configs/param_tuning/wideresnet28_5_5/conf1.yml index 8e4a7311..192228fb 100644 --- a/configs/param_tuning/wideresnet28_5_5/conf1.yml +++ b/configs/param_tuning/wideresnet28_5_5/conf1.yml @@ -50,12 +50,12 @@ regularization: L2 lmbda: 0.00001 # ===== Hardware setup ===== # -workers: 4 -#gpu: 2 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: True -#invert_sanity_check: True \ No newline at end of file +#invert_sanity_check: True diff --git a/configs/param_tuning/wideresnet28_5_5/conf2.yml b/configs/param_tuning/wideresnet28_5_5/conf2.yml index 4ad55b44..a8b99dc1 100644 --- a/configs/param_tuning/wideresnet28_5_5/conf2.yml +++ b/configs/param_tuning/wideresnet28_5_5/conf2.yml @@ -50,12 +50,12 @@ regularization: L2 lmbda: 0.000005 # ===== Hardware setup ===== # -workers: 4 -#gpu: 2 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: True -#invert_sanity_check: True \ No newline at end of file +#invert_sanity_check: True diff --git a/configs/param_tuning/wideresnet28_5_5/conf3.yml b/configs/param_tuning/wideresnet28_5_5/conf3.yml index 4bcf7186..bf29c3ef 100644 --- a/configs/param_tuning/wideresnet28_5_5/conf3.yml +++ b/configs/param_tuning/wideresnet28_5_5/conf3.yml @@ -50,12 +50,12 @@ regularization: L2 lmbda: 0.00005 # ===== Hardware setup ===== # -workers: 4 -#gpu: 2 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: True -#invert_sanity_check: True \ No newline at end of file +#invert_sanity_check: True diff --git a/configs/param_tuning/wideresnet28_5_5/conf4.yml b/configs/param_tuning/wideresnet28_5_5/conf4.yml index 2f932551..d90e8c38 100644 --- a/configs/param_tuning/wideresnet28_5_5/conf4.yml +++ b/configs/param_tuning/wideresnet28_5_5/conf4.yml @@ -50,12 +50,12 @@ regularization: L2 lmbda: 0.00001 # ===== Hardware setup ===== # -workers: 4 -#gpu: 2 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: True -#invert_sanity_check: True \ No newline at end of file +#invert_sanity_check: True diff --git a/configs/param_tuning/wideresnet28_5_5/conf5.yml b/configs/param_tuning/wideresnet28_5_5/conf5.yml index e8ce385a..07345219 100644 --- a/configs/param_tuning/wideresnet28_5_5/conf5.yml +++ b/configs/param_tuning/wideresnet28_5_5/conf5.yml @@ -50,12 +50,12 @@ regularization: L2 lmbda: 0.000005 # ===== Hardware setup ===== # -workers: 4 -#gpu: 2 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: True -#invert_sanity_check: True \ No newline at end of file +#invert_sanity_check: True diff --git a/configs/param_tuning/wideresnet28_5_5/conf6.yml b/configs/param_tuning/wideresnet28_5_5/conf6.yml index 41631c85..0895a86f 100644 --- a/configs/param_tuning/wideresnet28_5_5/conf6.yml +++ b/configs/param_tuning/wideresnet28_5_5/conf6.yml @@ -50,12 +50,12 @@ regularization: L2 lmbda: 0.00005 # ===== Hardware setup ===== # -workers: 4 -#gpu: 2 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: True -#invert_sanity_check: True \ No newline at end of file +#invert_sanity_check: True diff --git a/tinyimagenet_exec.sh b/tinyimagenet_exec.sh index 0eb562bc..fb73e600 100644 --- a/tinyimagenet_exec.sh +++ b/tinyimagenet_exec.sh @@ -2,11 +2,11 @@ ## REBUTTAL # Final run on full data # # NOTE: make sure to delete/comment subfolder from the config file or else it may not work -conf_file="configs/param_tuning/tinyimgnet/resnet50_5/conf1" +conf_file="configs/param_tuning/tinyimgnet/resnet50_0_5/conf1" conf_end=".yml" -log_root="tinyimagenet_resnet50_5_" +log_root="tinyimagenet_resnet50_0_5_" log_end="_log" -subfolder_root="tinyimagenet_resnet50_5_" +subfolder_root="tinyimagenet_resnet50_0_5_" for trial in 1 do From 2431c9df6d138054e05f9e104ac5d4bbfc99843a Mon Sep 17 00:00:00 2001 From: Kartik Date: Sat, 9 Apr 2022 09:39:19 +0000 Subject: [PATCH 16/72] modifying conf for wideresnet28 hyp tuning --- cifar_exec.sh | 8 ++++---- configs/param_tuning/resnet20_059_KS/conf4.yml | 4 ++-- configs/param_tuning/wideresnet28_1_4/conf1.yml | 6 +++--- configs/param_tuning/wideresnet28_1_4/conf2.yml | 6 +++--- configs/param_tuning/wideresnet28_1_4/conf3.yml | 6 +++--- configs/param_tuning/wideresnet28_1_4/conf4.yml | 6 +++--- configs/param_tuning/wideresnet28_1_4/conf5.yml | 6 +++--- configs/param_tuning/wideresnet28_1_4/conf6.yml | 6 +++--- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 340f2f7f..9d64ab7e 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -64,13 +64,13 @@ BLOCK # :< Date: Sun, 10 Apr 2022 04:45:45 +0000 Subject: [PATCH 17/72] updating exec for wideresnet28 --- cifar_exec.sh | 14 +++++++------- configs/param_tuning/wideresnet28_2_3/conf2.yml | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 9d64ab7e..9afce73c 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -61,7 +61,7 @@ python main.py \ --config configs/hypercube/wideresnet28/wideresnet28_weight_training.yml > wideresnet_wt_log 2>&1 BLOCK -# :< "invert_$log_root$trial$log_end" 2>&1 & done -# BLOCK +BLOCK -:< "invert_$log_root$trial$log_end" 2>&1 & done -BLOCK +#BLOCK diff --git a/configs/param_tuning/wideresnet28_2_3/conf2.yml b/configs/param_tuning/wideresnet28_2_3/conf2.yml index 572c1844..dea73b17 100644 --- a/configs/param_tuning/wideresnet28_2_3/conf2.yml +++ b/configs/param_tuning/wideresnet28_2_3/conf2.yml @@ -50,12 +50,12 @@ regularization: L2 lmbda: 0.000005 # ===== Hardware setup ===== # -workers: 4 -#gpu: 2 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # -skip_sanity_checks: True -#invert_sanity_check: True \ No newline at end of file +# skip_sanity_checks: True +#invert_sanity_check: True From 1c7ac7f2bb5e503cdd83457e45cc30f1f8df59a8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 10 Apr 2022 04:51:09 +0000 Subject: [PATCH 18/72] updating exec for wideresnet sp20 --- cifar_exec.sh | 6 +++--- configs/param_tuning/wideresnet28_20/conf2.yml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 9afce73c..9f7263ee 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -87,11 +87,11 @@ BLOCK #:< Date: Sun, 10 Apr 2022 04:59:02 +0000 Subject: [PATCH 19/72] pushing exec for mobilenet --- cifar_exec.sh | 8 ++++---- configs/param_tuning/mobile_5/conf3.yml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 9f7263ee..815ad39d 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -87,13 +87,13 @@ BLOCK #:< Date: Sun, 10 Apr 2022 14:21:20 -0500 Subject: [PATCH 20/72] updating readme file --- README.md | 120 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0cf18625..43269d38 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,118 @@ -# pruning_is_enough -Pruning is all you need (hopefully) +## Rare Gems: Finding Lottery Tickets at Initialization + +### Overview +--- +It has been widely observed that large neural networks can be pruned to a small fraction of their original size, with little loss in accuracy, by typically following a time-consuming "train, prune, re-train" approach. Frankle & Carbin (2018) conjecture that we can avoid this by training lottery tickets, i.e., special sparse subnetworks found at initialization, that can be trained to high accuracy. However, a subsequent line of work presents concrete evidence that current algorithms for finding trainable networks at initialization, fail simple baseline comparisons, e.g., against training random sparse subnetworks. Finding lottery tickets that train to better accuracy compared to simple baselines remains an open problem. In this work, we partially resolve this open problem by discovering rare gems: subnetworks at initialization that attain considerable accuracy, even before training. Refining these rare gems - "by means of fine-tuning" - beats current baselines and leads to accuracy competitive or better than magnitude pruning methods. + +### Depdendencies (tentative) +--- +Tested stable depdencises: +* python 3.6.5 (Anaconda) +* PyTorch 1.1.0 +* torchvision 0.2.2 +* CUDA 10.0.130 +* cuDNN 7.5.1 +* tensorboard +* tqdm +* ffcv (If you want to run ffcv imagenet) + +### Data Preparation +--- +1. For `tinyimagenet`, run `load_tiny_imagenet.sh` +2. For `imagenet`, you will need to download `imagenet` and specify the path in `data/imagenet.py` (Currently in branch. Will be merged soon) + +### Running Experients: +--- +The main script is `main.py`, to launch the jobs, we provide scripts `./cifar_exec.sh`, `imp_exec.sh`. And we provide a description of the main arguments. For more detailed descriptions, refer to `args_helper.py`. + + +| Argument | Description | +| ----------------------------- | ---------------------------------------- | +| `algo` | Specify the algorithm to run. `hc|ep|hc_iter|wt`. Note that GM is `hc_iter` in the code. | +| `lr` | Inital learning rate that will be used for the pruning process. | +| `fine_tune_lr` | Inital learning rate that will be used for the finetuning process. | +| `batch-size` | Batch size for the optimizers e.g. SGD or Adam. | +| `optimizer` | `sgd` or `adam`. | +| `dataset` | Dataset to use. | +| `arch` | Model to use. | +| `gamma` | the factor of learning rate decay, i.e. the effective learning rate is `lr*gamma^t`. | +| `iter_period` | Specifically for `hc_iter`, how often to run iterative thresholding. | +| `conv_type` | Will almost always be `SubnetConv` for pruning. | +| `target_sparsity` | Specify the target sparsity for the ticket. | +| `unflag_before_finetune` | Restore weights if the regularizer killed too many. | +| `init` | Weight initialization distribution. | +| `score_init` | Score initialization distribution. | +| `hc_quantized` | Enable for GM since it will round on forward pass. | +| `regularization` | `L2|L1` | +| `lmbda` | Regularization weight. | +| `gpu` | Specify which gpu to run on. | + + +#### Configs +Note that the workflow is managed by specifying the above arguments using `.yml` files specified in the `configs/` directory. Please refer them to create new configs like `configs/resnet20/resnet20_sparsity_0_59_unflagT.yml`. + +#### Sample Config +``` +# subfolder: target_sparsity_0_59_unflagT + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.59 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False + +``` \ No newline at end of file From 2dd2b22adfb99ce0eb03d44eba3949703c751eaa Mon Sep 17 00:00:00 2001 From: Kartik Sreenivasan Date: Sun, 10 Apr 2022 20:20:01 -0500 Subject: [PATCH 21/72] Update README.md Fixing Typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 43269d38..4e4a60a9 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ It has been widely observed that large neural networks can be pruned to a small ### Depdendencies (tentative) --- -Tested stable depdencises: +Tested stable dependencies: * python 3.6.5 (Anaconda) * PyTorch 1.1.0 * torchvision 0.2.2 @@ -115,4 +115,4 @@ checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: False -``` \ No newline at end of file +``` From 9a59614010651fc8dd6d0a34b468d58e836f3da4 Mon Sep 17 00:00:00 2001 From: Jy-yong Sohn <54528783+jysohn1108@users.noreply.github.com> Date: Mon, 11 Apr 2022 11:07:27 +0900 Subject: [PATCH 22/72] Delete related_works.md --- related_works.md | 43 ------------------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 related_works.md diff --git a/related_works.md b/related_works.md deleted file mode 100644 index 740c5c2c..00000000 --- a/related_works.md +++ /dev/null @@ -1,43 +0,0 @@ - -* Weak LTH, Pruning+Training - * [Learning both Weights and Connections for Efficient Neural Networks](https://arxiv.org/pdf/1506.02626.pdf): Learn only the "important" connections. Combination of pruning \& training. This can be considered as finding tickets - * [Lottery Ticket Hypothesis](https://arxiv.org/pdf/1803.03635.pdf): Neural networks contain sparse subnetworks that can be effectively trained from scratch when reset to their initialization - * [Deconstructing Lottery Tickets: Zeros, Signs, and the Supermask](https://arxiv.org/pdf/1905.01067.pdf): NOTE: Sec.5 has some technique using probabilistic masking. We need to compare it with ours - - -* Strong LTH, (only) Pruning - * [What’s Hidden in a Randomly Weighted Neural Network?](https://arxiv.org/pdf/1911.13299.pdf): Randomly weighted (overparameterized) neural network contains a subnetwork which performs near SOTA. Suggested Edge-popup (EP) algorithm. - -* Sparsity usinig L0 regularization - * [LEARNING SPARSE NEURAL NETWORKS THROUGH L0 REGULARIZATION](https://arxiv.org/pdf/1712.01312.pdf): Suggested "surrogate" L0 regularization, in order to sparsify NN. [Q] Not sure how they applied "reparameterization trick" - * [Winning the Lottery with Continuous Sparsification](https://arxiv.org/pdf/1912.04427.pdf) - * [SNIP: SINGLE-SHOT NETWORK PRUNING BASED ON CONNECTION SENSITIVITY](https://arxiv.org/pdf/1810.02340.pdf) - - - -* Optimization in real/binary values - * [Pseudo-boolean optimization](https://www.sciencedirect.com/science/article/pii/S0166218X01003419) - - -* Mode connectivity - * [Linear Mode Connectivity and the Lottery Ticket Hypothesis](https://arxiv.org/pdf/1912.05671.pdf) - * [Loss Surfaces, Mode Connectivity, and Fast Ensembling of DNNs](https://arxiv.org/pdf/1802.10026.pdf) - * [Analyzing Monotonic Linear Interpolation in Neural Network Loss Landscapes](https://arxiv.org/abs/2104.11044) - - MLI property: linear interpolation from initial to final neural net params typically decreases the loss monotonically - - Proved that MLI property holds with high probability for networks of sufficient width - - Proved that small "Gauss length" gives monotonicity - - Kind of related with "laze training" saying that training goes to the closest minima? - - NOTE: the connectivity is quite good. See what's happening - -* Mathematical analysis on pruning - * Proving the Lottery Ticket Hypothesis: Pruning is All You Need - - -* To Be Categorized - * [Sanity-Checking Pruning Methods: Random Tickets can Win the Jackpot](https://arxiv.org/pdf/2009.11094.pdf) - * [Pruning Neural Networks at Initialization: Why are We Missing the Mark?](https://arxiv.org/pdf/2009.08576.pdf) - * [Supermasks in Superposition](https://proceedings.neurips.cc//paper/2020/file/ad1f8bb9b51f023cdc80cf94bb615aa9-Paper.pdf): Superposition of supermasks can be used for target-task inference \& continual learning? - - * [PICKING WINNING TICKETS BEFORE TRAINING BY PRESERVING GRADIENT FLOW](https://openreview.net/pdf?id=SkgsACVKPH) - - From cdb3888f96f532f9cc4c8b6c90a4ce2f794c236a Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 11 Apr 2022 18:49:07 -0500 Subject: [PATCH 23/72] adding code for resnet32 cifar100 --- .../resnet32/resnet32_sparsity_2.yml | 62 +++++++++++++++++ data/cifar100.py | 68 +++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 configs/hypercube/resnet32/resnet32_sparsity_2.yml create mode 100644 data/cifar100.py diff --git a/configs/hypercube/resnet32/resnet32_sparsity_2.yml b/configs/hypercube/resnet32/resnet32_sparsity_2.yml new file mode 100644 index 00000000..735a55ea --- /dev/null +++ b/configs/hypercube/resnet32/resnet32_sparsity_2.yml @@ -0,0 +1,62 @@ +# subfolder: target_sparsity_2 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet32 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 # 1e-4 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/data/cifar100.py b/data/cifar100.py new file mode 100644 index 00000000..00f343d4 --- /dev/null +++ b/data/cifar100.py @@ -0,0 +1,68 @@ +import os +import torch +import torchvision +from torchvision import transforms +import random +from torch.utils.data.sampler import SubsetRandomSampler +from args_helper import parser_args +from torch.utils.data import random_split + + +class CIFAR10: + def __init__(self, args): + super(CIFAR10, self).__init__() + + data_root = os.path.join(parser_args.data, "cifar100") + + use_cuda = torch.cuda.is_available() + + # Data loading code + kwargs = {"num_workers": parser_args.workers, "pin_memory": True} if use_cuda else {} + + num_classes = 100 + transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), + ]) + + transform_test = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), + ]) + + dataset = torchvision.datasets.CIFAR100( + root=data_root, + train=True, + download=True, + transform=transform_train, + ) + + test_dataset = torchvision.datasets.CIFAR100( + root=data_root, + train=False, + download=True, + transform=transform_test, + ) + + if parser_args.use_full_data: + train_dataset = dataset + # use_full_data => we are not tuning hyperparameters + validation_dataset = test_dataset + else: + val_size = 5000 + train_size = len(dataset) - val_size + train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) + + self.train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=parser_args.batch_size, shuffle=True, **kwargs + ) + + self.val_loader = torch.utils.data.DataLoader( + test_dataset, batch_size=parser_args.batch_size, shuffle=False, **kwargs + ) + + self.actual_val_loader = torch.utils.data.DataLoader( + validation_dataset, batch_size=parser_args.batch_size, shuffle=True, **kwargs + ) From 6cd6a73c1f6ea8e059f9c78cb3472d71c1895fb8 Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 12 Apr 2022 00:04:01 +0000 Subject: [PATCH 24/72] cifar100 bugfixes --- configs/hypercube/resnet32/resnet32_sparsity_2.yml | 4 ++-- configs/param_tuning/resnet20_13_34/conf2.yml | 6 +++--- data/__init__.py | 1 + data/cifar100.py | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/configs/hypercube/resnet32/resnet32_sparsity_2.yml b/configs/hypercube/resnet32/resnet32_sparsity_2.yml index 735a55ea..edf9116c 100644 --- a/configs/hypercube/resnet32/resnet32_sparsity_2.yml +++ b/configs/hypercube/resnet32/resnet32_sparsity_2.yml @@ -8,7 +8,7 @@ iter_period: 5 arch: resnet32 # ===== Dataset ===== # -dataset: CIFAR10 +dataset: CIFAR100 name: resnet20_quantized_iter_hc # ===== Learning Rate Policy ======== # @@ -52,7 +52,7 @@ regularization: L2 lmbda: 0.00001 # 1e-4 # ===== Hardware setup ===== # -workers: 4 +workers: 6 gpu: 0 # ===== Checkpointing ===== # diff --git a/configs/param_tuning/resnet20_13_34/conf2.yml b/configs/param_tuning/resnet20_13_34/conf2.yml index 7b81c838..13f6fd9f 100644 --- a/configs/param_tuning/resnet20_13_34/conf2.yml +++ b/configs/param_tuning/resnet20_13_34/conf2.yml @@ -51,11 +51,11 @@ regularization: L2 lmbda: 0.000005 # 5e-6 # ===== Hardware setup ===== # -workers: 4 -#gpu: 2 +workers: 6 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # -skip_sanity_checks: True \ No newline at end of file +# skip_sanity_checks: True diff --git a/data/__init__.py b/data/__init__.py index a70f282b..68298408 100644 --- a/data/__init__.py +++ b/data/__init__.py @@ -1,4 +1,5 @@ from data.cifar import CIFAR10 +from data.cifar100 import CIFAR100 from data.imagenet import ImageNet from data.tinyimagenet import TinyImageNet from data.mnist import MNIST diff --git a/data/cifar100.py b/data/cifar100.py index 00f343d4..2d222726 100644 --- a/data/cifar100.py +++ b/data/cifar100.py @@ -8,9 +8,9 @@ from torch.utils.data import random_split -class CIFAR10: +class CIFAR100: def __init__(self, args): - super(CIFAR10, self).__init__() + super(CIFAR100, self).__init__() data_root = os.path.join(parser_args.data, "cifar100") From 832af7581258493110a566e4bb923a788c8f5889 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 11 Apr 2022 19:06:12 -0500 Subject: [PATCH 25/72] changing resnet last layer for cifar100 --- models/resnet_cifar.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/models/resnet_cifar.py b/models/resnet_cifar.py index 9ae30dc6..60d064b8 100644 --- a/models/resnet_cifar.py +++ b/models/resnet_cifar.py @@ -81,10 +81,14 @@ def __init__(self, builder, block, num_blocks): self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) self.avgpool = nn.AdaptiveAvgPool2d(1) + num_classes = 10 + if parser_args.dataset == "CIFAR100": + num_classes = 100 + if parser_args.last_layer_dense: - self.fc = nn.Conv2d(512 * block.expansion, 10, 1) + self.fc = nn.Conv2d(512 * block.expansion, num_classes, 1) else: - self.fc = builder.conv1x1(512 * block.expansion, 10) + self.fc = builder.conv1x1(512 * block.expansion, num_classes) self.prunable_layer_names, self.prunable_biases = self.get_prunable_param_names() From f9961fa6c834f74dc7ffafa3c5ecb4d756d4a253 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 11 Apr 2022 19:08:40 -0500 Subject: [PATCH 26/72] changing last layer for resnet32 --- models/resnet_kaiming.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/models/resnet_kaiming.py b/models/resnet_kaiming.py index 15a0860a..8cc91bfc 100644 --- a/models/resnet_kaiming.py +++ b/models/resnet_kaiming.py @@ -62,7 +62,12 @@ def __init__(self, builder, block, num_blocks, times=1): self.layer2 = self._make_layer(block, 32 * times, num_blocks[1], stride=2) self.layer3 = self._make_layer(block, 64 * times, num_blocks[2], stride=2) # self.avgpool = nn.AdaptiveAvgPool2d(1) - self.fc = builder.conv1x1(64 * block.expansion * times, 10) # 10 = num_classes for cifar10 + + num_classes = 10 + if parser_args.dataset == "CIFAR100": + num_classes = 100 + + self.fc = builder.conv1x1(64 * block.expansion * times, num_classes) # 10 = num_classes for cifar10 self.prunable_layer_names, self.prunable_biases = self.get_prunable_param_names() From 97797552446b673c9604ff5c5267a65b237f281a Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 12 Apr 2022 03:15:34 +0000 Subject: [PATCH 27/72] pushing configs and exec script --- cifar_exec.sh | 29 ++++----- .../resnet32/resnet32_sparsity_2_v2.yml | 62 +++++++++++++++++++ .../resnet32/resnet32_sparsity_2_v3.yml | 62 +++++++++++++++++++ .../resnet32/resnet32_sparsity_5_v1.yml | 62 +++++++++++++++++++ .../resnet32/resnet32_sparsity_5_v2.yml | 62 +++++++++++++++++++ 5 files changed, 263 insertions(+), 14 deletions(-) create mode 100644 configs/hypercube/resnet32/resnet32_sparsity_2_v2.yml create mode 100644 configs/hypercube/resnet32/resnet32_sparsity_2_v3.yml create mode 100644 configs/hypercube/resnet32/resnet32_sparsity_5_v1.yml create mode 100644 configs/hypercube/resnet32/resnet32_sparsity_5_v2.yml diff --git a/cifar_exec.sh b/cifar_exec.sh index 815ad39d..d1cf5237 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -61,39 +61,40 @@ python main.py \ --config configs/hypercube/wideresnet28/wideresnet28_weight_training.yml > wideresnet_wt_log 2>&1 BLOCK -:< "$log_root$trial$log_end" 2>&1 & #python main.py \ - #--config "$conf_file" \ + #--config "$conf_file$conf_end" \ #--trial-num $trial \ #--invert-sanity-check \ + #--skip-sanity-checks \ #--subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & done -BLOCK +#BLOCK -#:< "invert_$log_root$trial$log_end" 2>&1 & done -#BLOCK +BLOCK diff --git a/configs/hypercube/resnet32/resnet32_sparsity_2_v2.yml b/configs/hypercube/resnet32/resnet32_sparsity_2_v2.yml new file mode 100644 index 00000000..24e20f09 --- /dev/null +++ b/configs/hypercube/resnet32/resnet32_sparsity_2_v2.yml @@ -0,0 +1,62 @@ +# subfolder: target_sparsity_2 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet32 + +# ===== Dataset ===== # +dataset: CIFAR100 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 1e-4 + +# ===== Hardware setup ===== # +workers: 6 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/configs/hypercube/resnet32/resnet32_sparsity_2_v3.yml b/configs/hypercube/resnet32/resnet32_sparsity_2_v3.yml new file mode 100644 index 00000000..3b825231 --- /dev/null +++ b/configs/hypercube/resnet32/resnet32_sparsity_2_v3.yml @@ -0,0 +1,62 @@ +# subfolder: target_sparsity_2 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet32 + +# ===== Dataset ===== # +dataset: CIFAR100 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 + +# ===== Hardware setup ===== # +workers: 6 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/configs/hypercube/resnet32/resnet32_sparsity_5_v1.yml b/configs/hypercube/resnet32/resnet32_sparsity_5_v1.yml new file mode 100644 index 00000000..155feaa5 --- /dev/null +++ b/configs/hypercube/resnet32/resnet32_sparsity_5_v1.yml @@ -0,0 +1,62 @@ +# subfolder: target_sparsity_2 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet32 + +# ===== Dataset ===== # +dataset: CIFAR100 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00001 # 1e-4 + +# ===== Hardware setup ===== # +workers: 6 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/configs/hypercube/resnet32/resnet32_sparsity_5_v2.yml b/configs/hypercube/resnet32/resnet32_sparsity_5_v2.yml new file mode 100644 index 00000000..9f3a67f9 --- /dev/null +++ b/configs/hypercube/resnet32/resnet32_sparsity_5_v2.yml @@ -0,0 +1,62 @@ +# subfolder: target_sparsity_2 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet32 + +# ===== Dataset ===== # +dataset: CIFAR100 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 # 1e-4 + +# ===== Hardware setup ===== # +workers: 6 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False From 76e494886cdbbd9eecf345ff581131cbcaccd4b0 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 20 Apr 2022 21:47:03 -0500 Subject: [PATCH 28/72] adding config and exec for wt on cifar100 resnet32_double --- cifar_exec.sh | 6 +-- .../resnet32_sparsity_2.yml | 0 .../resnet32_sparsity_2_v2.yml | 0 .../resnet32_sparsity_2_v3.yml | 0 .../resnet32_sparsity_5_v1.yml | 0 .../resnet32_sparsity_5_v2.yml | 0 .../resnet32/cifar100_resnet32_training.yml | 41 +++++++++++++++++++ 7 files changed, 44 insertions(+), 3 deletions(-) rename configs/hypercube/resnet32/{ => half_width_configs}/resnet32_sparsity_2.yml (100%) rename configs/hypercube/resnet32/{ => half_width_configs}/resnet32_sparsity_2_v2.yml (100%) rename configs/hypercube/resnet32/{ => half_width_configs}/resnet32_sparsity_2_v3.yml (100%) rename configs/hypercube/resnet32/{ => half_width_configs}/resnet32_sparsity_5_v1.yml (100%) rename configs/hypercube/resnet32/{ => half_width_configs}/resnet32_sparsity_5_v2.yml (100%) create mode 100644 configs/training/resnet32/cifar100_resnet32_training.yml diff --git a/cifar_exec.sh b/cifar_exec.sh index d1cf5237..43eefb4f 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -64,11 +64,11 @@ BLOCK #:< Date: Thu, 21 Apr 2022 14:03:45 +0000 Subject: [PATCH 29/72] imp for resnet32 (double), cifar100 --- configs/imp/resnet32_cifar100.yml | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 configs/imp/resnet32_cifar100.yml diff --git a/configs/imp/resnet32_cifar100.yml b/configs/imp/resnet32_cifar100.yml new file mode 100644 index 00000000..8b176c4e --- /dev/null +++ b/configs/imp/resnet32_cifar100.yml @@ -0,0 +1,36 @@ +# IMP algorithm +algo: 'imp' +seed: 42 +name: cifar100_resnet32_double_imp + +# Architecture +arch: resnet32_double + +# ===== Dataset ===== # +dataset: CIFAR100 + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: multistep_lr +lr_gamma: 0.1 + +# ===== Network training config ===== # +# epochs: 300 +wd: 0.0001 +momentum: 0.9 +batch_size: 64 +bias: False + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +prune_rate: 0.2 +init: kaiming_normal +iter_period: 150 +imp_rewind_iter: 1000 + + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 From e408df34e8047a2318647306c01ea213a1ec5385 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 21 Apr 2022 19:08:27 -0500 Subject: [PATCH 30/72] trying ddp demo --- ddp_poc.py | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 ddp_poc.py diff --git a/ddp_poc.py b/ddp_poc.py new file mode 100644 index 00000000..66d47191 --- /dev/null +++ b/ddp_poc.py @@ -0,0 +1,119 @@ +import os +import sys +import tempfile +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.optim as optim +import torch.multiprocessing as mp + +from torch.nn.parallel import DistributedDataParallel as DDP + +# On Windows platform, the torch.distributed package only +# supports Gloo backend, FileStore and TcpStore. +# For FileStore, set init_method parameter in init_process_group +# to a local file. Example as follow: +# init_method="file:///f:/libtmp/some_file" +# dist.init_process_group( +# "gloo", +# rank=rank, +# init_method=init_method, +# world_size=world_size) +# For TcpStore, same way as on Linux. + +def setup(rank, world_size): + os.environ['MASTER_ADDR'] = 'localhost' + os.environ['MASTER_PORT'] = '12355' + + # initialize the process group + dist.init_process_group("gloo", rank=rank, world_size=world_size) + +def cleanup(): + dist.destroy_process_group() + + +class ToyModel(nn.Module): + def __init__(self): + super(ToyModel, self).__init__() + self.net1 = nn.Linear(10, 10) + self.relu = nn.ReLU() + self.net2 = nn.Linear(10, 5) + + def forward(self, x): + return self.net2(self.relu(self.net1(x))) + + +def demo_basic(rank, world_size): + print(f"Running basic DDP example on rank {rank}.") + setup(rank, world_size) + + # create model and move it to GPU with id rank + model = ToyModel().to(rank) + ddp_model = DDP(model, device_ids=[rank]) + + loss_fn = nn.MSELoss() + optimizer = optim.SGD(ddp_model.parameters(), lr=0.001) + + optimizer.zero_grad() + outputs = ddp_model(torch.randn(20, 10)) + labels = torch.randn(20, 5).to(rank) + loss_fn(outputs, labels).backward() + optimizer.step() + + cleanup() + + +def run_demo(demo_fn, world_size): + mp.spawn(demo_fn, + args=(world_size,), + nprocs=world_size, + join=True) + +def demo_checkpoint(rank, world_size): + print(f"Running DDP checkpoint example on rank {rank}.") + setup(rank, world_size) + + model = ToyModel().to(rank) + ddp_model = DDP(model, device_ids=[rank]) + + loss_fn = nn.MSELoss() + optimizer = optim.SGD(ddp_model.parameters(), lr=0.001) + + CHECKPOINT_PATH = tempfile.gettempdir() + "/model.checkpoint" + if rank == 0: + # All processes should see same parameters as they all start from same + # random parameters and gradients are synchronized in backward passes. + # Therefore, saving it in one process is sufficient. + torch.save(ddp_model.state_dict(), CHECKPOINT_PATH) + + # Use a barrier() to make sure that process 1 loads the model after process + # 0 saves it. + dist.barrier() + # configure map_location properly + map_location = {'cuda:%d' % 0: 'cuda:%d' % rank} + ddp_model.load_state_dict( + torch.load(CHECKPOINT_PATH, map_location=map_location)) + + optimizer.zero_grad() + outputs = ddp_model(torch.randn(20, 10)) + labels = torch.randn(20, 5).to(rank) + loss_fn = nn.MSELoss() + loss_fn(outputs, labels).backward() + optimizer.step() + + # Not necessary to use a dist.barrier() to guard the file deletion below + # as the AllReduce ops in the backward pass of DDP already served as + # a synchronization. + + if rank == 0: + os.remove(CHECKPOINT_PATH) + + cleanup() + + +if __name__ == "__main__": + n_gpus = torch.cuda.device_count() + assert n_gpus >= 2, f"Requires at least 2 GPUs to run, but got {n_gpus}" + world_size = n_gpus + run_demo(demo_basic, world_size) + # run_demo(demo_checkpoint, world_size) From 32194488a15b15670e5a3d346537f8f6744a0a01 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 22 Apr 2022 16:53:02 +0000 Subject: [PATCH 31/72] cifar100, resnet32 --- cifar_exec_GD.sh | 49 ++++++++++++++++++- .../resnet32/cifar100_resnet32_training.yml | 3 +- main_utils.py | 6 ++- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index e3791018..7180a36c 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -1,7 +1,52 @@ + +## NeurIPS prep + + +#:< "$log_root$trial$log_end" 2>&1 & + + #--smart_ratio 0.98 \ + + #python main.py \ + #--config "$conf_file$conf_end" \ + #--trial-num $trial \ + #--invert-sanity-check \ + #--skip-sanity-checks \ + #--subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +done +#BLOCK + + + + + + + + + # REBUTTAL # NOTE: make sure to delete/comment subfolder from the config file or else it may not work -#:< "$log_root$trial$log_end" 2>&1 #& done -#BLOCK +BLOCK # Final run on full data # # NOTE: make sure to delete/comment subfolder from the config file or else it may not work # conf_file="configs/param_tuning/resnet20_059_KS/conf4" diff --git a/configs/training/resnet32/cifar100_resnet32_training.yml b/configs/training/resnet32/cifar100_resnet32_training.yml index b3f90089..c4da5b92 100644 --- a/configs/training/resnet32/cifar100_resnet32_training.yml +++ b/configs/training/resnet32/cifar100_resnet32_training.yml @@ -37,5 +37,4 @@ skip_fine_tune: True # ===== Hardware setup ===== # workers: 4 -gpu: 3 - +#gpu: 3 diff --git a/main_utils.py b/main_utils.py index 55187b37..7f54a7cd 100644 --- a/main_utils.py +++ b/main_utils.py @@ -211,8 +211,10 @@ def test_random_subnet(model, data, criterion, parser_args, result_root, smart_r #print("Writing init model to {}".format(model_filename)) #torch.save(model.state_dict(), model_filename) - old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_reg_loss_list, old_model_sparsity_list = [], [], [], [], [] - model = finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_reg_loss_list, old_model_sparsity_list, result_root, shuffle=False, reinit=False, invert=False, chg_mask=False, chg_weight=False) + old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_val_acc_list, old_train_acc_list, old_reg_loss_list, old_model_sparsity_list = [], [], [], [], [], [], [] + model = finetune(model, parser_args, data, criterion, + old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_val_acc_list, old_train_acc_list, old_reg_loss_list, old_model_sparsity_list, + result_root, shuffle=False, reinit=False, invert=False, chg_mask=False, chg_weight=False) # save checkpoint for later debug model_filename = result_root + 'model_after_finetune.pth' From 49aab991abdc0d2f92105ff27a467f8e4384cc1b Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 16:12:53 -0500 Subject: [PATCH 32/72] trying ddp with cifar10 --- ddp_poc.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 66d47191..bab5b7c5 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -22,11 +22,11 @@ # For TcpStore, same way as on Linux. def setup(rank, world_size): - os.environ['MASTER_ADDR'] = 'localhost' + os.environ['MASTER_ADDR'] = '127.0.0.1' os.environ['MASTER_PORT'] = '12355' # initialize the process group - dist.init_process_group("gloo", rank=rank, world_size=world_size) + dist.init_process_group("nccl", rank=rank, world_size=world_size) def cleanup(): dist.destroy_process_group() @@ -43,6 +43,23 @@ def forward(self, x): return self.net2(self.relu(self.net1(x))) + def evaluate(model, device, test_loader): + model.eval() + + correct = 0 + total = 0 + with torch.no_grad(): + for data in test_loader: + images, labels = data[0].to(device), data[1].to(device) + outputs = model(images) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + accuracy = correct / total + + return accuracy + + def demo_basic(rank, world_size): print(f"Running basic DDP example on rank {rank}.") setup(rank, world_size) @@ -51,14 +68,57 @@ def demo_basic(rank, world_size): model = ToyModel().to(rank) ddp_model = DDP(model, device_ids=[rank]) + transform = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), + ]) + + # Data should be prefetched + # Download should be set to be False, because it is not multiprocess safe + train_set = torchvision.datasets.CIFAR10(root="data", train=True, download=False, transform=transform) + test_set = torchvision.datasets.CIFAR10(root="data", train=False, download=False, transform=transform) + + train_sampler = DistributedSampler(dataset=train_set) + + train_loader = DataLoader(dataset=train_set, batch_size=512, sampler=train_sampler, num_workers=8) + + # Test loader does not have to follow distributed sampling strategy + test_loader = DataLoader(dataset=test_set, batch_size=512, shuffle=False, num_workers=8) + loss_fn = nn.MSELoss() optimizer = optim.SGD(ddp_model.parameters(), lr=0.001) - optimizer.zero_grad() - outputs = ddp_model(torch.randn(20, 10)) - labels = torch.randn(20, 5).to(rank) - loss_fn(outputs, labels).backward() - optimizer.step() + device = torch.device("cuda:{}".format(rank)) + + for epoch in range(10): + print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) + + # Save and evaluate model routinely + if epoch % 2 == 0: + if local_rank == 0: + accuracy = evaluate(model=ddp_model, device=device, test_loader=test_loader) + torch.save(ddp_model.state_dict(), model_filepath) + print("-" * 75) + print("Epoch: {}, Accuracy: {}".format(epoch, accuracy)) + print("-" * 75) + + ddp_model.train() + + for data in train_loader: + inputs, labels = data[0].to(device), data[1].to(device) + optimizer.zero_grad() + outputs = ddp_model(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # optimizer.zero_grad() + # outputs = ddp_model(torch.randn(20, 10)) + # labels = torch.randn(20, 5).to(rank) + # loss_fn(outputs, labels).backward() + # optimizer.step() cleanup() From 0104f0e566c8692fb14c49d39944782aa3be3695 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 16:13:47 -0500 Subject: [PATCH 33/72] merging conflicts --- ddp_poc.py | 7 +++---- imp_exec.sh | 4 ++-- trainers/default.py | 1 + 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index bab5b7c5..54b8a8c1 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -35,9 +35,9 @@ def cleanup(): class ToyModel(nn.Module): def __init__(self): super(ToyModel, self).__init__() - self.net1 = nn.Linear(10, 10) + self.net1 = nn.Linear(5, 5) self.relu = nn.ReLU() - self.net2 = nn.Linear(10, 5) + self.net2 = nn.Linear(5, 5) def forward(self, x): return self.net2(self.relu(self.net1(x))) @@ -119,7 +119,6 @@ def demo_basic(rank, world_size): # labels = torch.randn(20, 5).to(rank) # loss_fn(outputs, labels).backward() # optimizer.step() - cleanup() @@ -155,7 +154,7 @@ def demo_checkpoint(rank, world_size): torch.load(CHECKPOINT_PATH, map_location=map_location)) optimizer.zero_grad() - outputs = ddp_model(torch.randn(20, 10)) + outputs = ddp_model(torch.randn(20, 5)) labels = torch.randn(20, 5).to(rank) loss_fn = nn.MSELoss() loss_fn(outputs, labels).backward() diff --git a/imp_exec.sh b/imp_exec.sh index b7b3ec4f..49e262c2 100644 --- a/imp_exec.sh +++ b/imp_exec.sh @@ -1,10 +1,10 @@ # ===== warm short IMP ===== # -subfd="imp_resnet20_sp0_59" +subfd="imp_resnet32_" # subfd="cifar_resnet_check_sparse_mask_1_4_at_init" n_gpu=0 python imp_main.py \ ---config configs/imp/resnet20.yml \ +--config configs/imp/resnet32_cifar100.yml \ --imp-rounds 30 \ --gpu $n_gpu \ --subfolder $subfd > imp_log 2>&1 diff --git a/trainers/default.py b/trainers/default.py index 6597353f..8325fccb 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -87,6 +87,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # compute gradient and do SGD step optimizer.zero_grad() + import ipdb; ipdb.set_trace() if scaler is None: loss.backward() optimizer.step() From 07ea02a544be880af9328151673a47aebd880832 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 16:14:31 -0500 Subject: [PATCH 34/72] indentation fix --- ddp_poc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 54b8a8c1..897416ff 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -43,9 +43,8 @@ def forward(self, x): return self.net2(self.relu(self.net1(x))) - def evaluate(model, device, test_loader): +def evaluate(model, device, test_loader): model.eval() - correct = 0 total = 0 with torch.no_grad(): From e723ea5b0e6d832ab09d8cfcb03358647358566d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 16:17:07 -0500 Subject: [PATCH 35/72] minor indentation fix --- ddp_poc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 897416ff..3a16b3ad 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -6,6 +6,7 @@ import torch.nn as nn import torch.optim as optim import torch.multiprocessing as mp +import torchvision.transforms as transforms from torch.nn.parallel import DistributedDataParallel as DDP @@ -92,8 +93,7 @@ def demo_basic(rank, world_size): device = torch.device("cuda:{}".format(rank)) for epoch in range(10): - print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) - + print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) # Save and evaluate model routinely if epoch % 2 == 0: if local_rank == 0: From 6dc75f19da22e8bd06c5c2090bd069a034e7931c Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 16:18:53 -0500 Subject: [PATCH 36/72] changing model to resnet18 --- ddp_poc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ddp_poc.py b/ddp_poc.py index 3a16b3ad..7c59e417 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -65,7 +65,8 @@ def demo_basic(rank, world_size): setup(rank, world_size) # create model and move it to GPU with id rank - model = ToyModel().to(rank) + model = torchvision.models.resnet18(pretrained=False).to(rank) + # model = ToyModel().to(rank) ddp_model = DDP(model, device_ids=[rank]) transform = transforms.Compose([ From 3d6ce30c8ac1df74265f51b0b73301c12fb7da25 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 16:30:52 -0500 Subject: [PATCH 37/72] tweaking things for ddp cifar10 --- ddp_poc.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 7c59e417..443e4854 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -7,6 +7,9 @@ import torch.optim as optim import torch.multiprocessing as mp import torchvision.transforms as transforms +import torchvision +from torch.utils.data.distributed import DistributedSampler +from torch.utils.data import DataLoader from torch.nn.parallel import DistributedDataParallel as DDP @@ -89,7 +92,8 @@ def demo_basic(rank, world_size): test_loader = DataLoader(dataset=test_set, batch_size=512, shuffle=False, num_workers=8) loss_fn = nn.MSELoss() - optimizer = optim.SGD(ddp_model.parameters(), lr=0.001) + optimizer = optim.SGD(ddp_model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-5) + criterion = nn.CrossEntropyLoss() device = torch.device("cuda:{}".format(rank)) @@ -97,9 +101,9 @@ def demo_basic(rank, world_size): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) # Save and evaluate model routinely if epoch % 2 == 0: - if local_rank == 0: + if rank == 0: accuracy = evaluate(model=ddp_model, device=device, test_loader=test_loader) - torch.save(ddp_model.state_dict(), model_filepath) + # torch.save(ddp_model.state_dict(), model_filepath) print("-" * 75) print("Epoch: {}, Accuracy: {}".format(epoch, accuracy)) print("-" * 75) From 1bf670f380c4d8c59bb7a5a9b30437e9b89a4d6e Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 16:39:27 -0500 Subject: [PATCH 38/72] testing if batches are disjoint cifar10 ddp --- ddp_poc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 443e4854..2202a4df 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -97,7 +97,7 @@ def demo_basic(rank, world_size): device = torch.device("cuda:{}".format(rank)) - for epoch in range(10): + for epoch in range(5): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) # Save and evaluate model routinely if epoch % 2 == 0: @@ -109,15 +109,18 @@ def demo_basic(rank, world_size): print("-" * 75) ddp_model.train() + total_data_size = [0, 0] for data in train_loader: inputs, labels = data[0].to(device), data[1].to(device) + # print("Device: {} | Batch Size: {} | Label sum: {}".format(rank, data[1].shape[0], torch.sum(data[1]))) + total_data_size[rank] += data[1].shape[0] optimizer.zero_grad() outputs = ddp_model(inputs) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + print("End of epoch total batch sizes: {}".format(total_data_size)) # optimizer.zero_grad() # outputs = ddp_model(torch.randn(20, 10)) # labels = torch.randn(20, 5).to(rank) From 71174866fb365030358fc3fcc255c4f2875e0e39 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 16:48:15 -0500 Subject: [PATCH 39/72] trying cifar10 ddp with toy fc model --- ddp_poc.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 2202a4df..0a91e244 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -13,18 +13,6 @@ from torch.nn.parallel import DistributedDataParallel as DDP -# On Windows platform, the torch.distributed package only -# supports Gloo backend, FileStore and TcpStore. -# For FileStore, set init_method parameter in init_process_group -# to a local file. Example as follow: -# init_method="file:///f:/libtmp/some_file" -# dist.init_process_group( -# "gloo", -# rank=rank, -# init_method=init_method, -# world_size=world_size) -# For TcpStore, same way as on Linux. - def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' os.environ['MASTER_PORT'] = '12355' @@ -39,9 +27,9 @@ def cleanup(): class ToyModel(nn.Module): def __init__(self): super(ToyModel, self).__init__() - self.net1 = nn.Linear(5, 5) + self.net1 = nn.Linear(3072, 100) self.relu = nn.ReLU() - self.net2 = nn.Linear(5, 5) + self.net2 = nn.Linear(100, 10) def forward(self, x): return self.net2(self.relu(self.net1(x))) @@ -53,7 +41,7 @@ def evaluate(model, device, test_loader): total = 0 with torch.no_grad(): for data in test_loader: - images, labels = data[0].to(device), data[1].to(device) + images, labels = data[0].to(device).reshape(-1, 32*32*3), data[1].to(device) outputs = model(images) _, predicted = torch.max(outputs.data, 1) total += labels.size(0) @@ -68,8 +56,8 @@ def demo_basic(rank, world_size): setup(rank, world_size) # create model and move it to GPU with id rank - model = torchvision.models.resnet18(pretrained=False).to(rank) - # model = ToyModel().to(rank) + # model = torchvision.models.resnet18(pretrained=False).to(rank) + model = ToyModel().to(rank) ddp_model = DDP(model, device_ids=[rank]) transform = transforms.Compose([ @@ -108,11 +96,16 @@ def demo_basic(rank, world_size): print("Epoch: {}, Accuracy: {}".format(epoch, accuracy)) print("-" * 75) + if epoch % 3 == 0: + if rank == 0: + # prune model + + ddp_model.train() total_data_size = [0, 0] for data in train_loader: - inputs, labels = data[0].to(device), data[1].to(device) + inputs, labels = data[0].to(device).reshape(-1, 32*32*3), data[1].to(device) # print("Device: {} | Batch Size: {} | Label sum: {}".format(rank, data[1].shape[0], torch.sum(data[1]))) total_data_size[rank] += data[1].shape[0] optimizer.zero_grad() From abc7a98b90a1f8a1d56634ae1f6de21c59f4824e Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 16:50:59 -0500 Subject: [PATCH 40/72] indentation fixes --- ddp_poc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddp_poc.py b/ddp_poc.py index 0a91e244..350fd052 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -99,7 +99,7 @@ def demo_basic(rank, world_size): if epoch % 3 == 0: if rank == 0: # prune model - + print("Gonna try to prune model") ddp_model.train() total_data_size = [0, 0] From 747e95ac79da8a51fbe70c77da0382b4c74eafdc Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 16:54:54 -0500 Subject: [PATCH 41/72] trying to periodically prune model ddp --- ddp_poc.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 350fd052..284bce0b 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -50,6 +50,12 @@ def evaluate(model, device, test_loader): return accuracy +def get_model_norm(model): + tot_norm = 0 + for name, params in ddp_model.named_parameters(): + tot_norm += torch.norm(params.data) + return tot_norm + def demo_basic(rank, world_size): print(f"Running basic DDP example on rank {rank}.") @@ -87,6 +93,7 @@ def demo_basic(rank, world_size): for epoch in range(5): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) + print("Model Norm: {}".format(get_model_norm(ddp_model))) # Save and evaluate model routinely if epoch % 2 == 0: if rank == 0: @@ -100,6 +107,10 @@ def demo_basic(rank, world_size): if rank == 0: # prune model print("Gonna try to prune model") + for name, params in ddp_model.named_parameters(): + # basically, prune everything + if re.match('.*\.weight', name) or re.match('.*\.bias', name):: + params.data = torch.zeros_like(params.data) ddp_model.train() total_data_size = [0, 0] @@ -171,8 +182,8 @@ def demo_checkpoint(rank, world_size): if __name__ == "__main__": - n_gpus = torch.cuda.device_count() - assert n_gpus >= 2, f"Requires at least 2 GPUs to run, but got {n_gpus}" - world_size = n_gpus - run_demo(demo_basic, world_size) - # run_demo(demo_checkpoint, world_size) + n_gpus = torch.cuda.device_count() + assert n_gpus >= 2, f"Requires at least 2 GPUs to run, but got {n_gpus}" + world_size = n_gpus + run_demo(demo_basic, world_size) + # run_demo(demo_checkpoint, world_size) From c2e9e9d471e4cfbdade925a05f0de9cca588bf65 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 17:02:20 -0500 Subject: [PATCH 42/72] prune at start of epoch and check norm immediately ddp --- ddp_poc.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 284bce0b..b08ff3d8 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -93,6 +93,15 @@ def demo_basic(rank, world_size): for epoch in range(5): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) + if epoch % 3 == 0: + if rank == 0: + # prune model + print("Gonna try to prune model") + for name, params in ddp_model.named_parameters(): + # basically, prune everything + if re.match('.*\.weight', name) or re.match('.*\.bias', name):: + params.data = torch.zeros_like(params.data) + print("Model Norm: {}".format(get_model_norm(ddp_model))) # Save and evaluate model routinely if epoch % 2 == 0: @@ -103,15 +112,6 @@ def demo_basic(rank, world_size): print("Epoch: {}, Accuracy: {}".format(epoch, accuracy)) print("-" * 75) - if epoch % 3 == 0: - if rank == 0: - # prune model - print("Gonna try to prune model") - for name, params in ddp_model.named_parameters(): - # basically, prune everything - if re.match('.*\.weight', name) or re.match('.*\.bias', name):: - params.data = torch.zeros_like(params.data) - ddp_model.train() total_data_size = [0, 0] From 807dea498374b11076c47270718dc161c70c50ea Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 17:19:34 -0500 Subject: [PATCH 43/72] i think the pruning poc is complete need to prune all the local models independently but rest of it seems to work! --- ddp_poc.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index b08ff3d8..04a21e52 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -10,6 +10,7 @@ import torchvision from torch.utils.data.distributed import DistributedSampler from torch.utils.data import DataLoader +import re from torch.nn.parallel import DistributedDataParallel as DDP @@ -52,7 +53,7 @@ def evaluate(model, device, test_loader): def get_model_norm(model): tot_norm = 0 - for name, params in ddp_model.named_parameters(): + for name, params in model.named_parameters(): tot_norm += torch.norm(params.data) return tot_norm @@ -91,18 +92,17 @@ def demo_basic(rank, world_size): device = torch.device("cuda:{}".format(rank)) - for epoch in range(5): + for epoch in range(15): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) if epoch % 3 == 0: - if rank == 0: - # prune model - print("Gonna try to prune model") - for name, params in ddp_model.named_parameters(): - # basically, prune everything - if re.match('.*\.weight', name) or re.match('.*\.bias', name):: - params.data = torch.zeros_like(params.data) - - print("Model Norm: {}".format(get_model_norm(ddp_model))) + # prune model + print("Rank: {} | Gonna try to prune model".format(rank)) + for name, params in ddp_model.named_parameters(): + # basically, prune everything + if re.match('.*\.weight', name) or re.match('.*\.bias', name): + params.data = torch.zeros_like(params.data) + + print("Rank: {} | Model Norm: {}".format(rank, get_model_norm(ddp_model))) # Save and evaluate model routinely if epoch % 2 == 0: if rank == 0: @@ -116,6 +116,7 @@ def demo_basic(rank, world_size): total_data_size = [0, 0] for data in train_loader: + print("Rank: {} | Model Norm: {}".format(rank, get_model_norm(ddp_model))) inputs, labels = data[0].to(device).reshape(-1, 32*32*3), data[1].to(device) # print("Device: {} | Batch Size: {} | Label sum: {}".format(rank, data[1].shape[0], torch.sum(data[1]))) total_data_size[rank] += data[1].shape[0] From 6334d1f75201871b13f9f5b46d6b6c3b51fcaa92 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 20:57:15 -0500 Subject: [PATCH 44/72] adding code to make ddp work --- main.py | 57 +++++++++++++++++++++++++++------------------------ main_utils.py | 29 ++++++++------------------ 2 files changed, 39 insertions(+), 47 deletions(-) diff --git a/main.py b/main.py index cc1af83e..90ea1070 100644 --- a/main.py +++ b/main.py @@ -6,56 +6,58 @@ def main(): print("\n\nBeginning of process.") print_time() set_seed(parser_args.seed * parser_args.trial_num) - #set_seed(parser_args.seed + parser_args.trial_num - 1) + # set_seed(parser_args.seed + parser_args.trial_num - 1) - # parser_args.distributed = parser_args.world_size > 1 or parser_args.multiprocessing_distributed + # world size = ngpus_per_node since we are assuming single node ngpus_per_node = torch.cuda.device_count() if parser_args.multiprocessing_distributed: - setup_distributed(ngpus_per_node) - mp.spawn(main_worker, nprocs=ngpus_per_node, - args=(ngpus_per_node,), join=True) + assert ngpus_per_node >= 2, f"Requires at least 2 GPUs to run, but got {ngpus_per_node}" + mp.spawn(main_worker, args=(ngpus_per_node,), nprocs=ngpus_per_node, join=True) else: # Simply call main_worker function main_worker(parser_args.gpu, ngpus_per_node) def main_worker(gpu, ngpus_per_node): - train, validate, modifier = get_trainer(parser_args) + # NOTE: gpu = rank in the multiprocessing setting parser_args.gpu = gpu + if parser_args.gpu is not None: print("Use GPU: {} for training".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: - parser_args.rank = parser_args.rank * ngpus_per_node + parser_args.gpu - # When using a single GPU per process and per DistributedDataParallel, we need to divide the batch size - # ourselves based on the total number of GPUs we have + parser_args.rank = parser_args.gpu + setup_distributed(rank, ngpus_per_node) + # if using ddp, divide batch size per gpu parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) - parser_args.num_workers = int( - (parser_args.num_workers + ngpus_per_node - 1) / ngpus_per_node) - # Since we have ngpus_per_node processes per node, the total world_size - # needs to be adjusted accordingly - parser_args.world_size = ngpus_per_node * parser_args.world_size - idty_str = get_idty_str(parser_args) - if parser_args.subfolder is not None: - if not os.path.isdir('results/'): - os.mkdir('results/') - result_subroot = 'results/' + parser_args.subfolder + '/' - if not os.path.isdir(result_subroot): - os.mkdir(result_subroot) - result_root = result_subroot + '/results_' + idty_str + '/' - else: - result_root = 'results/results_' + idty_str + '/' - if not os.path.isdir(result_root): - os.mkdir(result_root) + train, validate, modifier = get_trainer(parser_args) model = get_model(parser_args) - print_model(model, parser_args) + + if parser_args == 0: + idty_str = get_idty_str(parser_args) + if parser_args.subfolder is not None: + if not os.path.isdir('results/'): + os.mkdir('results/') + result_subroot = 'results/' + parser_args.subfolder + '/' + if not os.path.isdir(result_subroot): + os.mkdir(result_subroot) + result_root = result_subroot + '/results_' + idty_str + '/' + else: + result_root = 'results/results_' + idty_str + '/' + + if not os.path.isdir(result_root): + os.mkdir(result_root) + print_model(model, parser_args) if parser_args.weight_training: model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) model = switch_to_wt(model) + model = set_gpu(parser_args, model) + if parser_args.pretrained: pretrained(parser_args.pretrained, model) if parser_args.pretrained2: @@ -64,6 +66,7 @@ def main_worker(gpu, ngpus_per_node): pretrained(parser_args.pretrained2, model2) else: model2 = None + optimizer = get_optimizer(parser_args, model) data = get_dataset(parser_args) scheduler = get_scheduler(optimizer, parser_args.lr_policy) diff --git a/main_utils.py b/main_utils.py index 7f54a7cd..193e9661 100644 --- a/main_utils.py +++ b/main_utils.py @@ -504,15 +504,12 @@ def get_mask(model): return mask, flat_tensor -def setup_distributed(ngpus_per_node): - # for debugging - # os.environ['NCCL_DEBUG'] = 'INFO' - # os.environ['TORCH_DISTRIBUTED_DEBUG'] = 'INFO' - - # setup environment +def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' os.environ['MASTER_PORT'] = '29500' + dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) + def cleanup_distributed(): torch.distributed.destroy_process_group() @@ -886,21 +883,13 @@ def get_trainer(parser_args): def set_gpu(parser_args, model): assert torch.cuda.is_available(), "CPU-only experiments currently unsupported" - if parser_args.gpu is not None: - torch.cuda.set_device(parser_args.gpu) - model.cuda(parser_args.gpu) + # torch.cuda.set_device(parser_args.gpu) + model.to(parser_args.gpu) - if parser_args.multiprocessing_distributed: - torch.distributed.init_process_group( - backend=parser_args.dist_backend, - init_method='env://', - world_size=parser_args.world_size, - rank=parser_args.rank - ) - model = nn.parallel.DistributedDataParallel( - model, device_ids=[parser_args.gpu], find_unused_parameters=True) - else: - device = torch.device("cpu") + if parser_args.multiprocessing_distributed: + # TODO: not sure about find_unused_parameters. Need to check + model = nn.parallel.DistributedDataParallel( + model, device_ids=[parser_args.gpu], find_unused_parameters=True) return model From 9e1f0de4d9aa9c1098ce22db3bd81930159d9e64 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 21:06:28 -0500 Subject: [PATCH 45/72] trying out how args work in ddp --- ddp_args_helper.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ ddp_poc.py | 2 ++ 2 files changed, 55 insertions(+) create mode 100644 ddp_args_helper.py diff --git a/ddp_args_helper.py b/ddp_args_helper.py new file mode 100644 index 00000000..73324903 --- /dev/null +++ b/ddp_args_helper.py @@ -0,0 +1,53 @@ +import argparse +import sys +import yaml + +from configs import parser as _parser + +global parser_args + +class ArgsHelper: + def parse_arguments(self, jupyter_mode=False): + parser = argparse.ArgumentParser(description="Pruning random networks") + + # Config/Hyperparameters + parser.add_argument( + "--gpu", + type=int, + default=0, + help="gpu" + ) + parser.add_argument( + "--name", + default=None, + type=str, + help="Name of experiment" + ) + + if jupyter_mode: + args = parser.parse_args("") + else: + args = parser.parse_args() + self.get_config(args, jupyter_mode) + + return args + + def isNotebook(self): + try: + shell = get_ipython().__class__.__name__ + if shell == 'ZMQInteractiveShell': + return True # Jupyter notebook or qtconsole + elif shell == 'TerminalInteractiveShell': + return False # Terminal running IPython + else: + return False # Other type (?) + except NameError: + return False # Probably standard Python interpreter + + def get_args(self, jupyter_mode=False): + global parser_args + jupyter_mode = self.isNotebook() + parser_args = self.parse_arguments(jupyter_mode) + +argshelper = ArgsHelper() +argshelper.get_args() diff --git a/ddp_poc.py b/ddp_poc.py index 04a21e52..f06a311e 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -13,6 +13,7 @@ import re from torch.nn.parallel import DistributedDataParallel as DDP +from ddp_args_helper import parser_args def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' @@ -60,6 +61,7 @@ def get_model_norm(model): def demo_basic(rank, world_size): print(f"Running basic DDP example on rank {rank}.") + print("Parser args: gpu={}, name={}".format(parser_args.gpu, parser_args.name)) setup(rank, world_size) # create model and move it to GPU with id rank From f616ab6117e55030bba9489d1f63fcda6c0c5ff6 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 21:12:12 -0500 Subject: [PATCH 46/72] trying out stuff with parser_args in ddp setting --- ddp_args_helper.py | 3 +-- ddp_poc.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ddp_args_helper.py b/ddp_args_helper.py index 73324903..04909374 100644 --- a/ddp_args_helper.py +++ b/ddp_args_helper.py @@ -19,7 +19,7 @@ def parse_arguments(self, jupyter_mode=False): ) parser.add_argument( "--name", - default=None, + default="blah", type=str, help="Name of experiment" ) @@ -28,7 +28,6 @@ def parse_arguments(self, jupyter_mode=False): args = parser.parse_args("") else: args = parser.parse_args() - self.get_config(args, jupyter_mode) return args diff --git a/ddp_poc.py b/ddp_poc.py index f06a311e..be34223e 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -62,6 +62,8 @@ def get_model_norm(model): def demo_basic(rank, world_size): print(f"Running basic DDP example on rank {rank}.") print("Parser args: gpu={}, name={}".format(parser_args.gpu, parser_args.name)) + print("Setting gpu now, let's see what happens") + parser_args.gpu = rank setup(rank, world_size) # create model and move it to GPU with id rank @@ -96,6 +98,7 @@ def demo_basic(rank, world_size): for epoch in range(15): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) + print("Local Rank: {} | Parser args: gpu={}, Name={}".format(parser_args.gpu, parser_args.name)) if epoch % 3 == 0: # prune model print("Rank: {} | Gonna try to prune model".format(rank)) From e6b93a698bbd6bfe35fce0766972d6d0336ce2a7 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 21:16:06 -0500 Subject: [PATCH 47/72] adding ddp_utils --- ddp_poc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ddp_poc.py b/ddp_poc.py index be34223e..6fb105ce 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -14,6 +14,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP from ddp_args_helper import parser_args +from ddp_utils import do_something def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' @@ -131,6 +132,8 @@ def demo_basic(rank, world_size): loss.backward() optimizer.step() print("End of epoch total batch sizes: {}".format(total_data_size)) + + do_something_outside() # optimizer.zero_grad() # outputs = ddp_model(torch.randn(20, 10)) # labels = torch.randn(20, 5).to(rank) From f202a42886f6fc60f83ec7fdc2ce8c6c9609d7dd Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 21:17:16 -0500 Subject: [PATCH 48/72] adding ddp_utils --- ddp_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 ddp_utils.py diff --git a/ddp_utils.py b/ddp_utils.py new file mode 100644 index 00000000..0ed4b747 --- /dev/null +++ b/ddp_utils.py @@ -0,0 +1,6 @@ +from args_helper import parser_args + +def do_something(rank): + print("Re-imported parser_args, time to see if something funky happened.") + print("Local Rank: {} | parser_args.gpu={}, parser_args.name={}".format(parser_args.gpu, parser_args.name)) + return -1 \ No newline at end of file From 513b8a6397511fbd8b6550d08d8fce9cdb361f2d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Apr 2022 21:24:55 -0500 Subject: [PATCH 49/72] looks like reimporting doesn't hurt the global values --- ddp_poc.py | 6 +++--- ddp_utils.py | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 6fb105ce..5e291cf0 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -14,7 +14,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP from ddp_args_helper import parser_args -from ddp_utils import do_something +from ddp_utils import do_something_outside def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' @@ -99,7 +99,7 @@ def demo_basic(rank, world_size): for epoch in range(15): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) - print("Local Rank: {} | Parser args: gpu={}, Name={}".format(parser_args.gpu, parser_args.name)) + print("Local Rank: {} | Parser args: gpu={}, Name={}".format(rank, parser_args.gpu, parser_args.name)) if epoch % 3 == 0: # prune model print("Rank: {} | Gonna try to prune model".format(rank)) @@ -133,7 +133,7 @@ def demo_basic(rank, world_size): optimizer.step() print("End of epoch total batch sizes: {}".format(total_data_size)) - do_something_outside() + do_something_outside(rank) # optimizer.zero_grad() # outputs = ddp_model(torch.randn(20, 10)) # labels = torch.randn(20, 5).to(rank) diff --git a/ddp_utils.py b/ddp_utils.py index 0ed4b747..5f7a0c72 100644 --- a/ddp_utils.py +++ b/ddp_utils.py @@ -1,6 +1,5 @@ -from args_helper import parser_args +from ddp_args_helper import parser_args -def do_something(rank): - print("Re-imported parser_args, time to see if something funky happened.") - print("Local Rank: {} | parser_args.gpu={}, parser_args.name={}".format(parser_args.gpu, parser_args.name)) - return -1 \ No newline at end of file +def do_something_outside(rank): + print("Re-imported parser_args, time to see if something funky happened. --> Local Rank: {} | parser_args.gpu={}, parser_args.name={}".format(rank, parser_args.gpu, parser_args.name)) + return -1 From 02a6627c1036644d17040cd8f09c8c791d86424b Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 25 Apr 2022 22:06:26 -0500 Subject: [PATCH 50/72] adding more code to implement ddp --- data/imagenet.py | 52 ++++++++----- main.py | 186 ++++++++++++++++++++++------------------------- main_utils.py | 56 +++++++------- 3 files changed, 151 insertions(+), 143 deletions(-) diff --git a/data/imagenet.py b/data/imagenet.py index e8ec5aee..0379647a 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -2,8 +2,9 @@ import torch from torchvision import datasets, transforms - import torch.multiprocessing +from args_helper import parser_args +from torch.utils.data import random_split torch.multiprocessing.set_sharing_strategy("file_system") @@ -11,13 +12,12 @@ class ImageNet: def __init__(self, args): super(ImageNet, self).__init__() -# data_root = os.path.join(args.data, "imagenet") data_root = args.data use_cuda = torch.cuda.is_available() # Data loading code - kwargs = {"num_workers": args.num_workers, "pin_memory": True} if use_cuda else {} + kwargs = {"num_workers": parser_args.num_workers, "pin_memory": True} if use_cuda else {} # Data loading code traindir = os.path.join(data_root, "train") @@ -27,7 +27,7 @@ def __init__(self, args): mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ) - train_dataset = datasets.ImageFolder( + dataset = datasets.ImageFolder( traindir, transforms.Compose( [ @@ -39,6 +39,27 @@ def __init__(self, args): ), ) + test_dataset = datasets.ImageFolder( + valdir, + transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + normalize, + ] + ), + ) + + if parser_args.use_full_data: + train_dataset = dataset + # use_full_data => we are not tuning hyperparameters + validation_dataset = test_dataset + else: + val_size = 10000 + train_size = len(dataset) - val_size + train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) + if args.multiprocessing_distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: @@ -47,25 +68,22 @@ def __init__(self, args): self.train_loader = torch.utils.data.DataLoader( train_dataset, - batch_size=args.batch_size, + batch_size=parser_args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, **kwargs ) self.val_loader = torch.utils.data.DataLoader( - datasets.ImageFolder( - valdir, - transforms.Compose( - [ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - normalize, - ] - ), - ), - batch_size=args.batch_size, + test_dataset, + batch_size=parser_args.batch_size, shuffle=False, **kwargs ) + + self.actual_val_loader = torch.utils.data.DataLoader( + validation_dataset, + batch_size=parser_args.batch_size, + shuffle=True, + **kwargs + ) diff --git a/main.py b/main.py index 90ea1070..2b7ef2cd 100644 --- a/main.py +++ b/main.py @@ -70,7 +70,7 @@ def main_worker(gpu, ngpus_per_node): optimizer = get_optimizer(parser_args, model) data = get_dataset(parser_args) scheduler = get_scheduler(optimizer, parser_args.lr_policy) - #lr_policy = get_policy(parser_args.lr_policy)(optimizer, parser_args) + # lr_policy = get_policy(parser_args.lr_policy)(optimizer, parser_args) if parser_args.label_smoothing is None: criterion = nn.CrossEntropyLoss().cuda() else: @@ -112,10 +112,9 @@ def main_worker(gpu, ngpus_per_node): print("Overriding prune_rate to {}".format(parser_args.prune_rate)) #if parser_args.dataset == 'TinyImageNet': # print_num_dataset(data) - if not parser_args.weight_training: - print_layers(parser_args, model) - - + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed:: + if not parser_args.weight_training: + print_layers(parser_args, model) if parser_args.mixed_precision: scaler = torch.cuda.amp.GradScaler(enabled=True) # mixed precision @@ -142,7 +141,7 @@ def main_worker(gpu, ngpus_per_node): for epoch in range(parser_args.start_epoch, parser_args.epochs): if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) - #lr_policy(epoch, iteration=None) + # lr_policy(epoch, iteration=None) modifier(parser_args, epoch, model) cur_lr = get_lr(optimizer) @@ -166,37 +165,40 @@ def main_worker(gpu, ngpus_per_node): scheduler.step() # evaluate on validation set - start_validation = time.time() - if parser_args.algo in ['hc', 'hc_iter']: - br_acc1, br_acc5, br_acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding - print('Acc before rounding: {}'.format(br_acc1)) - acc_avg = 0 - for num_trial in range(parser_args.num_test): - cp_model = round_model(model, parser_args.round, noise=parser_args.noise, - ratio=parser_args.noise_ratio, rank=parser_args.gpu) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + start_validation = time.time() + if parser_args.algo in ['hc', 'hc_iter']: + br_acc1, br_acc5, br_acc10 = validate( + data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding + print('Acc before rounding: {}'.format(br_acc1)) + acc_avg = 0 + for num_trial in range(parser_args.num_test): + cp_model = round_model(model, parser_args.round, noise=parser_args.noise, + ratio=parser_args.noise_ratio, rank=parser_args.gpu) + acc1, acc5, acc10 = validate( + data.val_loader, cp_model, criterion, parser_args, writer, epoch) + acc_avg += acc1 + acc_avg /= parser_args.num_test + acc1 = acc_avg + print('Acc after rounding: {}'.format(acc1)) + val_acc1, val_acc5, val_acc10 = validate( + data.actual_val_loader, cp_model, criterion, parser_args, writer, epoch) + print('Validation Acc after rounding: {}'.format(val_acc1)) + else: acc1, acc5, acc10 = validate( - data.val_loader, cp_model, criterion, parser_args, writer, epoch) - acc_avg += acc1 - acc_avg /= parser_args.num_test - acc1 = acc_avg - print('Acc after rounding: {}'.format(acc1)) - val_acc1, val_acc5, val_acc10 = validate( - data.actual_val_loader, cp_model, criterion, parser_args, writer, epoch) - print('Validation Acc after rounding: {}'.format(val_acc1)) - else: - acc1, acc5, acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, epoch) - print('Acc: {}'.format(acc1)) - validation_time.update((time.time() - start_validation) / 60) + data.val_loader, model, criterion, parser_args, writer, epoch) + print('Acc: {}'.format(acc1)) + validation_time.update((time.time() - start_validation) / 60) # prune the model every T_{prune} epochs if not parser_args.weight_training and parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: prune(model) if parser_args.checkpoint_at_prune: - save_checkpoint_at_prune(model, parser_args) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + save_checkpoint_at_prune(model, parser_args) # get model sparsity + # DDP_TODO: This part could be a problem. What if models get out of sync? if not parser_args.weight_training: if parser_args.bottom_k_on_forward: cp_model = copy.deepcopy(model) @@ -212,7 +214,9 @@ def main_worker(gpu, ngpus_per_node): else: # haven't written a weight sparsity function yet avg_sparsity = -1 - print('Model avg sparsity: {}'.format(avg_sparsity)) + + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print('Model avg sparsity: {}'.format(avg_sparsity)) # if model has been "short-circuited", then no point in continuing training if avg_sparsity == 0: @@ -227,92 +231,78 @@ def main_worker(gpu, ngpus_per_node): break # update all results lists - epoch_list.append(epoch) - if parser_args.algo in ['hc', 'hc_iter']: - test_acc_before_round_list.append(br_acc1) - else: - # no before rounding for EP/weight training - test_acc_before_round_list.append(-1) - test_acc_list.append(acc1) - val_acc_list.append(val_acc1) - train_acc_list.append(train_acc1) - reg_loss_list.append(reg_loss) - model_sparsity_list.append(avg_sparsity) - - epoch_time.update((time.time() - end_epoch) / 60) - progress_overall.display(epoch) - progress_overall.write_to_tensorboard( - writer, prefix="diagnostics", global_step=epoch - ) - - if parser_args.conv_type == "SampleSubnetConv": - count = 0 - sum_pr = 0.0 - for n, m in model.named_modules(): - if isinstance(m, SampleSubnetConv): - # avg pr across 10 samples - pr = 0.0 - for _ in range(10): - pr += ( - (torch.rand_like(m.clamped_scores) >= m.clamped_scores) - .float() - .mean() - .item() - ) - pr /= 10.0 - writer.add_scalar("pr/{}".format(n), pr, epoch) - sum_pr += pr - count += 1 - - parser_args.prune_rate = sum_pr / count - writer.add_scalar("pr/average", parser_args.prune_rate, epoch) - - writer.add_scalar("test/lr", cur_lr, epoch) - end_epoch = time.time() - - if parser_args.algo in ['hc', 'hc_iter']: - results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, - 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) - else: - results_df = pd.DataFrame( - {'epoch': epoch_list, 'test_acc': test_acc_list, 'model_sparsity': model_sparsity_list}) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + epoch_list.append(epoch) + if parser_args.algo in ['hc', 'hc_iter']: + test_acc_before_round_list.append(br_acc1) + else: + # no before rounding for EP/weight training + test_acc_before_round_list.append(-1) + test_acc_list.append(acc1) + val_acc_list.append(val_acc1) + train_acc_list.append(train_acc1) + reg_loss_list.append(reg_loss) + model_sparsity_list.append(avg_sparsity) + + epoch_time.update((time.time() - end_epoch) / 60) + progress_overall.display(epoch) + progress_overall.write_to_tensorboard( + writer, prefix="diagnostics", global_step=epoch + ) + + writer.add_scalar("test/lr", cur_lr, epoch) + end_epoch = time.time() + + if parser_args.algo in ['hc', 'hc_iter']: + results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, + 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) + else: + results_df = pd.DataFrame( + {'epoch': epoch_list, 'test_acc': test_acc_list, 'model_sparsity': model_sparsity_list}) - if parser_args.results_filename: - results_filename = parser_args.results_filename - else: - results_filename = result_root + 'acc_and_sparsity.csv' - print("Writing results into: {}".format(results_filename)) - results_df.to_csv(results_filename, index=False) + if parser_args.results_filename: + results_filename = parser_args.results_filename + else: + results_filename = result_root + 'acc_and_sparsity.csv' + print("Writing results into: {}".format(results_filename)) + results_df.to_csv(results_filename, index=False) - # save checkpoint before fine-tuning - #torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + # save checkpoint before fine-tuning + torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') - print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") - print_time() + print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") + print_time() # finetune weights + # DDP_TODO: Check how DDP works with copy deepcopy cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: - print("Beginning fine-tuning") + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("Beginning fine-tuning") cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root) # print out the final acc - eval_and_print(validate, data.val_loader, cp_model, criterion, - parser_args, writer=None, description='final model after finetuning') - # save checkpoint after fine-tuning - #torch.save(cp_model.state_dict(), result_root + 'model_after_finetune.pth') + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + eval_and_print(validate, data.val_loader, cp_model, criterion, + parser_args, writer=None, description='final model after finetuning') + # save checkpoint after fine-tuning + torch.save(cp_model.state_dict(), result_root + 'model_after_finetune.pth') else: - print("Skipping finetuning!!!") + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("Skipping finetuning!!!") if not parser_args.skip_sanity_checks: do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root) else: - print("Skipping sanity checks!!!") + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("Skipping sanity checks!!!") - print("\n\nEnd of process. Exiting") - print_time() + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("\n\nEnd of process. Exiting") + print_time() if parser_args.multiprocessing_distributed: cleanup_distributed() diff --git a/main_utils.py b/main_utils.py index 193e9661..f4b26906 100644 --- a/main_utils.py +++ b/main_utils.py @@ -319,7 +319,6 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) # lr_policy(epoch, iteration=None) - # modifier(parser_args, epoch, model) cur_lr = get_lr(optimizer) print('epoch: {}, lr: {}'.format(epoch, cur_lr)) @@ -331,34 +330,35 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b train_time.update((time.time() - start_train) / 60) # evaluate on validation set - start_validation = time.time() - acc1, acc5, acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, epoch) - val_acc1, val_acc5, val_acc10 = validate( - data.actual_val_loader, model, criterion, parser_args, writer, epoch) - validation_time.update((time.time() - start_validation) / 60) - # copy & paste the sparsity of prev. epoch - avg_sparsity = model_sparsity_list[-1] - - # update all results lists - epoch_list.append(epoch) - test_acc_before_round_list.append(-1) - test_acc_list.append(acc1) - val_acc_list.append(val_acc1) - train_acc_list.append(train_acc1) - reg_loss_list.append(reg_loss) - model_sparsity_list.append(avg_sparsity) - - epoch_time.update((time.time() - end_epoch) / 60) - progress_overall.display(epoch) - progress_overall.write_to_tensorboard( - writer, prefix="diagnostics", global_step=epoch - ) - writer.add_scalar("test/lr", cur_lr, epoch) - end_epoch = time.time() + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + start_validation = time.time() + acc1, acc5, acc10 = validate( + data.val_loader, model, criterion, parser_args, writer, epoch) + val_acc1, val_acc5, val_acc10 = validate( + data.actual_val_loader, model, criterion, parser_args, writer, epoch) + validation_time.update((time.time() - start_validation) / 60) + # copy & paste the sparsity of prev. epoch + avg_sparsity = model_sparsity_list[-1] + + # update all results lists + epoch_list.append(epoch) + test_acc_before_round_list.append(-1) + test_acc_list.append(acc1) + val_acc_list.append(val_acc1) + train_acc_list.append(train_acc1) + reg_loss_list.append(reg_loss) + model_sparsity_list.append(avg_sparsity) + + epoch_time.update((time.time() - end_epoch) / 60) + progress_overall.display(epoch) + progress_overall.write_to_tensorboard( + writer, prefix="diagnostics", global_step=epoch + ) + writer.add_scalar("test/lr", cur_lr, epoch) + end_epoch = time.time() - results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, - 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) + results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, + 'regularization_loss': reg_loss_list, 'model_sparsity': model_sparsity_list}) if not chg_mask and not chg_weight: results_filename = result_root + 'acc_and_sparsity.csv' # elif chg_weight and shuffle: From 7f1e0f6b48aae20d717d6a00862eb3449f99a912 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 14:35:53 -0500 Subject: [PATCH 51/72] testing ddp with deepcopy --- ddp_poc.py | 27 +++++++++++++++++++++------ main.py | 2 ++ main_utils.py | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 5e291cf0..87414b96 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -15,6 +15,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP from ddp_args_helper import parser_args from ddp_utils import do_something_outside +import copy def setup(rank, world_size): os.environ['MASTER_ADDR'] = '127.0.0.1' @@ -97,7 +98,7 @@ def demo_basic(rank, world_size): device = torch.device("cuda:{}".format(rank)) - for epoch in range(15): + for epoch in range(5): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) print("Local Rank: {} | Parser args: gpu={}, Name={}".format(rank, parser_args.gpu, parser_args.name)) if epoch % 3 == 0: @@ -134,11 +135,25 @@ def demo_basic(rank, world_size): print("End of epoch total batch sizes: {}".format(total_data_size)) do_something_outside(rank) - # optimizer.zero_grad() - # outputs = ddp_model(torch.randn(20, 10)) - # labels = torch.randn(20, 5).to(rank) - # loss_fn(outputs, labels).backward() - # optimizer.step() + + print("Local rank: {} | Entering barrier".format(rank)) + dist.barrier() + print("Local rank: {} | Past barrier".format(rank)) + cp_model = copy.deepcopy(model) + print("Local rank: {} | Copied Model".format(rank)) + + for data in train_loader: + print("Rank: {} | Copied Model Norm: {}".format(rank, get_model_norm(cp_model))) + inputs, labels = data[0].to(device).reshape(-1, 32*32*3), data[1].to(device) + # print("Device: {} | Batch Size: {} | Label sum: {}".format(rank, data[1].shape[0], torch.sum(data[1]))) + total_data_size[rank] += data[1].shape[0] + optimizer.zero_grad() + outputs = cp_model(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + print("End of epoch total batch sizes: {}".format(total_data_size)) + cleanup() diff --git a/main.py b/main.py index 2b7ef2cd..5d810dde 100644 --- a/main.py +++ b/main.py @@ -276,6 +276,8 @@ def main_worker(gpu, ngpus_per_node): # finetune weights # DDP_TODO: Check how DDP works with copy deepcopy + dist.barrier() + cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: diff --git a/main_utils.py b/main_utils.py index f4b26906..37a25a73 100644 --- a/main_utils.py +++ b/main_utils.py @@ -18,7 +18,7 @@ import torch.backends.cudnn as cudnn import torch.optim import torch.utils.data -import torch.utils.data.distributed +import torch.utils.data.distributed as dist import torch.multiprocessing as mp import sys From 6cc557bdd4c06253a784d42f5016b58a8c2a9af2 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 15:24:19 -0500 Subject: [PATCH 52/72] looks like copy.deepcopy causes sync issues --- ddp_poc.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ddp_poc.py b/ddp_poc.py index 87414b96..5efc1b63 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -98,7 +98,7 @@ def demo_basic(rank, world_size): device = torch.device("cuda:{}".format(rank)) - for epoch in range(5): + for epoch in range(2): print("Local Rank: {}, Epoch: {}, Training ...".format(rank, epoch)) print("Local Rank: {} | Parser args: gpu={}, Name={}".format(rank, parser_args.gpu, parser_args.name)) if epoch % 3 == 0: @@ -120,7 +120,7 @@ def demo_basic(rank, world_size): print("-" * 75) ddp_model.train() - total_data_size = [0, 0] + total_data_size = [0, 0, 0, 0] for data in train_loader: print("Rank: {} | Model Norm: {}".format(rank, get_model_norm(ddp_model))) @@ -142,6 +142,7 @@ def demo_basic(rank, world_size): cp_model = copy.deepcopy(model) print("Local rank: {} | Copied Model".format(rank)) + optimizer = optim.SGD(cp_model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-5) for data in train_loader: print("Rank: {} | Copied Model Norm: {}".format(rank, get_model_norm(cp_model))) inputs, labels = data[0].to(device).reshape(-1, 32*32*3), data[1].to(device) @@ -154,6 +155,11 @@ def demo_basic(rank, world_size): optimizer.step() print("End of epoch total batch sizes: {}".format(total_data_size)) + print("Local rank: {} | Entering barrier".format(rank)) + dist.barrier() + print("Local rank: {} | Past barrier".format(rank)) + print("Rank: {} | Copied Model Norm: {}".format(rank, get_model_norm(cp_model))) + cleanup() From dd0ba73848da714b61f506217223eb214410eae9 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 15:38:51 -0500 Subject: [PATCH 53/72] fixing typo in ddp deepcopy --- ddp_poc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddp_poc.py b/ddp_poc.py index 5efc1b63..021aa4ea 100644 --- a/ddp_poc.py +++ b/ddp_poc.py @@ -139,7 +139,7 @@ def demo_basic(rank, world_size): print("Local rank: {} | Entering barrier".format(rank)) dist.barrier() print("Local rank: {} | Past barrier".format(rank)) - cp_model = copy.deepcopy(model) + cp_model = copy.deepcopy(ddp_model) print("Local rank: {} | Copied Model".format(rank)) optimizer = optim.SGD(cp_model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-5) From 1d21c5a227ad3b6ff6a194591aa7969de9a76619 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 16:23:47 -0500 Subject: [PATCH 54/72] finishing up ddp for sanity checks and finetune --- data/cifar.py | 19 +++++++++++++++--- data/imagenet.py | 2 +- main.py | 15 ++++++++++----- main_utils.py | 50 +++++++++++++++++++++++++++++++++--------------- 4 files changed, 62 insertions(+), 24 deletions(-) diff --git a/data/cifar.py b/data/cifar.py index 696ff388..5a5159ff 100644 --- a/data/cifar.py +++ b/data/cifar.py @@ -23,6 +23,11 @@ def __init__(self, args): mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262] ) + if args.multiprocessing_distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + dataset = torchvision.datasets.CIFAR10( root=data_root, train=True, @@ -54,13 +59,21 @@ def __init__(self, args): train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) self.train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=parser_args.batch_size, shuffle=True, **kwargs + train_dataset, + batch_size=parser_args.batch_size, + shuffle=(train_sampler is None), + sampler=train_sampler, + **kwargs ) self.val_loader = torch.utils.data.DataLoader( - test_dataset, batch_size=parser_args.batch_size, shuffle=False, **kwargs + test_dataset, + batch_size=parser_args.batch_size, + shuffle=False, **kwargs ) self.actual_val_loader = torch.utils.data.DataLoader( - validation_dataset, batch_size=parser_args.batch_size, shuffle=True, **kwargs + validation_dataset, + batch_size=parser_args.batch_size, + shuffle=False, **kwargs ) diff --git a/data/imagenet.py b/data/imagenet.py index 0379647a..a78ac4f4 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -84,6 +84,6 @@ def __init__(self, args): self.actual_val_loader = torch.utils.data.DataLoader( validation_dataset, batch_size=parser_args.batch_size, - shuffle=True, + shuffle=False, **kwargs ) diff --git a/main.py b/main.py index 5d810dde..d0d10603 100644 --- a/main.py +++ b/main.py @@ -112,7 +112,7 @@ def main_worker(gpu, ngpus_per_node): print("Overriding prune_rate to {}".format(parser_args.prune_rate)) #if parser_args.dataset == 'TinyImageNet': # print_num_dataset(data) - if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed:: + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: if not parser_args.weight_training: print_layers(parser_args, model) @@ -131,9 +131,9 @@ def main_worker(gpu, ngpus_per_node): do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, reg_loss_list, model_sparsity_list, parser_args.results_root) - #cp_model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, + # cp_model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, # ratio=parser_args.noise_ratio, rank=parser_args.gpu) - #print(get_model_sparsity(cp_model)) + # print(get_model_sparsity(cp_model)) return @@ -275,8 +275,10 @@ def main_worker(gpu, ngpus_per_node): print_time() # finetune weights - # DDP_TODO: Check how DDP works with copy deepcopy + # DDP works surprisingly well with copy deepcopy. Might cause memory issues TODO + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: @@ -294,10 +296,13 @@ def main_worker(gpu, ngpus_per_node): if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: print("Skipping finetuning!!!") + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if not parser_args.skip_sanity_checks: do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root) - else: if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: print("Skipping sanity checks!!!") diff --git a/main_utils.py b/main_utils.py index 37a25a73..61bb83ca 100644 --- a/main_utils.py +++ b/main_utils.py @@ -94,6 +94,11 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b print("Beginning Sanity Checks:") # do the sanity check for shuffled mask/weights, reinit weights print("Sanity Check 1: Weight Reinit") + + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root, reinit=True, chg_weight=True) @@ -105,6 +110,11 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b reg_loss_list, model_sparsity_list, result_root, shuffle=True, chg_weight=True) ''' print("Sanity Check 2: Mask Reshuffle") + + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, reg_loss_list, model_sparsity_list, result_root, shuffle=True, chg_mask=True) @@ -262,12 +272,17 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b post_round_sparsity = get_model_sparsity(model) # apply reinit/shuffling masks/weights (if necessary) + # DDP_TODO: Check if this works well model = redraw(model, shuffle=shuffle, reinit=reinit, invert=invert, chg_mask=chg_mask, chg_weight=chg_weight) # switch to weight training mode (turn on the requires_grad for weight/bias, and turn off the requires_grad for other parameters) model = switch_to_wt(model) + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + # not to use score regulaization during the weight training parser_args.regularization = False @@ -298,20 +313,24 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b train, validate, modifier = get_trainer(parser_args) # check the performance of loaded model (after rounding) - acc1, acc5, acc10 = validate( - data.val_loader, model, criterion, parser_args, writer, parser_args.epochs-1) - val_acc1, val_acc5, val_acc10 = validate( - data.actual_val_loader, model, criterion, parser_args, writer, parser_args.epochs-1) - train_acc1, train_acc5, train_acc10 = validate( - data.train_loader, model, criterion, parser_args, writer, parser_args.epochs-1) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + acc1, acc5, acc10 = validate( + data.val_loader, model, criterion, parser_args, writer, parser_args.epochs-1) + val_acc1, val_acc5, val_acc10 = validate( + data.actual_val_loader, model, criterion, parser_args, writer, parser_args.epochs-1) + train_acc1, train_acc5, train_acc10 = validate( + data.train_loader, model, criterion, parser_args, writer, parser_args.epochs-1) + avg_sparsity = post_round_sparsity - epoch_list.append(parser_args.epochs-1) - test_acc_before_round_list.append(-1) - test_acc_list.append(acc1) - val_acc_list.append(val_acc1) - train_acc_list.append(train_acc1) - reg_loss_list.append(0.0) - model_sparsity_list.append(avg_sparsity) + + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + epoch_list.append(parser_args.epochs-1) + test_acc_before_round_list.append(-1) + test_acc_list.append(acc1) + val_acc_list.append(val_acc1) + train_acc_list.append(train_acc1) + reg_loss_list.append(0.0) + model_sparsity_list.append(avg_sparsity) end_epoch = time.time() for epoch in range(parser_args.epochs, parser_args.epochs*2): @@ -372,8 +391,9 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b else: raise NotImplementedError - print("Writing results into: {}".format(results_filename)) - results_df.to_csv(results_filename, index=False) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + print("Writing results into: {}".format(results_filename)) + results_df.to_csv(results_filename, index=False) scheduler.step() return model From 691cffacd9abade09c0ad84a7983153dd15c728f Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 16:40:29 -0500 Subject: [PATCH 55/72] minor bugfixes for ddp --- cifar_exec.sh | 8 ++--- configs/ddp_debug/conf1.yml | 64 +++++++++++++++++++++++++++++++++++++ data/cifar.py | 10 +++--- data/imagenet.py | 4 +-- main.py | 2 +- main_utils.py | 2 +- trainers/default.py | 1 - 7 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 configs/ddp_debug/conf1.yml diff --git a/cifar_exec.sh b/cifar_exec.sh index 43eefb4f..7d7a1efd 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -64,15 +64,15 @@ BLOCK #:< "$log_root$trial$log_end" 2>&1 & diff --git a/configs/ddp_debug/conf1.yml b/configs/ddp_debug/conf1.yml new file mode 100644 index 00000000..23e5733c --- /dev/null +++ b/configs/ddp_debug/conf1.yml @@ -0,0 +1,64 @@ +# subfolder: target_sparsity_0_59_unflagT_real + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 20 +wd: 0.0 +momentum: 0.9 +batch_size: 512 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + + + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 + +# ===== Hardware setup ===== # +workers: 8 +# gpu: 1 +multiprocessing_distributed: True + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/data/cifar.py b/data/cifar.py index 5a5159ff..a782b917 100644 --- a/data/cifar.py +++ b/data/cifar.py @@ -23,11 +23,6 @@ def __init__(self, args): mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262] ) - if args.multiprocessing_distributed: - train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) - else: - train_sampler = None - dataset = torchvision.datasets.CIFAR10( root=data_root, train=True, @@ -58,6 +53,11 @@ def __init__(self, args): train_size = len(dataset) - val_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) + if parser_args.multiprocessing_distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + else: + train_sampler = None + self.train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=parser_args.batch_size, diff --git a/data/imagenet.py b/data/imagenet.py index a78ac4f4..5e978d27 100644 --- a/data/imagenet.py +++ b/data/imagenet.py @@ -12,7 +12,7 @@ class ImageNet: def __init__(self, args): super(ImageNet, self).__init__() - data_root = args.data + data_root = parser_args.data use_cuda = torch.cuda.is_available() @@ -60,7 +60,7 @@ def __init__(self, args): train_size = len(dataset) - val_size train_dataset, validation_dataset = random_split(dataset, [train_size, val_size]) - if args.multiprocessing_distributed: + if parser_args.multiprocessing_distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None diff --git a/main.py b/main.py index d0d10603..b7e1c364 100644 --- a/main.py +++ b/main.py @@ -28,7 +28,7 @@ def main_worker(gpu, ngpus_per_node): if parser_args.multiprocessing_distributed: parser_args.rank = parser_args.gpu - setup_distributed(rank, ngpus_per_node) + setup_distributed(parser_args.rank, ngpus_per_node) # if using ddp, divide batch size per gpu parser_args.batch_size = int(parser_args.batch_size / ngpus_per_node) diff --git a/main_utils.py b/main_utils.py index 61bb83ca..06caaaf9 100644 --- a/main_utils.py +++ b/main_utils.py @@ -18,7 +18,7 @@ import torch.backends.cudnn as cudnn import torch.optim import torch.utils.data -import torch.utils.data.distributed as dist +import torch.distributed as dist import torch.multiprocessing as mp import sys diff --git a/trainers/default.py b/trainers/default.py index 8325fccb..6597353f 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -87,7 +87,6 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # compute gradient and do SGD step optimizer.zero_grad() - import ipdb; ipdb.set_trace() if scaler is None: loss.backward() optimizer.step() From ffca64810cb1ba85a4366fcb130068c3fee18659 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 16:43:27 -0500 Subject: [PATCH 56/72] removing .cuda() from the codebase --- main.py | 2 +- utils/net_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b7e1c364..87172299 100644 --- a/main.py +++ b/main.py @@ -72,7 +72,7 @@ def main_worker(gpu, ngpus_per_node): scheduler = get_scheduler(optimizer, parser_args.lr_policy) # lr_policy = get_policy(parser_args.lr_policy)(optimizer, parser_args) if parser_args.label_smoothing is None: - criterion = nn.CrossEntropyLoss().cuda() + criterion = nn.CrossEntropyLoss() else: criterion = LabelSmoothing(smoothing=parser_args.label_smoothing) # if isinstance(model, nn.parallel.DistributedDataParallel): diff --git a/utils/net_utils.py b/utils/net_utils.py index 46462981..58434248 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -563,7 +563,7 @@ def get_regularization_loss(model, regularizer='L2', lmbda=1, alpha=1, alpha_pri def get_special_reg_sum(layer): # reg_loss = \sum_{i} w_i^2 * p_i(1-p_i) # NOTE: alpha = alpha' = 1 here. Change if needed. - reg_sum = torch.tensor(0.).cuda() + reg_sum = torch.tensor(0.).to(parser_args.gpu) w_i = layer.weight p_i = layer.scores reg_sum += torch.sum(torch.pow(w_i, 2) * @@ -576,7 +576,7 @@ def get_special_reg_sum(layer): return reg_sum #pdb.set_trace() - regularization_loss = torch.tensor(0.).cuda() + regularization_loss = torch.tensor(0.).to(parser_args.gpu) if regularizer == 'L2': # reg_loss = ||p||_2^2 for name, params in model.named_parameters(): From e1c9810e358252bbc9e635529db8591871c56b95 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 17:10:09 -0500 Subject: [PATCH 57/72] trying to catch the rank=0 leak --- main.py | 2 +- main_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 87172299..43ab73d2 100644 --- a/main.py +++ b/main.py @@ -35,7 +35,7 @@ def main_worker(gpu, ngpus_per_node): train, validate, modifier = get_trainer(parser_args) model = get_model(parser_args) - if parser_args == 0: + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: idty_str = get_idty_str(parser_args) if parser_args.subfolder is not None: if not os.path.isdir('results/'): diff --git a/main_utils.py b/main_utils.py index 06caaaf9..694e101e 100644 --- a/main_utils.py +++ b/main_utils.py @@ -903,7 +903,7 @@ def get_trainer(parser_args): def set_gpu(parser_args, model): assert torch.cuda.is_available(), "CPU-only experiments currently unsupported" - # torch.cuda.set_device(parser_args.gpu) + torch.cuda.set_device(parser_args.gpu) model.to(parser_args.gpu) if parser_args.multiprocessing_distributed: From 64cbf57c4719eeaa7951fc163d0295da2251231c Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 26 Apr 2022 17:43:28 -0500 Subject: [PATCH 58/72] I think ddp works! --- configs/ddp_debug/conf1.yml | 1 + utils/net_utils.py | 6 +++--- utils/utils.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/configs/ddp_debug/conf1.yml b/configs/ddp_debug/conf1.yml index 23e5733c..7e02189e 100644 --- a/configs/ddp_debug/conf1.yml +++ b/configs/ddp_debug/conf1.yml @@ -56,6 +56,7 @@ lmbda: 0.0001 # 1e-4 workers: 8 # gpu: 1 multiprocessing_distributed: True +mixed_precision: True # ===== Checkpointing ===== # checkpoint_at_prune: False diff --git a/utils/net_utils.py b/utils/net_utils.py index 58434248..9e5828d9 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -348,9 +348,9 @@ def round_model(model, round_scheme, noise=False, ratio=0.0, rank=None): params.data = (params.data + delta) % 2 ''' - if isinstance(model, nn.parallel.DistributedDataParallel): - cp_model = nn.parallel.DistributedDataParallel( - cp_model, device_ids=[rank], find_unused_parameters=True) + # if isinstance(model, nn.parallel.DistributedDataParallel): + # cp_model = nn.parallel.DistributedDataParallel( + # cp_model, device_ids=[rank], find_unused_parameters=True) return cp_model diff --git a/utils/utils.py b/utils/utils.py index 2ed92dba..bc4c509a 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -24,9 +24,9 @@ def set_seed(seed): torch.cuda.manual_seed_all(seed) np.random.seed(seed) os.environ['PYTHONHASHSEED'] = str(seed) - # making sure GPU runs are deterministic even if they are slower + # set this=True if you want deterministic runs torch.backends.cudnn.deterministic = False - # this causes the code to vary across runs. I don't want that for now. + # set this=False if you want deterministic runs torch.backends.cudnn.benchmark = True print("Seeded everything: {}".format(seed)) From eb12a2d874b719e85c1445f2753a8ebb59247e95 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:00:53 -0500 Subject: [PATCH 59/72] skipping checkpoint, trying to get rid of tqdm --- cifar_exec.sh | 2 +- configs/ddp_debug/conf1.yml | 2 -- main.py | 30 +++++++++++++++----------- main_utils.py | 21 +++++++++++-------- trainers/default.py | 42 ++++++++++++++++++------------------- 5 files changed, 52 insertions(+), 45 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 7d7a1efd..6ba68efa 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -72,7 +72,7 @@ subfolder_root="ddp_debug_" for trial in 1 do - CUDA_VISIBLE_DEVICES=0,2,3 python main.py \ + CUDA_VISIBLE_DEVICES=0,1 python main.py \ --config "$conf_file$conf_end" \ --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 & diff --git a/configs/ddp_debug/conf1.yml b/configs/ddp_debug/conf1.yml index 7e02189e..d94092b8 100644 --- a/configs/ddp_debug/conf1.yml +++ b/configs/ddp_debug/conf1.yml @@ -37,8 +37,6 @@ init: signed_constant score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True - - # ===== Rounding ===== # round: naive noise: True diff --git a/main.py b/main.py index 43ab73d2..95bdda4b 100644 --- a/main.py +++ b/main.py @@ -46,6 +46,9 @@ def main_worker(gpu, ngpus_per_node): result_root = result_subroot + '/results_' + idty_str + '/' else: result_root = 'results/results_' + idty_str + '/' + else: + idty_str = get_idty_str(parser_args) + result_root = 'results/results_' + idty_str + '/' if not os.path.isdir(result_root): os.mkdir(result_root) @@ -266,19 +269,21 @@ def main_worker(gpu, ngpus_per_node): results_filename = result_root + 'acc_and_sparsity.csv' print("Writing results into: {}".format(results_filename)) results_df.to_csv(results_filename, index=False) + print("Local rank: {} | About to enter save model logic".format(parser_args.gpu)) + if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: + # save checkpoint before fine-tuning + # torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') - if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: - # save checkpoint before fine-tuning - torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') - - print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") - print_time() + print("\n\nHigh accuracy subnetwork found! Rest is just finetuning") + print("Local rank: {}".format(parser_args.gpu)) + print_time() # finetune weights # DDP works surprisingly well with copy deepcopy. Might cause memory issues TODO - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) if not parser_args.skip_fine_tune: @@ -296,9 +301,10 @@ def main_worker(gpu, ngpus_per_node): if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: print("Skipping finetuning!!!") - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) if not parser_args.skip_sanity_checks: do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, diff --git a/main_utils.py b/main_utils.py index 694e101e..55a93499 100644 --- a/main_utils.py +++ b/main_utils.py @@ -95,9 +95,10 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b # do the sanity check for shuffled mask/weights, reinit weights print("Sanity Check 1: Weight Reinit") - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, @@ -111,9 +112,10 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b ''' print("Sanity Check 2: Mask Reshuffle") - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, @@ -279,9 +281,10 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b # switch to weight training mode (turn on the requires_grad for weight/bias, and turn off the requires_grad for other parameters) model = switch_to_wt(model) - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + if parser_args.multiprocessing_distributed: + print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + dist.barrier() + print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) # not to use score regulaization during the weight training parser_args.regularization = False diff --git a/trainers/default.py b/trainers/default.py index 6597353f..8b11a9e9 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -1,11 +1,11 @@ import time import torch -import tqdm +# import tqdm import copy import pdb from utils.eval_utils import accuracy -from utils.logging import AverageMeter, ProgressMeter +# from utils.logging import AverageMeter, ProgressMeter from utils.net_utils import get_regularization_loss, prune, get_layers from torch import optim @@ -15,17 +15,17 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler=None): - batch_time = AverageMeter("Time", ":6.3f") - data_time = AverageMeter("Data", ":6.3f") - losses = AverageMeter("Loss", ":.3f") - top1 = AverageMeter("Acc@1", ":6.2f") - top5 = AverageMeter("Acc@5", ":6.2f") - top10 = AverageMeter("Acc@10", ":6.2f") - progress = ProgressMeter( - len(train_loader), - [batch_time, data_time, losses, top1, top5], - prefix=f"Epoch: [{epoch}]", - ) + # batch_time = AverageMeter("Time", ":6.3f") + # data_time = AverageMeter("Data", ":6.3f") + # losses = AverageMeter("Loss", ":.3f") + # top1 = AverageMeter("Acc@1", ":6.2f") + # top5 = AverageMeter("Acc@5", ":6.2f") + # top10 = AverageMeter("Acc@10", ":6.2f") + # progress = ProgressMeter( + # len(train_loader), + # [batch_time, data_time, losses, top1, top5], + # prefix=f"GPU:[{args.gpu}] | Epoch: [{epoch}]", + # ) # switch to train mode model.train() @@ -120,14 +120,14 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler def validate(val_loader, model, criterion, args, writer, epoch): - batch_time = AverageMeter("Time", ":6.3f", write_val=False) - losses = AverageMeter("Loss", ":.3f", write_val=False) - top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) - top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) - top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) - progress = ProgressMeter( - len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " - ) + # batch_time = AverageMeter("Time", ":6.3f", write_val=False) + # losses = AverageMeter("Loss", ":.3f", write_val=False) + # top1 = AverageMeter("Acc@1", ":6.2f", write_val=False) + # top5 = AverageMeter("Acc@5", ":6.2f", write_val=False) + # top10 = AverageMeter("Acc@10", ":6.2f", write_val=False) + # progress = ProgressMeter( + # len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " + # ) # switch to evaluate mode model.eval() From f4fbff04a39f6042224594cf343eada119d5020c Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:26:40 -0500 Subject: [PATCH 60/72] trying to eliminate tqdm and progress meters I think it's just slowing things down and is known to cause memory leaks --- main_utils.py | 36 ++++++++++-------- trainers/default.py | 91 ++++++++++++++++++++++++++++----------------- utils/logging.py | 2 +- 3 files changed, 78 insertions(+), 51 deletions(-) diff --git a/main_utils.py b/main_utils.py index 55a93499..7180c3c7 100644 --- a/main_utils.py +++ b/main_utils.py @@ -25,7 +25,7 @@ import re from utils.conv_type import FixedSubnetConv, SampleSubnetConv -from utils.logging import AverageMeter, ProgressMeter +# from utils.logging import AverageMeter, ProgressMeter from utils.net_utils import ( set_model_prune_rate, freeze_model_weights, @@ -349,7 +349,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b train_acc1, train_acc5, train_acc10, reg_loss = train( data.train_loader, model, criterion, optimizer, epoch, parser_args, writer=writer ) - train_time.update((time.time() - start_train) / 60) + # train_time.update((time.time() - start_train) / 60) + train_time = (time.time() - start_train) / 60 # evaluate on validation set if (parser_args.multiprocessing_distributed and parser_args.gpu == 0) or not parser_args.multiprocessing_distributed: @@ -358,7 +359,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b data.val_loader, model, criterion, parser_args, writer, epoch) val_acc1, val_acc5, val_acc10 = validate( data.actual_val_loader, model, criterion, parser_args, writer, epoch) - validation_time.update((time.time() - start_validation) / 60) + # validation_time.update((time.time() - start_validation) / 60) + validation_time = (time.time() - start_validation) / 60 # copy & paste the sparsity of prev. epoch avg_sparsity = model_sparsity_list[-1] @@ -371,12 +373,12 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b reg_loss_list.append(reg_loss) model_sparsity_list.append(avg_sparsity) - epoch_time.update((time.time() - end_epoch) / 60) - progress_overall.display(epoch) - progress_overall.write_to_tensorboard( - writer, prefix="diagnostics", global_step=epoch - ) - writer.add_scalar("test/lr", cur_lr, epoch) + # epoch_time.update((time.time() - end_epoch) / 60) + # progress_overall.display(epoch) + # progress_overall.write_to_tensorboard( + # writer, prefix="diagnostics", global_step=epoch + # ) + # writer.add_scalar("test/lr", cur_lr, epoch) end_epoch = time.time() results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, @@ -439,12 +441,16 @@ def get_settings(parser_args): parser_args.ckpt_base_dir = ckpt_base_dir writer = SummaryWriter(log_dir=log_base_dir) # writer = None - epoch_time = AverageMeter("epoch_time", ":.4f", write_avg=False) - validation_time = AverageMeter("validation_time", ":.4f", write_avg=False) - train_time = AverageMeter("train_time", ":.4f", write_avg=False) - progress_overall = ProgressMeter( - 1, [epoch_time, validation_time, train_time], prefix="Overall Timing" - ) + # epoch_time = AverageMeter("epoch_time", ":.4f", write_avg=False) + # validation_time = AverageMeter("validation_time", ":.4f", write_avg=False) + # train_time = AverageMeter("train_time", ":.4f", write_avg=False) + # progress_overall = ProgressMeter( + # 1, [epoch_time, validation_time, train_time], prefix="Overall Timing" + # ) + epoch_time = 0 + validation_time = 0 + train_time = 0 + progress_overall = None return run_base_dir, ckpt_base_dir, log_base_dir, writer, epoch_time, validation_time, train_time, progress_overall diff --git a/trainers/default.py b/trainers/default.py index 8b11a9e9..4df27010 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -26,6 +26,10 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # [batch_time, data_time, losses, top1, top5], # prefix=f"GPU:[{args.gpu}] | Epoch: [{epoch}]", # ) + top1 = 0 + top5 = 0 + top10 = 0 + num_images = 0 # switch to train mode model.train() @@ -33,16 +37,16 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler batch_size = train_loader.batch_size num_batches = len(train_loader) end = time.time() - for i, (images, target) in tqdm.tqdm( - enumerate(train_loader), ascii=True, total=len(train_loader) - ): + # for i, (images, target) in tqdm.tqdm( + # enumerate(train_loader), ascii=True, total=len(train_loader) + # ): + for i, (images, target) in enumerate(train_loader): # measure data loading time - data_time.update(time.time() - end) - #print(images.shape, target.shape) - - if args.gpu is not None: - images = images.cuda(args.gpu, non_blocking=True) + data_time = time.time() - end + # print("Data Time: {}".format(data_time)) + # print(images.shape, target.shape) + images = images.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) # update score thresholds for global ep @@ -80,10 +84,15 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # measure accuracy and record loss acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - losses.update(loss.item(), images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - top10.update(acc10.item(), images.size(0)) + # losses.update(loss.item(), images.size(0)) + # top1.update(acc1.item(), images.size(0)) + # top5.update(acc5.item(), images.size(0)) + # top10.update(acc10.item(), images.size(0)) + # compute weighted sum for each accuracy so we can average it later + top1 += acc1.item() * images.size(0) + top5 += acc5.item() * images.size(0) + top10 += acc10.item() * images.size(0) + num_images += images.size(0) # compute gradient and do SGD step optimizer.zero_grad() @@ -96,14 +105,17 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler scaler.update() # measure elapsed time - batch_time.update(time.time() - end) + # batch_time.update(time.time() - end) + batch_time = time.time() - end end = time.time() if i % args.print_freq == 0: t = (num_batches * epoch + i) * batch_size - progress.display(i) - progress.write_to_tensorboard( - writer, prefix="train", global_step=t) + # progress.display(i) + # progress.write_to_tensorboard( + # writer, prefix="train", global_step=t) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item()), batch_time) + # before completing training, clean up model based on latest scores # update score thresholds for global ep @@ -116,7 +128,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) - return top1.avg, top5.avg, top10.avg, regularization_loss.item() + return top1/num_images, top5/num_images, top10/num_images, regularization_loss.item() def validate(val_loader, model, criterion, args, writer, epoch): @@ -128,18 +140,20 @@ def validate(val_loader, model, criterion, args, writer, epoch): # progress = ProgressMeter( # len(val_loader), [batch_time, losses, top1, top5, top10], prefix="Test: " # ) + top1 = 0 + top5 = 0 + top10 = 0 # switch to evaluate mode model.eval() with torch.no_grad(): end = time.time() - for i, (images, target) in tqdm.tqdm( - enumerate(val_loader), ascii=True, total=len(val_loader) - ): - if args.gpu is not None: - images = images.cuda(args.gpu, non_blocking=True) - + # for i, (images, target) in tqdm.tqdm( + # enumerate(val_loader), ascii=True, total=len(val_loader) + # ): + for i, (images, target) in enumerate(val_loader): + images = images.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) #print(images.shape, target.shape) @@ -151,26 +165,33 @@ def validate(val_loader, model, criterion, args, writer, epoch): # measure accuracy and record loss acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) - losses.update(loss.item(), images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - top10.update(acc10.item(), images.size(0)) + # losses.update(loss.item(), images.size(0)) + # top1.update(acc1.item(), images.size(0)) + # top5.update(acc5.item(), images.size(0)) + # top10.update(acc10.item(), images.size(0)) + # compute weighted sum for each accuracy so we can average it later + top1 += acc1.item() * images.size(0) + top5 += acc5.item() * images.size(0) + top10 += acc10.item() * images.size(0) + num_images += images.size(0) # measure elapsed time - batch_time.update(time.time() - end) + # batch_time.update(time.time() - end) + batch_time = time.time() - end end = time.time() if i % args.print_freq == 0: - progress.display(i) + # progress.display(i) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item()), batch_time) - progress.display(len(val_loader)) + # progress.display(len(val_loader)) - if writer is not None: - progress.write_to_tensorboard( - writer, prefix="test", global_step=epoch) + # if writer is not None: + # progress.write_to_tensorboard( + # writer, prefix="test", global_step=epoch) - print("Model top1 Accuracy: {}".format(top1.avg)) - return top1.avg, top5.avg, top10.avg + print("Model top1 Accuracy: {}".format(top1/num_images)) + return top1/num_images, top5/num_images, top10/num_images def modifier(args, epoch, model): diff --git a/utils/logging.py b/utils/logging.py index 8cce423a..7f809ba8 100644 --- a/utils/logging.py +++ b/utils/logging.py @@ -1,5 +1,5 @@ import abc -import tqdm +# import tqdm # from torch.utils.tensorboard import SummaryWriter From 69048e0bac37eb3ac6335f9882eff3ef13b50025 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:30:56 -0500 Subject: [PATCH 61/72] trainer doesn't see parser_args --- main_utils.py | 2 ++ trainers/default.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/main_utils.py b/main_utils.py index 7180c3c7..991b8178 100644 --- a/main_utils.py +++ b/main_utils.py @@ -379,6 +379,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b # writer, prefix="diagnostics", global_step=epoch # ) # writer.add_scalar("test/lr", cur_lr, epoch) + epoch_time = (time.time() - end_epoch) / 60 + print("GPU:{} | Epoch: {} | Acc={} | Epoch Time={}".format(parser_args.gpu, epoch, acc1, epoch_time)) end_epoch = time.time() results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, 'val_acc': val_acc_list, 'train_acc': train_acc_list, diff --git a/trainers/default.py b/trainers/default.py index 4df27010..0cd8e5eb 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -114,7 +114,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # progress.display(i) # progress.write_to_tensorboard( # writer, prefix="train", global_step=t) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item()), batch_time) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item()), batch_time) # before completing training, clean up model based on latest scores @@ -182,7 +182,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): if i % args.print_freq == 0: # progress.display(i) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(parser_args.gpu, epoch, loss.item(), acc1.item()), batch_time) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item()), batch_time) # progress.display(len(val_loader)) From 3834b3d70018600406a9a2775bb957c8f65cd697 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:46:35 -0500 Subject: [PATCH 62/72] still removing leftover calls to tqdm and progresmeters --- main.py | 6 ++++-- trainers/default.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 95bdda4b..0cf1f349 100644 --- a/main.py +++ b/main.py @@ -164,7 +164,8 @@ def main_worker(gpu, ngpus_per_node): train_acc1, train_acc5, train_acc10, reg_loss = train( data.train_loader, model, criterion, optimizer, epoch, parser_args, writer=writer, scaler=scaler ) - train_time.update((time.time() - start_train) / 60) + # train_time.update((time.time() - start_train) / 60) + train_time = (time.time() - start_train) / 60 scheduler.step() # evaluate on validation set @@ -191,7 +192,8 @@ def main_worker(gpu, ngpus_per_node): acc1, acc5, acc10 = validate( data.val_loader, model, criterion, parser_args, writer, epoch) print('Acc: {}'.format(acc1)) - validation_time.update((time.time() - start_validation) / 60) + # validation_time.update((time.time() - start_validation) / 60) + validation_time = (time.time() - start_validation) / 60 # prune the model every T_{prune} epochs if not parser_args.weight_training and parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0: diff --git a/trainers/default.py b/trainers/default.py index 0cd8e5eb..8b1754c9 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -114,7 +114,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # progress.display(i) # progress.write_to_tensorboard( # writer, prefix="train", global_step=t) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item()), batch_time) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) # before completing training, clean up model based on latest scores @@ -143,6 +143,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): top1 = 0 top5 = 0 top10 = 0 + num_images = 0 # switch to evaluate mode model.eval() @@ -182,7 +183,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): if i % args.print_freq == 0: # progress.display(i) - print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item()), batch_time) + print("GPU:{} | Epoch: {} | loss={} | Batch Time={}".format(args.gpu, epoch, loss.item(), acc1.item(), batch_time)) # progress.display(len(val_loader)) From 3ba1f9344e032ec97df4242f105b16838a25e54f Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 15:49:20 -0500 Subject: [PATCH 63/72] removing more instances of progressmeter and replacing them with vanilla prints --- main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 0cf1f349..b92e5efa 100644 --- a/main.py +++ b/main.py @@ -166,6 +166,7 @@ def main_worker(gpu, ngpus_per_node): ) # train_time.update((time.time() - start_train) / 60) train_time = (time.time() - start_train) / 60 + scheduler.step() # evaluate on validation set @@ -249,13 +250,15 @@ def main_worker(gpu, ngpus_per_node): reg_loss_list.append(reg_loss) model_sparsity_list.append(avg_sparsity) - epoch_time.update((time.time() - end_epoch) / 60) - progress_overall.display(epoch) - progress_overall.write_to_tensorboard( - writer, prefix="diagnostics", global_step=epoch - ) + # epoch_time.update((time.time() - end_epoch) / 60) + epoch_time = (time.time() - end_epoch) / 60 + # progress_overall.display(epoch) + # progress_overall.write_to_tensorboard( + # writer, prefix="diagnostics", global_step=epoch + # ) + print("GPU:{} | Epoch: {} | Acc={} | Epoch Time={}".format(parser_args.gpu, epoch, acc1, epoch_time)) - writer.add_scalar("test/lr", cur_lr, epoch) + # writer.add_scalar("test/lr", cur_lr, epoch) end_epoch = time.time() if parser_args.algo in ['hc', 'hc_iter']: From 77fbe481cfd12f42b596a29d17bb0f8689962b06 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 17:24:12 -0500 Subject: [PATCH 64/72] fixing dir creation bug --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b92e5efa..5116f2f5 100644 --- a/main.py +++ b/main.py @@ -46,13 +46,14 @@ def main_worker(gpu, ngpus_per_node): result_root = result_subroot + '/results_' + idty_str + '/' else: result_root = 'results/results_' + idty_str + '/' - else: - idty_str = get_idty_str(parser_args) - result_root = 'results/results_' + idty_str + '/' if not os.path.isdir(result_root): os.mkdir(result_root) print_model(model, parser_args) + else: + idty_str = get_idty_str(parser_args) + result_root = 'results/results_' + idty_str + '/' + if parser_args.weight_training: model = round_model(model, round_scheme="all_ones", noise=parser_args.noise, From 1ffc0000ad8fce0eb178f708f154aa1b5c189c50 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 28 Apr 2022 17:25:47 -0500 Subject: [PATCH 65/72] adding config for serial debug --- configs/ddp_debug/conf2.yml | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 configs/ddp_debug/conf2.yml diff --git a/configs/ddp_debug/conf2.yml b/configs/ddp_debug/conf2.yml new file mode 100644 index 00000000..3df27c06 --- /dev/null +++ b/configs/ddp_debug/conf2.yml @@ -0,0 +1,62 @@ +# subfolder: target_sparsity_0_59_unflagT_real + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 20 +wd: 0.0 +momentum: 0.9 +batch_size: 512 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 + +# ===== Hardware setup ===== # +workers: 8 +# gpu: 3 +mixed_precision: True + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From 53b90c0d5541c76c1b97022030bd6953a996401e Mon Sep 17 00:00:00 2001 From: root Date: Thu, 28 Apr 2022 21:15:38 -0500 Subject: [PATCH 66/72] stuff works, removing an unnecessary barrier --- configs/ddp_debug/conf2.yml | 2 +- data/cifar.py | 2 +- main_utils.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/ddp_debug/conf2.yml b/configs/ddp_debug/conf2.yml index 3df27c06..6ac12696 100644 --- a/configs/ddp_debug/conf2.yml +++ b/configs/ddp_debug/conf2.yml @@ -52,7 +52,7 @@ lmbda: 0.0001 # 1e-4 # ===== Hardware setup ===== # workers: 8 -# gpu: 3 +gpu: 0 mixed_precision: True # ===== Checkpointing ===== # diff --git a/data/cifar.py b/data/cifar.py index a782b917..2ee881ed 100644 --- a/data/cifar.py +++ b/data/cifar.py @@ -17,7 +17,7 @@ def __init__(self, args): use_cuda = torch.cuda.is_available() # Data loading code - kwargs = {"num_workers": parser_args.workers, "pin_memory": True} if use_cuda else {} + kwargs = {"num_workers": parser_args.num_workers, "pin_memory": True} if use_cuda else {} normalize = transforms.Normalize( mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262] diff --git a/main_utils.py b/main_utils.py index 991b8178..1439182e 100644 --- a/main_utils.py +++ b/main_utils.py @@ -95,10 +95,10 @@ def do_sanity_checks(model, parser_args, data, criterion, epoch_list, test_acc_b # do the sanity check for shuffled mask/weights, reinit weights print("Sanity Check 1: Weight Reinit") - if parser_args.multiprocessing_distributed: - print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) - dist.barrier() - print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + # if parser_args.multiprocessing_distributed: + # print("TORCH BARRIER: GPU:{}".format(parser_args.gpu)) + # dist.barrier() + # print("CLEARED TORCH BARRIER: GPU:{}".format(parser_args.gpu)) cp_model = copy.deepcopy(model) cp_model = finetune(cp_model, parser_args, data, criterion, epoch_list, test_acc_before_round_list, test_acc_list, val_acc_list, train_acc_list, From 10b1ee4424ef9e711e73ed9e0fe3f7d1bb49cf22 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 29 Apr 2022 16:59:55 +0000 Subject: [PATCH 67/72] adding imagenet configs and exec script --- .../resnet50/imagenet/resnet50_sparsity_5.yml | 68 +++++++++++++++++++ .../resnet50_sc_hypercube_reg_exp1.yml | 0 .../resnet50_sc_hypercube_reg_exp2.yml | 0 .../resnet50_sc_hypercube_reg_exp3.yml | 0 imagenet_exec.sh | 29 ++++++++ main_utils.py | 4 +- 6 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml rename configs/hypercube/resnet50/{ => old_configs}/resnet50_sc_hypercube_reg_exp1.yml (100%) rename configs/hypercube/resnet50/{ => old_configs}/resnet50_sc_hypercube_reg_exp2.yml (100%) rename configs/hypercube/resnet50/{ => old_configs}/resnet50_sc_hypercube_reg_exp3.yml (100%) create mode 100644 imagenet_exec.sh diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml new file mode 100644 index 00000000..4eb571f9 --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml @@ -0,0 +1,68 @@ +# subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/ILSVRC2012/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.004 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-6 + +# ===== Hardware setup ===== # +workers: 12 +multiprocessing_distributed: True +mixed_precision: True +# gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp1.yml b/configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp1.yml similarity index 100% rename from configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp1.yml rename to configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp1.yml diff --git a/configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp2.yml b/configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp2.yml similarity index 100% rename from configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp2.yml rename to configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp2.yml diff --git a/configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp3.yml b/configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp3.yml similarity index 100% rename from configs/hypercube/resnet50/resnet50_sc_hypercube_reg_exp3.yml rename to configs/hypercube/resnet50/old_configs/resnet50_sc_hypercube_reg_exp3.yml diff --git a/imagenet_exec.sh b/imagenet_exec.sh new file mode 100644 index 00000000..3e992ff9 --- /dev/null +++ b/imagenet_exec.sh @@ -0,0 +1,29 @@ +# Running trials in parallel +# NOTE: make sure to delete/comment subfolder from the config file or else it may not work +:< "$log_root$trial$log_end" 2>&1 & + + python main.py \ + --config "$conf_file" \ + --trial-num $trial \ + --invert-sanity-check \ + --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +done + +BLOCK + +conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" +log_root="resnet50_sp5" +log_end="_log" +python main.py \ + --config "$conf_file" > "$log_root$log_end" 2>&1 & diff --git a/main_utils.py b/main_utils.py index 1439182e..1af1101b 100644 --- a/main_utils.py +++ b/main_utils.py @@ -11,7 +11,7 @@ import random import time import pandas as pd -from torch.utils.tensorboard import SummaryWriter +# from torch.utils.tensorboard import SummaryWriter import torch import torch.nn as nn import torch.nn.parallel @@ -441,7 +441,7 @@ def get_settings(parser_args): run_base_dir, ckpt_base_dir, log_base_dir = get_directories(parser_args) parser_args.ckpt_base_dir = ckpt_base_dir - writer = SummaryWriter(log_dir=log_base_dir) + writer = None # SummaryWriter(log_dir=log_base_dir) # writer = None # epoch_time = AverageMeter("epoch_time", ":.4f", write_avg=False) # validation_time = AverageMeter("validation_time", ":.4f", write_avg=False) From 38550af457cb9e5f5ec4ed33b2cc978e82a2d3bf Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 3 May 2022 02:48:51 +0000 Subject: [PATCH 68/72] setting find_unused_parameters=False and adding config for sp10 --- .../imagenet/resnet50_sparsity_10.yml | 68 +++++++++++++++++++ main_utils.py | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml new file mode 100644 index 00000000..8a5bf08e --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml @@ -0,0 +1,68 @@ +# subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/ILSVRC2012/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.04 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 10 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000001 # 1e-7 + +# ===== Hardware setup ===== # +workers: 10 +multiprocessing_distributed: True +mixed_precision: True +# gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/main_utils.py b/main_utils.py index 1af1101b..11242bda 100644 --- a/main_utils.py +++ b/main_utils.py @@ -920,7 +920,7 @@ def set_gpu(parser_args, model): if parser_args.multiprocessing_distributed: # TODO: not sure about find_unused_parameters. Need to check model = nn.parallel.DistributedDataParallel( - model, device_ids=[parser_args.gpu], find_unused_parameters=True) + model, device_ids=[parser_args.gpu], find_unused_parameters=False) return model From 50805e31aa22fe5ed89cc32fb552bf49fb6e9837 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 4 May 2022 20:30:52 +0000 Subject: [PATCH 69/72] pushing config for sp15 imagenet --- cifar_exec.sh | 8 +-- .../imagenet/resnet50_sparsity_15.yml | 68 +++++++++++++++++++ .../resnet50/imagenet/resnet50_sparsity_5.yml | 2 +- main_utils.py | 2 +- 4 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml diff --git a/cifar_exec.sh b/cifar_exec.sh index 6ba68efa..8986d2f8 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -64,15 +64,15 @@ BLOCK #:< "$log_root$trial$log_end" 2>&1 & diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml new file mode 100644 index 00000000..c36fd337 --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml @@ -0,0 +1,68 @@ +# subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/ILSVRC2012/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.04 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 10 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00000005 # 5e-8 + +# ===== Hardware setup ===== # +workers: 12 +multiprocessing_distributed: True +mixed_precision: True +# gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml index 4eb571f9..a7cf06bd 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml @@ -19,7 +19,7 @@ data: /home/ubuntu/ILSVRC2012/ optimizer: sgd lr: 0.4 #0.01 lr_policy: cosine_lr #constant_lr #multistep_lr -fine_tune_lr: 0.004 +fine_tune_lr: 0.04 fine_tune_lr_policy: multistep_lr # ===== Network training config ===== # diff --git a/main_utils.py b/main_utils.py index 11242bda..64f7b3dc 100644 --- a/main_utils.py +++ b/main_utils.py @@ -537,7 +537,7 @@ def get_mask(model): def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' - os.environ['MASTER_PORT'] = '29500' + os.environ['MASTER_PORT'] = '29600' dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) From fb1eab0e32a5f31a5a70b0d867810af5dff8b176 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 4 May 2022 21:33:08 +0000 Subject: [PATCH 70/72] minor tweaks to configs for imagenet --- .../imagenet/resnet50_sparsity_10.yml | 2 +- .../imagenet/resnet50_sparsity_15.yml | 2 +- .../imagenet/resnet50_sparsity_25.yml | 68 +++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_25.yml diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml index 8a5bf08e..6d6ea32a 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml @@ -56,7 +56,7 @@ regularization: L2 lmbda: 0.0000001 # 1e-7 # ===== Hardware setup ===== # -workers: 10 +workers: 12 multiprocessing_distributed: True mixed_precision: True # gpu: 1 diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml index c36fd337..5a9b6a0a 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_15.yml @@ -35,7 +35,7 @@ bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK # enter target sparsity here -target_sparsity: 10 +target_sparsity: 15 # decide if you want to "unflag" unflag_before_finetune: False init: signed_constant diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_25.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_25.yml new file mode 100644 index 00000000..4adf0655 --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_25.yml @@ -0,0 +1,68 @@ +# subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/ILSVRC2012/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.04 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 25 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00000001 # 1e-8 + +# ===== Hardware setup ===== # +workers: 12 +multiprocessing_distributed: True +mixed_precision: True +# gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From 5ccc7a32c3c7467a2bf2168273373b8022ad6ee9 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 4 May 2022 16:45:49 -0500 Subject: [PATCH 71/72] adding port as param for multiple runs --- args_helper.py | 58 ++++++++++++++++++++++++++++++++++---------------- main_utils.py | 2 +- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/args_helper.py b/args_helper.py index 94370efa..c73d83e2 100644 --- a/args_helper.py +++ b/args_helper.py @@ -878,35 +878,52 @@ def parse_arguments(self, jupyter_mode=False): default=0, help="Use mixed precision or not" ) - parser.add_argument('--transformer_emsize', type=int, default=200, - help='size of word embeddings') - parser.add_argument('--transformer_nhid', type=int, default=200, - help='number of hidden units per layer') - parser.add_argument('--transformer_nlayers', type=int, default=2, - help='number of layers') - parser.add_argument('--transformer_clip', type=float, default=0.25, - help='gradient clipping') - parser.add_argument('--transformer_bptt', type=int, default=35, - help='sequence length') - parser.add_argument('--transformer_dropout', type=float, default=0.2, - help='dropout applied to layers (0 = no dropout)') - parser.add_argument('--transformer_nhead', type=int, default=2, - help='the number of heads in the encoder/decoder of the transformer model') - + parser.add_argument('--transformer_emsize', + type=int, default=200, + help='size of word embeddings' + ) + parser.add_argument('--transformer_nhid', + type=int, + default=200, + help='number of hidden units per layer' + ) + parser.add_argument('--transformer_nlayers', + type=int, + default=2, + help='number of layers' + ) + parser.add_argument('--transformer_clip', + type=float, + default=0.25, + help='gradient clipping' + ) + parser.add_argument('--transformer_bptt', + type=int, + default=35, + help='sequence length' + ) + parser.add_argument('--transformer_dropout', + type=float, + default=0.2, + help='dropout applied to layers (0 = no dropout)' + ) + parser.add_argument('--transformer_nhead', + type=int, + default=2, + help='the number of heads in the encoder/decoder of the transformer model' + ) parser.add_argument( "--only-sanity", action="store_true", default=False, help="Only run sanity checks on the files in specific directory or subdirectories" ) - parser.add_argument( "--invert-sanity-check", action="store_true", default=False, help="Enable this to run the inverted sanity check (for HC)" ) - parser.add_argument( "--sanity-folder", default=None, @@ -914,7 +931,6 @@ def parse_arguments(self, jupyter_mode=False): metavar="PATH", help="directory(s) to access for only sanity check", ) - parser.add_argument( "--sr-version", default=1, @@ -927,6 +943,12 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="Enable this use full train data and not leave anything for validation" ) + parser.add_argument( + "--port", + default=29500, + type=int, + help="Specify port to use for DDP", + ) if jupyter_mode: args = parser.parse_args("") diff --git a/main_utils.py b/main_utils.py index 64f7b3dc..a95ef5da 100644 --- a/main_utils.py +++ b/main_utils.py @@ -537,7 +537,7 @@ def get_mask(model): def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' - os.environ['MASTER_PORT'] = '29600' + os.environ['MASTER_PORT'] = parser_args.port dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node) From 56e32cbb62004f668adef92aac2bebf4c20225f6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 6 May 2022 19:23:22 +0000 Subject: [PATCH 72/72] updating port to str not int --- imagenet_exec.sh | 1 + main_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index 3e992ff9..7fd3bf33 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -24,6 +24,7 @@ BLOCK conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" log_root="resnet50_sp5" +# conf_file="configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml" log_end="_log" python main.py \ --config "$conf_file" > "$log_root$log_end" 2>&1 & diff --git a/main_utils.py b/main_utils.py index 79a14d37..bc31193b 100644 --- a/main_utils.py +++ b/main_utils.py @@ -537,7 +537,7 @@ def get_mask(model): def setup_distributed(rank, ngpus_per_node): os.environ['MASTER_ADDR'] = '127.0.0.1' - os.environ['MASTER_PORT'] = parser_args.port + os.environ['MASTER_PORT'] = '{}'.format(parser_args.port) dist.init_process_group("nccl", rank=rank, world_size=ngpus_per_node)