From 8d13a7eb3f167e2341f4e3e5a367f065d7c05373 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 13 Dec 2021 04:33:29 +0000 Subject: [PATCH 1/8] random rounding --- args_helper.py | 6 +++++ cifar_exec_GD.sh | 20 +++++++++++----- config_rr1.yml | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 config_rr1.yml diff --git a/args_helper.py b/args_helper.py index a1d0aab7..e21e685e 100644 --- a/args_helper.py +++ b/args_helper.py @@ -837,6 +837,12 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="Enable this to unprune weights if possible, before fine-tune" ) + parser.add_argument( + "--random_round", + action="store_true", + default=False, + help="randomized rounding for hc" + ) if jupyter_mode: args = parser.parse_args("") diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index f0e72aaa..410a10b9 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -13,18 +13,26 @@ #python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml -## testing adding finetune loss -python main.py --config config1.yml --run_idx 1 # -#python main.py --config config2.yml --run_idx 2 - - - +# Add finetune-loss +#python main.py --config config1.yml --run_idx 1 # +#python main.py --config config2.yml --run_idx 2 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1.yml --run_idx 1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1e-2.yml --run_idx 1 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1e-4.yml --run_idx 2 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_0.yml --run_idx 3 + +# Random rounding +python main.py --config config_rr1.yml --run_idx 1 + + + + + + + + #python main.py --config config10.yml --run_idx 10 #> log_config$r 2>&1 :< Date: Sun, 12 Dec 2021 23:17:16 -0600 Subject: [PATCH 2/8] edit forward function --- args_helper.py | 6 +++++ cifar_exec_GD.sh | 1 + config_rr1.yml | 4 ++- config_rr2.yml | 62 ++++++++++++++++++++++++++++++++++++++++++++++ utils/conv_type.py | 24 ++++++++++++++++-- 5 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 config_rr2.yml diff --git a/args_helper.py b/args_helper.py index e21e685e..7d4f0933 100644 --- a/args_helper.py +++ b/args_helper.py @@ -843,6 +843,12 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="randomized rounding for hc" ) + parser.add_argument( + "--random_round_type", + type=str, + default='one_flip', + help="Type of random rounding : one_flip | majority | best " + ) if jupyter_mode: args = parser.parse_args("") diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 410a10b9..a1082a83 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -25,6 +25,7 @@ # Random rounding python main.py --config config_rr1.yml --run_idx 1 +#python main.py --config config_rr2.yml --run_idx 2 diff --git a/config_rr1.yml b/config_rr1.yml index 3a9f725c..b0ebdacc 100644 --- a/config_rr1.yml +++ b/config_rr1.yml @@ -1,4 +1,6 @@ -subfolder: random_rounding +subfolder: random_rounding_one_flip +random_rounding: True +random_round_type: one_flip # Hypercube optimization algo: 'hc_iter' diff --git a/config_rr2.yml b/config_rr2.yml new file mode 100644 index 00000000..93bb8b27 --- /dev/null +++ b/config_rr2.yml @@ -0,0 +1,62 @@ +subfolder: random_rounding_majority +random_rounding: True +random_round_type: majority + +# Hypercube optimization +algo: 'hc_iter' +# effectively prune once in the entire process +iter_period: 100 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +prune_rate: 0.982 +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 0.01 #0.0001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/utils/conv_type.py b/utils/conv_type.py index d58b67fc..71f46100 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -78,8 +78,28 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc out = torch.gt(scores, torch.ones_like(scores)*scores_prune_threshold).float() bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float() else: - out = torch.gt(scores, torch.ones_like(scores)*parser_args.quantize_threshold).float() - bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*parser_args.quantize_threshold).float() + if parser_args.random_round: + if parser_args.random_round_type == 'one_flip': + out = torch.bernoulli(torch.clamp(scores, 0, 1)) + bias_out = torch.bernoulli(torch.clamp(bias_scores, 0, 1)) + elif parser_args.random_round_type == 'majority': + # flip 5 coins and take dimension-wise majority voting + out = torch.zeros_like(scores) + bias_out = torch.zeros_like(bias_scores) + for flip_iter in range(5): + out += torch.bernoulli(torch.clamp(scores, 0, 1)) + bias_out += torch.bernoulli(torch.clamp(scores, 0, 1)) + out = torch.gt(out, torch.ones_like(out)*2).float() # among 5 trials, we need at least 3 heads + bias_out = torch.gt(bias_out, torch.ones_like(bias_out)*2).float() # among 5 trials, we need at least 3 heads + + raise NotImplementedError + elif parser_args.random_round_type == 'best': + # compute loss of 5 coin flips and take the best one (how to compute loss?) + raise NotImplementedError + + else: + out = torch.gt(scores, torch.ones_like(scores)*parser_args.quantize_threshold).float() + bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*parser_args.quantize_threshold).float() else: print("INVALID PRUNING ALGO") From 4cabe64607dca4168167c592ad2ebf4bbbf5ecf4 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 13 Dec 2021 05:25:08 +0000 Subject: [PATCH 3/8] minor --- trainers/default.py | 2 +- utils/conv_type.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index 50ada91a..c0848ff9 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -162,7 +162,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): loss.backward() optimizer.step() - pdb.set_trace() + #pdb.set_trace() # TODO: print the updated score # TODO: print the weight diff --git a/utils/conv_type.py b/utils/conv_type.py index 71f46100..dca30737 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -79,6 +79,7 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float() else: if parser_args.random_round: + pdb.set_trace() if parser_args.random_round_type == 'one_flip': out = torch.bernoulli(torch.clamp(scores, 0, 1)) bias_out = torch.bernoulli(torch.clamp(bias_scores, 0, 1)) @@ -90,9 +91,7 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc out += torch.bernoulli(torch.clamp(scores, 0, 1)) bias_out += torch.bernoulli(torch.clamp(scores, 0, 1)) out = torch.gt(out, torch.ones_like(out)*2).float() # among 5 trials, we need at least 3 heads - bias_out = torch.gt(bias_out, torch.ones_like(bias_out)*2).float() # among 5 trials, we need at least 3 heads - - raise NotImplementedError + bias_out = torch.gt(bias_out, torch.ones_like(bias_out)*2).float() # among 5 trials, we need at least 3 heads elif parser_args.random_round_type == 'best': # compute loss of 5 coin flips and take the best one (how to compute loss?) raise NotImplementedError From 39254b6c350413b7b2f5813fd2e9a5d2fbd40933 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 13 Dec 2021 05:45:01 +0000 Subject: [PATCH 4/8] run random rounding for one_flip, majority --- cifar_exec_GD.sh | 2 +- utils/conv_type.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index a1082a83..be25a23b 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -12,7 +12,7 @@ ### ResNet-20 #python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml - +#python main.py --config config_current_best.yml --run_idx 0 # Add finetune-loss #python main.py --config config1.yml --run_idx 1 # diff --git a/utils/conv_type.py b/utils/conv_type.py index dca30737..ff2d82bc 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -79,7 +79,6 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float() else: if parser_args.random_round: - pdb.set_trace() if parser_args.random_round_type == 'one_flip': out = torch.bernoulli(torch.clamp(scores, 0, 1)) bias_out = torch.bernoulli(torch.clamp(bias_scores, 0, 1)) @@ -89,7 +88,7 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc bias_out = torch.zeros_like(bias_scores) for flip_iter in range(5): out += torch.bernoulli(torch.clamp(scores, 0, 1)) - bias_out += torch.bernoulli(torch.clamp(scores, 0, 1)) + bias_out += torch.bernoulli(torch.clamp(bias_scores, 0, 1)) out = torch.gt(out, torch.ones_like(out)*2).float() # among 5 trials, we need at least 3 heads bias_out = torch.gt(bias_out, torch.ones_like(bias_out)*2).float() # among 5 trials, we need at least 3 heads elif parser_args.random_round_type == 'best': From c921c06590a98c7a963eb324962eee214fca7fde Mon Sep 17 00:00:00 2001 From: root Date: Mon, 13 Dec 2021 17:17:14 +0000 Subject: [PATCH 5/8] random rounding running exec --- cifar_exec_GD.sh | 2 +- config_rr1.yml | 2 +- config_rr2.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index be25a23b..3dcad087 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -24,7 +24,7 @@ # Random rounding -python main.py --config config_rr1.yml --run_idx 1 +#python main.py --config config_rr1.yml --run_idx 1 #python main.py --config config_rr2.yml --run_idx 2 diff --git a/config_rr1.yml b/config_rr1.yml index b0ebdacc..ca466927 100644 --- a/config_rr1.yml +++ b/config_rr1.yml @@ -1,5 +1,5 @@ subfolder: random_rounding_one_flip -random_rounding: True +random_round: True random_round_type: one_flip # Hypercube optimization diff --git a/config_rr2.yml b/config_rr2.yml index 93bb8b27..0e1f81c9 100644 --- a/config_rr2.yml +++ b/config_rr2.yml @@ -1,5 +1,5 @@ subfolder: random_rounding_majority -random_rounding: True +random_round: True random_round_type: majority # Hypercube optimization @@ -53,7 +53,7 @@ lmbda: 0.00005 # 0.01 #0.0001 #0.000001 # ===== Hardware setup ===== # workers: 4 -gpu: 3 +gpu: 2 # ===== Checkpointing ===== # checkpoint_at_prune: True From 3373e320728427d57b2cbcaa346a30c2e1c014cb Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Thu, 16 Dec 2021 15:53:31 -0600 Subject: [PATCH 6/8] random rounding test --- cifar_exec_GD.sh | 14 ++++- .../target_sparsity_1_4_naive_rounding.yml | 61 +++++++++++++++++++ ...dom_rounding_majority_5_train_and_test.yml | 61 +++++++++++++++++++ 3 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml create mode 100644 configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 9e79141a..8ffcc0af 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -29,12 +29,24 @@ #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_0.yml --run_idx 3 -# Random rounding +# Random rounding (majority/greedy, num_coin_flips, train, test) #python main.py --config config_rr1.yml --run_idx 1 #python main.py --config config_rr2.yml --run_idx 2 +python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_1_train_only.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_only.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_train_only.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_1_train_and_test.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_train_and_test.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_train_only.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_train_only.yml + +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_train_and_test.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_train_and_test.yml diff --git a/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml new file mode 100644 index 00000000..eee83885 --- /dev/null +++ b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml @@ -0,0 +1,61 @@ +subfolder: target_sparsity_1_4_naive_rounding + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml new file mode 100644 index 00000000..1c268dc7 --- /dev/null +++ b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml @@ -0,0 +1,61 @@ +subfolder: target_sparsity_1_4_highreg_modded + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True From 16710b83a2848b685bfaf6a4b8910c9dc9e11cbb Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Thu, 16 Dec 2021 16:38:01 -0600 Subject: [PATCH 7/8] testing majority train_and_test --- args_helper.py | 55 ++++++++-------- cifar_exec_GD.sh | 15 ++--- ...4_random_rounding_majority_5_test_only.yml | 63 +++++++++++++++++++ ...dom_rounding_majority_5_train_and_test.yml | 5 +- utils/conv_type.py | 17 ++--- utils/net_utils.py | 7 +++ 6 files changed, 121 insertions(+), 41 deletions(-) create mode 100644 configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml diff --git a/args_helper.py b/args_helper.py index fbfc1e08..4e13fc75 100644 --- a/args_helper.py +++ b/args_helper.py @@ -206,9 +206,24 @@ def parse_arguments(self, jupyter_mode=False): "--round", type=str, default='naive', - help='rounding technique to use |naive|prob|pb|' + help='rounding technique to use |naive|prob|pb|majority|greedy|' # naive: threshold(0.5), prob: probabilistic rounding, pb: pseudo-boolean paper's choice (RoundDown) + # round scheme for test (use same scheme for train) ) + parser.add_argument( + "--random_round_train", + action="store_true", + default=False, + help="randomized rounding for hc (at train)" + ) + parser.add_argument( + "--num_coin_flip_round", + type=int, + default=5, + help="Number of random coin flipping: 1 | 5 | 15 " + ) + + parser.add_argument( '--noise', action='store_true', @@ -251,19 +266,19 @@ def parse_arguments(self, jupyter_mode=False): type=int, help="rounding period for hypercube" ) - parser.add_argument( - "--num-round", - type=int, - default=1, - help='number of different models testing in rounding' - ) + # parser.add_argument( + # "--num-round", + # type=int, + # default=1, + # help='number of different models testing in rounding' + # ) # do we need it? - parser.add_argument( - "--num-test", - type=int, - default=1, - help='number of different models testing in prob rounding' - ) + # parser.add_argument( + # "--num-test", + # type=int, + # default=1, + # help='number of different models testing in prob rounding' + # ) parser.add_argument( "--save-model", action='store_true', @@ -837,18 +852,8 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="Enable this to unprune weights if possible, before fine-tune" ) - parser.add_argument( - "--random_round", - action="store_true", - default=False, - help="randomized rounding for hc" - ) - parser.add_argument( - "--random_round_type", - type=str, - default='one_flip', - help="Type of random rounding : one_flip | majority | best " - ) + + parser.add_argument( "--override-prune-rate", action="store_true", diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 8ffcc0af..f56cdcd5 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -32,18 +32,19 @@ # Random rounding (majority/greedy, num_coin_flips, train, test) #python main.py --config config_rr1.yml --run_idx 1 #python main.py --config config_rr2.yml --run_idx 2 -python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_1_train_only.yml -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_only.yml -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_train_only.yml +## Dec.16 +#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml # GPU 0 + +python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_test_only.yml -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_1_train_and_test.yml # python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml # python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_train_and_test.yml -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_train_only.yml -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_train_only.yml + +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_test_only.yml +# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_test_only.yml # python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_train_and_test.yml # python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_train_and_test.yml diff --git a/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml new file mode 100644 index 00000000..4341ce71 --- /dev/null +++ b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml @@ -0,0 +1,63 @@ +subfolder: target_sparsity_1_4_highreg_modded + + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: majority +num_coin_flip_round: 5 +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml index 1c268dc7..edd4e299 100644 --- a/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml +++ b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml @@ -1,5 +1,6 @@ subfolder: target_sparsity_1_4_highreg_modded + # Hypercube optimization algo: 'hc_iter' iter_period: 5 @@ -38,7 +39,9 @@ score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True # ===== Rounding ===== # -round: naive +round: majority +random_round_train: True +num_coin_flip_round: 5 noise: True noise_ratio: 0 diff --git a/utils/conv_type.py b/utils/conv_type.py index ff2d82bc..2689c4fc 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -78,21 +78,22 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc out = torch.gt(scores, torch.ones_like(scores)*scores_prune_threshold).float() bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float() else: - if parser_args.random_round: - if parser_args.random_round_type == 'one_flip': + if parser_args.random_round_train: + if parser_args.round == 'one_flip': out = torch.bernoulli(torch.clamp(scores, 0, 1)) bias_out = torch.bernoulli(torch.clamp(bias_scores, 0, 1)) - elif parser_args.random_round_type == 'majority': + elif parser_args.round == 'majority': # flip 5 coins and take dimension-wise majority voting out = torch.zeros_like(scores) bias_out = torch.zeros_like(bias_scores) - for flip_iter in range(5): + for flip_iter in range(parser_args.num_coin_flip_round): out += torch.bernoulli(torch.clamp(scores, 0, 1)) bias_out += torch.bernoulli(torch.clamp(bias_scores, 0, 1)) - out = torch.gt(out, torch.ones_like(out)*2).float() # among 5 trials, we need at least 3 heads - bias_out = torch.gt(bias_out, torch.ones_like(bias_out)*2).float() # among 5 trials, we need at least 3 heads - elif parser_args.random_round_type == 'best': - # compute loss of 5 coin flips and take the best one (how to compute loss?) + out = torch.gt(out, torch.ones_like(out)*np.floor(parser_args.num_coin_flip_round/2)).float() + bias_out = torch.gt(bias_out, torch.ones_like(bias_out)*np.floor(parser_args.num_coin_flip_round/2)).float() + elif parser_args.round == 'greedy': + # compute loss of 5 coin flips and take the best one (how to compute loss?) + #-> need to do outside of this function (here, just randomly pick the subnetwork, and do the comparison at outside) raise NotImplementedError else: diff --git a/utils/net_utils.py b/utils/net_utils.py index e3860523..1faa4f74 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -268,6 +268,13 @@ def round_model(model, round_scheme, noise=False, ratio=0.0, rank=None): params.data = torch.gt(params.data, torch.ones_like(params.data)*0.5).int().float() elif round_scheme == 'all_ones': params.data = torch.ones_like(params.data) + elif round_scheme == 'majority': + pdb.set_trace() + temp = torch.zeros_like(params.data) + for flip_iter in range(parser_args.num_coin_flip_round): + temp += torch.bernoulli(torch.clamp(params.data, 0.0, 1.0)) + temp = torch.gt(temp, torch.ones_like(temp)*np.floor(parser_args.num_coin_flip_round/2)).float() + params.data = temp else: print("INVALID ROUNDING") print("EXITING") From 4310bfae0dcc8ad2921c8c777a483d8062af7a1f Mon Sep 17 00:00:00 2001 From: root Date: Fri, 17 Dec 2021 02:18:53 +0000 Subject: [PATCH 8/8] random rounding train only, test only run --- args_helper.py | 10 ++++++ cifar_exec_GD.sh | 13 ++++--- ...4_random_rounding_majority_5_test_only.yml | 2 +- main.py | 18 +++++++--- utils/conv_type.py | 36 +++++++++---------- utils/net_utils.py | 11 +++--- 6 files changed, 54 insertions(+), 36 deletions(-) diff --git a/args_helper.py b/args_helper.py index 4e13fc75..7a8e121a 100644 --- a/args_helper.py +++ b/args_helper.py @@ -210,12 +210,22 @@ def parse_arguments(self, jupyter_mode=False): # naive: threshold(0.5), prob: probabilistic rounding, pb: pseudo-boolean paper's choice (RoundDown) # round scheme for test (use same scheme for train) ) + parser.add_argument( + "--round_train", + type=str, + default='naive', + help='rounding technique to use |naive|prob|pb|majority|greedy|' + # naive: threshold(0.5), prob: probabilistic rounding, pb: pseudo-boolean paper's choice (RoundDown) + # round scheme for test (use same scheme for train) + ) + ''' parser.add_argument( "--random_round_train", action="store_true", default=False, help="randomized rounding for hc (at train)" ) + ''' parser.add_argument( "--num_coin_flip_round", type=int, diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index f56cdcd5..e3f7f56e 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -35,17 +35,16 @@ ## Dec.16 #python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml # GPU 0 +#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml # gpu 0 +#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_test_only.yml # gpu 1 +#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_test_only.yml # gpu 2 +#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_test_only.yml # gpu 3 -python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_test_only.yml +python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_only.yml # gpu 1 +#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_train_only.yml # gpu 1 # python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml # python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_train_and_test.yml - - -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_test_only.yml -# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_test_only.yml - # python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_train_and_test.yml # python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_train_and_test.yml diff --git a/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml index 4341ce71..12804b99 100644 --- a/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml +++ b/configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml @@ -1,4 +1,4 @@ -subfolder: target_sparsity_1_4_highreg_modded +subfolder: target_sparsity_1_4_random_rounding_majority_5_test_only # Hypercube optimization diff --git a/main.py b/main.py index d685f90c..676c8543 100644 --- a/main.py +++ b/main.py @@ -133,13 +133,21 @@ def main_worker(gpu, ngpus_per_node): if parser_args.algo in ['hc', 'hc_iter']: br_acc1, br_acc5, br_acc10 = validate(data.val_loader, model, criterion, parser_args, writer, epoch) # before rounding print('Acc before rounding: {}'.format(br_acc1)) - acc_avg = 0 - for num_trial in range(parser_args.num_test): + if parser_args.round == 'greedy': + best_train_acc = 0 + best_model = None + for trial_idx in range(parser_args.num_coin_flip_round): + cp_model = round_model(model, parser_args.round, noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) + train_acc1, _, _ = validate(data.train_loader, cp_model, criterion, parser_args, writer, epoch) + if train_acc1 > best_train_acc: + best_train_acc = train_acc1 + best_model = cp_model + print('trial_idx: {}, best_train_acc: {}'.format(trial_idx, best_train_acc)) + acc1, acc5, acc10 = validate(data.val_loader, best_model, criterion, parser_args, writer, epoch) + cp_model = best_model + else: cp_model = round_model(model, parser_args.round, noise=parser_args.noise, ratio=parser_args.noise_ratio, rank=parser_args.gpu) acc1, acc5, acc10 = validate(data.val_loader, cp_model, criterion, parser_args, writer, epoch) - acc_avg += acc1 - acc_avg /= parser_args.num_test - acc1 = acc_avg print('Acc after rounding: {}'.format(acc1)) else: acc1, acc5, acc10 = validate(data.val_loader, model, criterion, parser_args, writer, epoch) diff --git a/utils/conv_type.py b/utils/conv_type.py index 2689c4fc..8216f3c8 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -78,27 +78,25 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc out = torch.gt(scores, torch.ones_like(scores)*scores_prune_threshold).float() bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float() else: - if parser_args.random_round_train: - if parser_args.round == 'one_flip': - out = torch.bernoulli(torch.clamp(scores, 0, 1)) - bias_out = torch.bernoulli(torch.clamp(bias_scores, 0, 1)) - elif parser_args.round == 'majority': - # flip 5 coins and take dimension-wise majority voting - out = torch.zeros_like(scores) - bias_out = torch.zeros_like(bias_scores) - for flip_iter in range(parser_args.num_coin_flip_round): - out += torch.bernoulli(torch.clamp(scores, 0, 1)) - bias_out += torch.bernoulli(torch.clamp(bias_scores, 0, 1)) - out = torch.gt(out, torch.ones_like(out)*np.floor(parser_args.num_coin_flip_round/2)).float() - bias_out = torch.gt(bias_out, torch.ones_like(bias_out)*np.floor(parser_args.num_coin_flip_round/2)).float() - elif parser_args.round == 'greedy': - # compute loss of 5 coin flips and take the best one (how to compute loss?) - #-> need to do outside of this function (here, just randomly pick the subnetwork, and do the comparison at outside) - raise NotImplementedError - - else: + if parser_args.round_train == 'naive': out = torch.gt(scores, torch.ones_like(scores)*parser_args.quantize_threshold).float() bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*parser_args.quantize_threshold).float() + elif parser_args.round_train == 'one_flip': + out = torch.bernoulli(torch.clamp(scores, 0, 1)) + bias_out = torch.bernoulli(torch.clamp(bias_scores, 0, 1)) + elif parser_args.round_train == 'majority': + # flip 5 coins and take dimension-wise majority voting + out = torch.zeros_like(scores) + bias_out = torch.zeros_like(bias_scores) + for flip_iter in range(parser_args.num_coin_flip_round): + out += torch.bernoulli(torch.clamp(scores, 0, 1)) + bias_out += torch.bernoulli(torch.clamp(bias_scores, 0, 1)) + out = torch.gt(out, torch.ones_like(out)*np.floor(parser_args.num_coin_flip_round/2)).float() + bias_out = torch.gt(bias_out, torch.ones_like(bias_out)*np.floor(parser_args.num_coin_flip_round/2)).float() + elif parser_args.round_train == 'greedy': + # compute loss of 5 coin flips and take the best one (how to compute loss?) + #-> need to do outside of this function (here, just randomly pick the subnetwork, and do the comparison at outside) + raise NotImplementedError else: print("INVALID PRUNING ALGO") diff --git a/utils/net_utils.py b/utils/net_utils.py index 1faa4f74..e474cf46 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -248,14 +248,14 @@ def round_model(model, round_scheme, noise=False, ratio=0.0, rank=None): else: cp_model = copy.deepcopy(model) for name, params in cp_model.named_parameters(): - if ".score" in name: + if ".score" in name: # or (parser_args.bias and ".bias_score" in name): if noise: delta = torch.randn_like(params.data)*ratio params.data += delta if round_scheme == 'naive': params.data = torch.gt(params.data, torch.ones_like(params.data)*parser_args.quantize_threshold).int().float() - elif round_scheme == 'prob': + elif round_scheme in ['prob', 'greedy']: params.data = torch.clamp(params.data, 0.0, 1.0) params.data = torch.bernoulli(params.data).float() elif round_scheme == 'naive_prob': @@ -269,11 +269,14 @@ def round_model(model, round_scheme, noise=False, ratio=0.0, rank=None): elif round_scheme == 'all_ones': params.data = torch.ones_like(params.data) elif round_scheme == 'majority': - pdb.set_trace() + # just for debug + #params.data = torch.gt(params.data, torch.ones_like(params.data)*parser_args.quantize_threshold).int().float() + + #pdb.set_trace() temp = torch.zeros_like(params.data) for flip_iter in range(parser_args.num_coin_flip_round): temp += torch.bernoulli(torch.clamp(params.data, 0.0, 1.0)) - temp = torch.gt(temp, torch.ones_like(temp)*np.floor(parser_args.num_coin_flip_round/2)).float() + temp = torch.gt(temp, torch.ones_like(temp)*np.floor(parser_args.num_coin_flip_round/2)).int().float() params.data = temp else: print("INVALID ROUNDING")