[FastHC]: Get to target sparsity Fast

args_helper.py

@@ -245,6 +245,12 @@ def parse_arguments(self, jupyter_mode=False):
             type=int,
             help="warmup epochs for hypercube"
         )
+        parser.add_argument(
+            "--toggle-warmup",
+            action='store_true',
+            default=False,
+            help='flag that decides if we add noise to the rounded p_i'
+        )
         parser.add_argument(
             "--hc-period",
             default=1,

cifar_exec.sh

@@ -25,7 +25,7 @@ BLOCK
 
 # target sparsity 0.5
 python main.py \
---config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_0_5.yml > cifar_log_target_0_5 2>&1
+--config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml #> cifar_log 2>&1
 
 
 # EP

configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov22_3pm.yml

@@ -40,7 +40,7 @@ hc_quantized: True
 quantize_threshold: 0.5
 
 # ===== Regularization ===== #
-regularization: L2
+regularization: L1
 #lmbda: 0.000001 # 0.01 #0.0001 #0.000001
 
 # ===== Hardware setup ===== #

configs/hypercube/resnet20/old_configs/resnet20_hypercube_bottom_K_Nov23_11am_SGD.yml

@@ -1,6 +1,6 @@
 # Hypercube optimization
 algo: 'hc_iter'
-#iter_period: 8 #15
+iter_period: 8 #15
 
 # Architecture
 arch: resnet20
@@ -12,10 +12,15 @@ name: resnet20_cifar10_hypercube_bottom_K_SGD
 # ===== Learning Rate Policy ======== #
 optimizer: sgd
 lr: 0.1 #0.01
-#lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr
+lr_policy: cosine_lr #constant_lr #multistep_lr # cosine_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
 
 # ===== Network training config ===== #
-epochs: 50 #150 
+epochs: 30 #150 
+unif_prune: False
+PRs: .60,.50,.50,.05,.40,.40
+epoch_pr: 1,5,10,15,20,30,40
 wd: 0.0 
 momentum: 0.9
 batch_size: 128
@@ -25,9 +30,9 @@ conv_type: SubnetConv
 bn_type: NonAffineBatchNorm
 freeze_weights: True
 prune_type: BottomK
-#prune_rate: 0.2
+prune_rate: 0.2
 init: signed_constant
-#score_init: skew #half #bimodal #skew # bern
+score_init: unif #half #bimodal #skew # bern
 scale_fan: False #True
 
 # ===== Rounding ===== #
@@ -40,8 +45,8 @@ hc_quantized: True
 quantize_threshold: 0.5
 
 # ===== Regularization ===== #
-#regularization: L2
-#lmbda: 0.000001 # 0.01 #0.0001 #0.000001
+regularization: L1
+lmbda: 0.00005 # 0.01 #0.0001 #0.000001
 
 # ===== Hardware setup ===== #
 workers: 4
@@ -53,3 +58,8 @@ checkpoint_at_prune: True
 # ==== sanity check ==== #
 skip_sanity_checks: True
 
+# ==== get sparsity quickly and train much ==== #
+fast_sparse: True
+
+# ====  ==== #
+finetune_standard: False

configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml

@@ -1,4 +1,4 @@
-subfolder: quantize_0_75
+subfolder: quantize_0_75_checking
 
 # Hypercube optimization
 algo: 'hc_iter'
@@ -14,7 +14,7 @@ name: resnet20_quantized_iter_hc
 
 # ===== Learning Rate Policy ======== #
 optimizer: sgd
-lr: 0.1 #0.01
+lr: 0.1
 lr_policy: cosine_lr #constant_lr #multistep_lr
 fine_tune_lr: 0.01
 fine_tune_lr_policy: multistep_lr
@@ -31,7 +31,7 @@ bn_type: NonAffineBatchNorm
 freeze_weights: True
 prune_type: BottomK
 # enter target sparsity here
-prune_rate: 0.995
+prune_rate: 0.982
 init: signed_constant
 score_init: unif #skew #half #bimodal #skew # bern
 scale_fan: False #True
@@ -51,10 +51,14 @@ lmbda: 0.00005 # 0.01 #0.0001 #0.000001
 
 # ===== Hardware setup ===== #
 workers: 4
-gpu: 0
+gpu: 3
 
 # ===== Checkpointing ===== #
 checkpoint_at_prune: True
 
 # ==== sanity check ==== #
-skip_sanity_checks: False
+skip_sanity_checks: True
+
+# ==== HC with warm-up ==== #
+hc_warmup: 3
+toggle_warmup: True

configs/hypercube/resnet20/resnet20_quantized_nonunif_iter_hc_0_5.yml

@@ -0,0 +1,67 @@
+subfolder: quantize_0_75_checking
+
+# Hypercube optimization
+algo: 'hc_iter'
+# effectively prune once in the entire process
+iter_period: 100
+unif_prune: False
+
+# Architecture
+arch: resnet20
+
+# ===== Dataset ===== #
+dataset: CIFAR10
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150 
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+prune_rate: 0.982
+PRs: 0.5,0.5,0.5,0.5,0.288
+epoch_pr: 20,40,60,80,100,200
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.00005 # 0.01 #0.0001 #0.000001
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 3
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: True
+
+# ==== HC with warm-up ==== #
+hc_warmup: 3
+toggle_warmup: False

configs/hypercube/resnet20/test_warmup.yml

@@ -0,0 +1,64 @@
+subfolder: quantize_0_75
+
+# Hypercube optimization
+algo: 'hc_iter'
+# effectively prune once in the entire process
+iter_period: 100
+
+# Architecture
+arch: TwoLayerFC
+
+# ===== Dataset ===== #
+dataset: MNIST
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150 
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+prune_rate: 0.995
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.00005 # 0.01 #0.0001 #0.000001
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: False
+
+# ==== HC with warm-up ==== #
+hc_warmup: 3
+toggle_warmup: True

main.py

@@ -1,3 +1,4 @@
+from yaml import parse
 from main_utils import *
 
 
@@ -41,6 +42,12 @@ def main_worker(gpu, ngpus_per_node):
 
     if not os.path.isdir(result_root):
         os.mkdir(result_root)
+
+    if not parser_args.unif_prune:
+        prune_rates = list(map(float, parser_args.PRs.split(',')))
+        prune_epochs = list(map(float, parser_args.epoch_pr.split(',')))
+        parser_args.prune_rate = prune_rates[0]
+
     model = get_model(parser_args)
 
     '''
@@ -147,10 +154,19 @@ def main_worker(gpu, ngpus_per_node):
         validation_time.update((time.time() - start_validation) / 60)
 
         # prune the model every T_{prune} epochs
-        if parser_args.algo in ['hc_iter', 'global_ep_iter'] and epoch % (parser_args.iter_period) == 0 and epoch != 0:
+        if parser_args.algo in ['hc_iter', 'global_ep_iter'] and parser_args.unif_prune and epoch % (parser_args.iter_period) == 0 and epoch != 0:
             prune(model)
             if parser_args.checkpoint_at_prune:
                 save_checkpoint_at_prune(model, parser_args)
+
+        #prune model non-uniformly
+        if not parser_args.unif_prune:
+            if epoch == prune_epochs[0]-1:
+                print('PRUNING NON-UNIFORMLY ######################################********')
+                prune_epochs.pop(0)
+                parser_args.prune_rate = prune_rates.pop(0)
+                set_model_prune_rate(model, parser_args.prune_rate)
+                prune(model)
 
         # get model sparsity
         if not parser_args.weight_training:

main_utils.py

@@ -213,6 +213,13 @@ def eval_and_print(validate, data_loader, model, criterion, parser_args, writer=
     return acc1
 
 def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_before_round_list, old_test_acc_list, old_reg_loss_list, old_model_sparsity_list, result_root, shuffle=False, reinit=False, invert=False, chg_mask=False, chg_weight=False):
+    #this order is important because the second will always override the first!!!
+    if parser_args.fast_sparse:
+        finetune_epochs = 300 - parser_args.epochs
+    if parser_args.finetune_standard:
+        finetune_epochs = 150
+    else:
+        finetune_epochs = 300 - parser_args.epochs
     epoch_list = copy.deepcopy(old_epoch_list)
     test_acc_before_round_list = copy.deepcopy(old_test_acc_before_round_list)
     test_acc_list = copy.deepcopy(old_test_acc_list)
@@ -269,7 +276,7 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b
     model_sparsity_list.append(avg_sparsity)
 
     end_epoch = time.time()
-    for epoch in range(parser_args.epochs, parser_args.epochs*2):
+    for epoch in range(parser_args.epochs, parser_args.epochs+finetune_epochs):
 
         if parser_args.multiprocessing_distributed:
             data.train_loader.sampler.set_epoch(epoch)
@@ -335,7 +342,10 @@ def get_idty_str(parser_args):
     model_str = parser_args.arch
     algo_str = parser_args.algo
     rate_str = parser_args.prune_rate
+    #rate_str = parser_args.PRs
     period_str = parser_args.iter_period
+    #period_str = parser_args.epoch_pr
+    epoch_str = parser_args.epochs
     reg_str = 'reg_{}'.format(parser_args.regularization)
     reg_lmbda = parser_args.lmbda if parser_args.regularization else ''
     opt_str = parser_args.optimizer
@@ -356,7 +366,10 @@ def get_idty_str(parser_args):
         format(train_mode_str, dataset_str, model_str, algo_str, rate_str, period_str, reg_str, reg_lmbda,
         opt_str, policy_str, lr_str, lr_gamma, lr_adj, finetune_lr_str, lam_ft_str, n_step_ft_str, fan_str, w_str, s_str,
         width_str, seed_str, run_idx_str).replace(".", "_")
-
+     # idty_str = "{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_finetune_{}_fan_{}_{}_{}_width_{}_seed_{}_idx_{}".\
+     #  format(train_mode_str, dataset_str, model_str, algo_str, rate_str, period_str, reg_str, reg_lmbda,
+     #  opt_str, policy_str, lr_str, lr_gamma, lr_adj, finetune_lr_str, fan_str, w_str, s_str,
+     #      width_str, seed_str, run_idx_str).replace(".", "_")
 
     return idty_str
 
@@ -415,10 +428,31 @@ def compare_rounding(validate, data_loader, model, criterion, parser_args, resul
 
 
 # switches off gradients for scores and flags and switches it on for weights and biases
-def switch_to_wt(model):
+def switch_to_pruning(model, reinit_scores=False):
     print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.')
 
     parser_args.lam_finetune_loss = 0   # this is for the case considering finetune loss
+    for name, params in model.named_parameters():
+        # make sure param_name ends with .weight or .bias
+        if re.match('.*\.weight', name):
+            params.requires_grad = False
+        elif parser_args.bias and re.match('.*\.bias$', name):
+            params.requires_grad = False
+        elif "score" in name:
+            params.requires_grad = True
+            if reinit_scores:
+                nn.init.uniform_(params, a=0.0, b=1.0)
+            # NOTE: this does not take parser_args.score_init into consideration
+            # HACK: does not work for bias, yet
+        else:
+            # flags and everything else
+            params.requires_grad = False
+    return model
+
+
+# switches off gradients for scores and flags and switches it on for weights and biases
+def switch_to_wt(model, set_scores_to_one=False):
+    print('Switching to weight training by switching off requires_grad for scores and switching it on for weights.')
     for name, params in model.named_parameters():
         # make sure param_name ends with .weight or .bias
         if re.match('.*\.weight', name):
@@ -427,6 +461,10 @@ def switch_to_wt(model):
             params.requires_grad = True
         elif "score" in name:
             params.requires_grad = False
+            # set all scores to one if you want to do vanilla weight training
+            if set_scores_to_one:
+                params.data = torch.ones_like(params.data)
+                # NOTE: Hack | does not work for bias, yet
         else:
             # flags and everything else
             params.requires_grad = False

matt.sh

@@ -0,0 +1,5 @@
+### ResNet20
+
+do
+	python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5.yml
+done