config file for adding finetune_loss

args_helper.py

@@ -720,6 +720,18 @@ def parse_arguments(self, jupyter_mode=False):
             default=False,
             help="chg weights before sanity check"
         )
+        parser.add_argument(
+            "--finetune_last_epochs",
+            action="store_true",
+            default=False,
+            help="used for adding finetune loss at last 10 epochs of HC"
+        )
+        parser.add_argument(
+            "--finetuned",
+            action="store_true",
+            default=False,
+            help="used for finetuned loss (please set it false in the command line. The code will automatically turn on/off it"
+        )
         parser.add_argument(
             "--fine-tune-optimizer",
             type=str,

cifar_exec_GD.sh

@@ -1,5 +1,6 @@
 #export cuda_visible_devices=3
 
+# ResNet-18
 
 #### ResNet-18
 #python main.py --config configs/hypercube/resnet18/resnet18_sc_hypercube_reg.yml # 93.17% at 150 epoch
@@ -12,13 +13,41 @@
 
 
 
-### ResNet-20
+# ResNet-20
+#python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #> log_EP_sparsity_50 2>&1
 
+## HC for multiple trials
+:<<BLOCK
+python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t2.yml > log_hc_sparsity_3_72_t2 2>&1
+python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t3.yml > log_hc_sparsity_3_72_t3 2>&1
+python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t4.yml > log_hc_sparsity_3_72_t4 2>&1
+python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t5.yml > log_hc_sparsity_3_72_t5 2>&1
+BLOCK
+
+:<<BLOCK
+python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t2.yml > log_hc_sparsity_0_59_t2 2>&1
+python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t3.yml > log_hc_sparsity_0_59_t3 2>&1
+python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t4.yml > log_hc_sparsity_0_59_t4 2>&1
+python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t5.yml > log_hc_sparsity_0_59_t5 2>&1
+BLOCK
+
+## ran in finetune_loss repo
+#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t1.yml > log_hc_sparsity_1_35_t1 2>&1
+#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t2.yml > log_hc_sparsity_1_35_t2 2>&1
+
+#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t3.yml > log_hc_sparsity_1_35_t3 2>&1
+#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t4.yml > log_hc_sparsity_1_35_t4 2>&1
+#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t5.yml > log_hc_sparsity_1_35_t5 2>&1
+
+
+
+## EP
+#python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_50.yml > log_EP_sparsity_50 2>&1
 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t1.yml #> log_hc_sparsity_3_72_t1 2>&1
 
 # HC for multiple trials
 #:<<BLOCK
-python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t1.yml > log_hc_sparsity_3_72_t1 2>&1
+#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t1.yml > log_hc_sparsity_3_72_t1 2>&1
 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t2.yml > log_hc_sparsity_3_72_t2 2>&1
 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t3.yml > log_hc_sparsity_3_72_t3 2>&1
 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t4.yml > log_hc_sparsity_3_72_t4 2>&1
@@ -42,6 +71,7 @@ python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3
 #python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_1_44.yml > log_EP_sparsity_1_44 2>&1
 #python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_0_59.yml > log_EP_sparsity_0_59 2>&1
 #python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_0_15.yml > log_EP_sparsity_0_15 2>&1
+
 #python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml 
 
 
@@ -59,11 +89,40 @@ python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3
 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag.yml > log_target_sparsity_50_without_flag_lam_0 2>&1
 
 
+## adding finetune for denser models
+#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag_with_finetune.yml > log_target_sparsity_5_without_unflag_with_finetune_lam_3e-5 2>&1
+#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune.yml > log_target_sparsity_20_without_unflag_with_finetune_lam_1e-5 2>&1
+#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag_with_finetune.yml > log_target_sparsity_50_without_unflag_with_finetune_lam_0 2>&1
+
+#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_5_without_unflag_with_finetune_last_10_epochs_lam_3e-5 2>&1
+#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_20_without_unflag_with_finetune_last_10_epochs_lam_1e-5 2>&1
+#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_50_without_unflag_with_finetune_last_10_epochs_lam_0 2>&1
 
 
 ## testing adding finetune loss
+#python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml > log_no_finetune_5e-5 2>&1
+#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs_without_unflag.yml > log_finetune_1e-4_lam_1_num_5_last_10_epochs_without_unflag 2>&1
+#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs.yml > log_finetune_1e-4_lam_1_num_5_last_10_epochs 2>&1
+#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs_without_unflag.yml > log_finetune_5e-5_lam_1_num_5_last_10_epochs_without_unflag 2>&1
+#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs.yml > log_finetune_5e-5_lam_1_num_5_last_10_epochs 2>&1
+
+
+#python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune_without_unflag.yml
+#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_without_unflag.yml
+#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5_without_unflag.yml
+
+
+
 #python main.py --config config1.yml --run_idx 1 #  
+#python main.py --config config1.yml --run_idx 0 #  
 #python main.py --config config2.yml --run_idx 2
+#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_10.yml
+#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_10.yml
+
+
+
+
+
 
 
 

config1.yml

@@ -0,0 +1,62 @@
+subfolder: quantize_0_5_finetune_loss_test
+lam_finetune_loss: 0 #100
+num_step_finetune: 10
+
+# Hypercube optimization
+algo: 'hc_iter'
+# effectively prune once in the entire process
+iter_period: 100
+
+# Architecture
+arch: resnet20
+
+# ===== Dataset ===== #
+dataset: CIFAR10
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150 
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+prune_rate: 0.982 #0.995
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0 #0.00005 # 0.01 #0.0001 #0.000001
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 2
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: False

configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_without_unflag.yml

@@ -0,0 +1,63 @@
+subfolder: target_sparsity_1_4_finetune_lam_1_num_5_without_unflag
+lam_finetune_loss: 1
+num_step_finetune: 5
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: resnet20
+
+# ===== Dataset ===== #
+dataset: CIFAR10
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 1.4
+# decide if you want to "unflag"
+#unflag_before_finetune: True
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.0001 # 1e-4 #0.00005 # 5e-5
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 2
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: True