random rounding

args_helper.py

@@ -206,9 +206,34 @@ def parse_arguments(self, jupyter_mode=False):
             "--round",
             type=str,
             default='naive',
-            help='rounding technique to use |naive|prob|pb|'
+            help='rounding technique to use |naive|prob|pb|majority|greedy|'
             # naive: threshold(0.5), prob: probabilistic rounding, pb: pseudo-boolean paper's choice (RoundDown)
+            # round scheme for test (use same scheme for train)
         )
+        parser.add_argument(
+            "--round_train",
+            type=str,
+            default='naive',
+            help='rounding technique to use |naive|prob|pb|majority|greedy|'
+            # naive: threshold(0.5), prob: probabilistic rounding, pb: pseudo-boolean paper's choice (RoundDown)
+            # round scheme for test (use same scheme for train)
+        )
+        '''
+        parser.add_argument(
+            "--random_round_train",
+            action="store_true",
+            default=False,
+            help="randomized rounding for hc (at train)"
+        )
+        '''
+        parser.add_argument(
+            "--num_coin_flip_round",
+            type=int,
+            default=5,
+            help="Number of random coin flipping: 1 | 5 | 15 "
+        )
+
+
         parser.add_argument(
             '--noise',
             action='store_true',
@@ -251,19 +276,19 @@ def parse_arguments(self, jupyter_mode=False):
             type=int,
             help="rounding period for hypercube"
         )
-        parser.add_argument(
-            "--num-round",
-            type=int,
-            default=1,
-            help='number of different models testing in rounding'
-        )
+        # parser.add_argument(
+        #     "--num-round",
+        #     type=int,
+        #     default=1,
+        #     help='number of different models testing in rounding'
+        # )
         # do we need it?
-        parser.add_argument(
-            "--num-test",
-            type=int,
-            default=1,
-            help='number of different models testing in prob rounding'
-        )
+        # parser.add_argument(
+        #     "--num-test",
+        #     type=int,
+        #     default=1,
+        #     help='number of different models testing in prob rounding'
+        # )
         parser.add_argument(
             "--save-model",
             action='store_true',
@@ -837,6 +862,8 @@ def parse_arguments(self, jupyter_mode=False):
             default=False,
             help="Enable this to unprune weights if possible, before fine-tune"
         )
+
+
         parser.add_argument(
             "--override-prune-rate",
             action="store_true",

cifar_exec_GD.sh

@@ -14,21 +14,44 @@
 ### ResNet-20
 #python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml 
 #python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml 
-python main.py --config config_current_best.yml --run_idx 1
+#python main.py --config config_current_best.yml --run_idx 0
+
+# Add finetune-loss
+#python main.py --config config_current_best.yml --run_idx 1
 
 
 ## testing adding finetune loss
 #python main.py --config config1.yml --run_idx 1 #  
 #python main.py --config config2.yml --run_idx 2
-
-
-
-
 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1.yml --run_idx 1
 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1e-2.yml --run_idx 1
 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1e-4.yml --run_idx 2
 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_0.yml --run_idx 3
 
+
+# Random rounding (majority/greedy, num_coin_flips, train, test)
+#python main.py --config config_rr1.yml --run_idx 1
+#python main.py --config config_rr2.yml --run_idx 2
+
+## Dec.16
+#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml # GPU 0
+#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml # gpu 0
+#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_test_only.yml # gpu 1
+#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_test_only.yml # gpu 2
+#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_test_only.yml # gpu 3
+
+python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_only.yml # gpu 1
+#python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_train_only.yml # gpu 1
+
+# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_train_and_test.yml
+# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_15_train_and_test.yml
+# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_5_train_and_test.yml
+# python main.py --config configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_greedy_15_train_and_test.yml
+
+
+
+
+
 #python main.py --config config10.yml --run_idx 10 #> log_config$r 2>&1	
 :<<BLOCK
 run_list=(3 4)

config_rr1.yml

@@ -0,0 +1,62 @@
+subfolder: random_rounding_one_flip
+random_round: True
+random_round_type: one_flip
+
+# Hypercube optimization
+algo: 'hc_iter'
+# effectively prune once in the entire process
+iter_period: 100
+
+# Architecture
+arch: resnet20
+
+# ===== Dataset ===== #
+dataset: CIFAR10
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150 
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+prune_rate: 0.982
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.00005 # 0.01 #0.0001 #0.000001
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 3
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: False

config_rr2.yml

@@ -0,0 +1,62 @@
+subfolder: random_rounding_majority
+random_round: True
+random_round_type: majority
+
+# Hypercube optimization
+algo: 'hc_iter'
+# effectively prune once in the entire process
+iter_period: 100
+
+# Architecture
+arch: resnet20
+
+# ===== Dataset ===== #
+dataset: CIFAR10
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150 
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+prune_rate: 0.982
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.00005 # 0.01 #0.0001 #0.000001
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 2
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: False

configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_naive_rounding.yml

@@ -0,0 +1,61 @@
+subfolder: target_sparsity_1_4_naive_rounding
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: resnet20
+
+# ===== Dataset ===== #
+dataset: CIFAR10
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 1.4
+# decide if you want to "unflag"
+unflag_before_finetune: True
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.00005 # 5e-5
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/hypercube/resnet20/random_rounding/target_sparsity_1_4_random_rounding_majority_5_test_only.yml

@@ -0,0 +1,63 @@
+subfolder: target_sparsity_1_4_random_rounding_majority_5_test_only
+
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: resnet20
+
+# ===== Dataset ===== #
+dataset: CIFAR10
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 1.4
+# decide if you want to "unflag"
+unflag_before_finetune: True
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: majority 
+num_coin_flip_round: 5
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.00005 # 5e-5
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: True