mobile tiny

args_helper.py

@@ -742,6 +742,12 @@ def parse_arguments(self, jupyter_mode=False):
             metavar="LR",
             help="Learning rate for fine-tuning weights"
         )
+        parser.add_argument(
+            "--fine-tune-lr-gamma",
+            type=float,
+            default=0.1,
+            help="Learning rate decay for multi-step"
+        )
         parser.add_argument(
             "--fine-tune-wd",
             type=float,

configs/ep/tinyImageNet/mobilenetV2/sparsity_1_4.yml

@@ -0,0 +1,43 @@
+#subfolder: EP_sparsity_1_4
+algo: 'ep'
+override_prune_rate: True
+
+# Architecture
+arch: tinyMobileNetV2
+
+# ===== Dataset ===== #
+dataset: TinyImageNet
+name: mobileV2_tiny
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1
+lr_policy: cosine_lr
+fine_tune_lr: 0.1
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 300 
+fine_tune_wd: 0.00004
+momentum: 0.9
+batch_size: 128 #256
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+init: signed_constant
+mode: fan_in
+nonlinearity: relu
+prune_rate: 0.014
+scale_fan: True
+freeze_weights: True
+
+# ===== Hardware setup ===== #
+workers: 4
+#gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/ep/tinyImageNet/mobilenetV2/sparsity_20.yml

@@ -0,0 +1,43 @@
+#subfolder: EP_sparsity_20
+algo: 'ep'
+override_prune_rate: True
+
+# Architecture
+arch: tinyMobileNetV2
+
+# ===== Dataset ===== #
+dataset: TinyImageNet
+name: mobileV2_tiny
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1
+lr_policy: cosine_lr
+fine_tune_lr: 0.1
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 300 
+fine_tune_wd: 0.00004
+momentum: 0.9
+batch_size: 128 #256
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+init: signed_constant
+mode: fan_in
+nonlinearity: relu
+prune_rate: 0.20
+scale_fan: True
+freeze_weights: True
+
+# ===== Hardware setup ===== #
+workers: 4
+#gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/ep/tinyImageNet/mobilenetV2/sparsity_5.yml

@@ -0,0 +1,43 @@
+#subfolder: EP_sparsity_5
+algo: 'ep'
+override_prune_rate: True
+
+# Architecture
+arch: tinyMobileNetV2
+
+# ===== Dataset ===== #
+dataset: TinyImageNet
+name: mobileV2_tiny
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1
+lr_policy: cosine_lr
+fine_tune_lr: 0.1
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 300 
+fine_tune_wd: 0.00004
+momentum: 0.9
+batch_size: 128 #256
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+init: signed_constant
+mode: fan_in
+nonlinearity: relu
+prune_rate: 0.05
+scale_fan: True
+freeze_weights: True
+
+# ===== Hardware setup ===== #
+workers: 4
+#gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/ep/tinyImageNet/mobilenetV2/sparsity_5_adam.yml

@@ -0,0 +1,43 @@
+#subfolder: EP_sparsity_5
+algo: 'ep'
+override_prune_rate: True
+
+# Architecture
+arch: tinyMobileNetV2
+
+# ===== Dataset ===== #
+dataset: TinyImageNet
+name: mobileV2_tiny
+
+# ===== Learning Rate Policy ======== #
+optimizer: adam
+lr: 0.001
+lr_policy: cosine_lr
+fine_tune_lr: 0.1
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 300 
+fine_tune_wd: 0.00004
+momentum: 0.9
+batch_size: 128 #256
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+init: signed_constant
+mode: fan_in
+nonlinearity: relu
+prune_rate: 0.05
+scale_fan: True
+freeze_weights: True
+
+# ===== Hardware setup ===== #
+workers: 4
+#gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/hypercube/tinyImageNet/mobilenetV2/adam.yml

@@ -0,0 +1,62 @@
+#subfolder: tiny_mobilenetV2_hc_sparsity_1_4
+trial_num: 1
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: tinyMobileNetV2
+
+# ===== Dataset ===== #
+dataset: TinyImageNet
+name: mobilenetV2_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: adam
+lr: 0.001 #0.01
+wd: 0.0
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.1
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 300 #300
+fine_tune_wd: 0.00004
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+#target_sparsity: 1.4
+unflag_before_finetune: True
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+#lmbda: 0.000008
+
+# ===== Hardware setup ===== #
+workers: 4
+#gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/hypercube/tinyImageNet/mobilenetV2/sgd.yml

@@ -0,0 +1,62 @@
+#subfolder: tiny_mobilenetV2_hc_sparsity_1_4
+trial_num: 1
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: tinyMobileNetV2
+
+# ===== Dataset ===== #
+dataset: TinyImageNet
+name: mobilenetV2_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+wd: 0.0
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.1
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 300 #300
+fine_tune_wd: 0.00004
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+#target_sparsity: 1.4
+unflag_before_finetune: True
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+#lmbda: 0.000008
+
+# ===== Hardware setup ===== #
+workers: 4
+#gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml

@@ -0,0 +1,62 @@
+subfolder: tiny_mobilenetV2_hc_sparsity_1_4
+trial_num: 1
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: tinyMobileNetV2
+
+# ===== Dataset ===== #
+dataset: TinyImageNet
+name: mobilenetV2_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: adam
+lr: 0.001 #0.01
+wd: 0.0
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.1
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 300 #300
+fine_tune_wd: 0.00004
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 1.4
+unflag_before_finetune: True
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.000008
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: True