diff --git a/args_helper.py b/args_helper.py index b03fbb95..72ba940b 100644 --- a/args_helper.py +++ b/args_helper.py @@ -742,6 +742,12 @@ def parse_arguments(self, jupyter_mode=False): metavar="LR", help="Learning rate for fine-tuning weights" ) + parser.add_argument( + "--fine-tune-lr-gamma", + type=float, + default=0.1, + help="Learning rate decay for multi-step" + ) parser.add_argument( "--fine-tune-wd", type=float, diff --git a/configs/ep/tinyImageNet/mobilenetV2/sparsity_1_4.yml b/configs/ep/tinyImageNet/mobilenetV2/sparsity_1_4.yml new file mode 100644 index 00000000..40ff568b --- /dev/null +++ b/configs/ep/tinyImageNet/mobilenetV2/sparsity_1_4.yml @@ -0,0 +1,43 @@ +#subfolder: EP_sparsity_1_4 +algo: 'ep' +override_prune_rate: True + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobileV2_tiny + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 #256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +prune_rate: 0.014 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/ep/tinyImageNet/mobilenetV2/sparsity_20.yml b/configs/ep/tinyImageNet/mobilenetV2/sparsity_20.yml new file mode 100644 index 00000000..3d32df4c --- /dev/null +++ b/configs/ep/tinyImageNet/mobilenetV2/sparsity_20.yml @@ -0,0 +1,43 @@ +#subfolder: EP_sparsity_20 +algo: 'ep' +override_prune_rate: True + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobileV2_tiny + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 #256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +prune_rate: 0.20 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/ep/tinyImageNet/mobilenetV2/sparsity_5.yml b/configs/ep/tinyImageNet/mobilenetV2/sparsity_5.yml new file mode 100644 index 00000000..0a8013c1 --- /dev/null +++ b/configs/ep/tinyImageNet/mobilenetV2/sparsity_5.yml @@ -0,0 +1,43 @@ +#subfolder: EP_sparsity_5 +algo: 'ep' +override_prune_rate: True + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobileV2_tiny + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 #256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +prune_rate: 0.05 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/ep/tinyImageNet/mobilenetV2/sparsity_5_adam.yml b/configs/ep/tinyImageNet/mobilenetV2/sparsity_5_adam.yml new file mode 100644 index 00000000..6ec42381 --- /dev/null +++ b/configs/ep/tinyImageNet/mobilenetV2/sparsity_5_adam.yml @@ -0,0 +1,43 @@ +#subfolder: EP_sparsity_5 +algo: 'ep' +override_prune_rate: True + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobileV2_tiny + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 #256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +prune_rate: 0.05 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/adam.yml b/configs/hypercube/tinyImageNet/mobilenetV2/adam.yml new file mode 100644 index 00000000..d712c7f0 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/adam.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +#target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +#lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sgd.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sgd.yml new file mode 100644 index 00000000..ac3abbe3 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sgd.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +#target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +#lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml new file mode 100644 index 00000000..6b1cafd3 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml @@ -0,0 +1,62 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml new file mode 100644 index 00000000..a58dae7b --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml @@ -0,0 +1,63 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 +invert_sanity_check: True + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000009 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml new file mode 100644 index 00000000..1c6d2d27 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000009 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml new file mode 100644 index 00000000..a7e4b9a8 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml @@ -0,0 +1,64 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 +invert_sanity_check: True + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000003 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True + diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_sanity.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_sanity.yml new file mode 100644 index 00000000..b480b3ed --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_sanity.yml @@ -0,0 +1,63 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000003 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False + diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml new file mode 100644 index 00000000..d2cefdca --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml @@ -0,0 +1,62 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_2_5 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.5 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000005 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml new file mode 100644 index 00000000..1b67b5d2 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml @@ -0,0 +1,63 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 +invert_sanity_check: True + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml new file mode 100644 index 00000000..b7ff3cf7 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml new file mode 100644 index 00000000..7b54d82c --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml @@ -0,0 +1,62 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_5 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml new file mode 100644 index 00000000..541cfc6e --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml @@ -0,0 +1,61 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_50 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 50.0 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml new file mode 100644 index 00000000..69c42963 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml @@ -0,0 +1,62 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_5 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/imp/tiny_mobilenet.yml b/configs/imp/tiny_mobilenet.yml new file mode 100644 index 00000000..b176907b --- /dev/null +++ b/configs/imp/tiny_mobilenet.yml @@ -0,0 +1,36 @@ +# IMP algorithm +algo: 'imp' +seed: 42 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: multistep_lr +lr_gamma: 0.1 + +# ===== Network training config ===== # +# epochs: 300 +wd: 0.00004 +momentum: 0.9 +batch_size: 128 +bias: False + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +prune_rate: 0.2 +init: kaiming_normal +iter_period: 300 # 5 +imp_rewind_iter: 1000 + + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + diff --git a/configs/sr/tiny_sr.yml b/configs/sr/tiny_sr.yml index 3cfd00a4..c0901d0b 100644 --- a/configs/sr/tiny_sr.yml +++ b/configs/sr/tiny_sr.yml @@ -20,9 +20,9 @@ fine_tune_lr_policy: multistep_lr # cosine_lr fine_tune_wd: 0.0001 # ===== Network training config ===== # -epochs: 200 #150 +epochs: 300 #150 momentum: 0.9 -batch_size: 256 +batch_size: 128 bias: False # ===== Sparsity =========== # @@ -40,7 +40,7 @@ workers: 4 # gpu: 3 # ===== Checkpointing ===== # -checkpoint_at_prune: True +checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: True diff --git a/configs/sr/tiny_sr_mobilenet.yml b/configs/sr/tiny_sr_mobilenet.yml new file mode 100644 index 00000000..b68bfdad --- /dev/null +++ b/configs/sr/tiny_sr_mobilenet.yml @@ -0,0 +1,46 @@ +# subfolder: sr +algo: 'hc_iter' +random_subnet: True +# smart_ratio: 0.98 # 2% sparsity + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_mobilenetV2_sr + +# ===== Learning Rate Policy ======== # +optimizer: sgd +# doesn't matter +# lr: 0.1 +# lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: cosine_lr #multistep_lr # cosine_lr +fine_tune_wd: 0 #0.00004 + +# ===== Network training config ===== # +epochs: 300 #150 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True + +init: kaiming_normal # signed_constant +mode: fan_in +nonlinearity: relu +# prune_rate: 0.5 +scale_fan: True + +# ===== Hardware setup ===== # +workers: 4 +# gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/training/mobilenetV2/tiny_sgd.yml b/configs/training/mobilenetV2/tiny_sgd.yml new file mode 100644 index 00000000..0144d997 --- /dev/null +++ b/configs/training/mobilenetV2/tiny_sgd.yml @@ -0,0 +1,44 @@ + +subfolder: tiny_mobile_weight_training_sgd +trial_num: 1 + + +# algorithm +algo: 'hc_iter' + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_mobile_training + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: multistep_lr #cosine_lr #constant_lr + +# ===== Network training config ===== # +epochs: 300 +wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +weight_training: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_rate: -1 +init: kaiming_normal +scale_fan: True + +# ===== Hardware setup ===== # +workers: 4 +gpu: 1 + +# ==== sanity check ==== # +skip_fine_tune: True +skip_sanity_checks: True + diff --git a/main.py b/main.py index 4e6512a5..2fe01432 100644 --- a/main.py +++ b/main.py @@ -64,7 +64,7 @@ def main_worker(gpu, ngpus_per_node): model2 = None optimizer = get_optimizer(parser_args, model) data = get_dataset(parser_args) - scheduler = get_scheduler(optimizer, parser_args.lr_policy) + scheduler = get_scheduler(optimizer, policy=parser_args.lr_policy, gamma=parser_args.gamma) #lr_policy = get_policy(parser_args.lr_policy)(optimizer, parser_args) if parser_args.label_smoothing is None: criterion = nn.CrossEntropyLoss().cuda() @@ -257,7 +257,8 @@ def main_worker(gpu, ngpus_per_node): parser_args.prune_rate = sum_pr / count writer.add_scalar("pr/average", parser_args.prune_rate, epoch) - writer.add_scalar("test/lr", cur_lr, epoch) + if writer is not None: + writer.add_scalar("test/lr", cur_lr, epoch) end_epoch = time.time() if parser_args.algo in ['hc', 'hc_iter']: diff --git a/main_utils.py b/main_utils.py index db0652ab..e403424e 100644 --- a/main_utils.py +++ b/main_utils.py @@ -11,7 +11,7 @@ import random import time import pandas as pd -from torch.utils.tensorboard import SummaryWriter +# from torch.utils.tensorboard import SummaryWriter import torch import torch.nn as nn import torch.nn.parallel @@ -272,7 +272,7 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b parser_args) optimizer = get_optimizer(parser_args, model, finetune_flag=True) - scheduler = get_scheduler(optimizer, policy=parser_args.fine_tune_lr_policy) + scheduler = get_scheduler(optimizer, policy=parser_args.fine_tune_lr_policy, gamma=parser_args.fine_tune_lr_gamma) ''' if parser_args.epochs == 150: scheduler = get_scheduler(optimizer, parser_args.fine_tune_lr_policy, milestones=[ @@ -340,7 +340,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b progress_overall.write_to_tensorboard( writer, prefix="diagnostics", global_step=epoch ) - writer.add_scalar("test/lr", cur_lr, epoch) + if writer is not None: + writer.add_scalar("test/lr", cur_lr, epoch) end_epoch = time.time() results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, @@ -400,8 +401,8 @@ def get_settings(parser_args): run_base_dir, ckpt_base_dir, log_base_dir = get_directories(parser_args) parser_args.ckpt_base_dir = ckpt_base_dir - writer = SummaryWriter(log_dir=log_base_dir) - # writer = None + # writer = SummaryWriter(log_dir=log_base_dir) + writer = None epoch_time = AverageMeter("epoch_time", ":.4f", write_avg=False) validation_time = AverageMeter("validation_time", ":.4f", write_avg=False) train_time = AverageMeter("train_time", ":.4f", write_avg=False) diff --git a/models/__init__.py b/models/__init__.py index 216a0bfb..f6160cc8 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -8,6 +8,7 @@ #### TODO: delete below ones (merge with above code) from models.resnet_cifar import cResNet18, cResNet50 from models.resnet_tiny import TinyResNet18 +from models.mobilenet import tinyMobileNetV2 __all__ = [ "tinyvgg16", @@ -20,6 +21,7 @@ "resnet20", "resnet32", "resnet32_double", + "tinyMobileNetV2", "MobileNetV2", "FC", "Conv2", diff --git a/models/mobilenet.py b/models/mobilenet.py index 65328a30..06e40e5e 100644 --- a/models/mobilenet.py +++ b/models/mobilenet.py @@ -108,8 +108,8 @@ def forward(self, x): def MobileNetV2(): return MobileNet_base(get_builder()) -def tinyMobileNetV2(num_classes=200): - return MobileNet_base(get_builder(), num_classes=num_classes) +def tinyMobileNetV2(): + return MobileNet_base(get_builder(), num_classes=200) ''' class BlockNormal(nn.Module): diff --git a/models/resnet_cifar.py b/models/resnet_cifar.py index 5b4f763e..2408f993 100644 --- a/models/resnet_cifar.py +++ b/models/resnet_cifar.py @@ -85,6 +85,21 @@ def __init__(self, builder, block, num_blocks): self.fc = nn.Conv2d(512 * block.expansion, 10, 1) else: self.fc = builder.conv1x1(512 * block.expansion, 10) + + self.prunable_layer_names, self.prunable_biases = self.get_prunable_param_names() + + def get_prunable_param_names(model): + prunable_weights = [name + '.weight' for name, module in model.named_modules() if + isinstance(module, torch.nn.modules.conv.Conv2d) or + isinstance(module, torch.nn.modules.linear.Linear)] + if parser_args.bias: + prunable_biases = [name + '.bias' for name, module in model.named_modules() if + isinstance(module, torch.nn.modules.conv.Conv2d) or + isinstance(module, torch.nn.modules.linear.Linear)] + else: + prunable_biases = [""] + + return prunable_weights, prunable_biases def _make_layer(self, block, planes, num_blocks, stride): strides = [stride] + [1] * (num_blocks - 1) diff --git a/tiny_mobile_exec.sh b/tiny_mobile_exec.sh new file mode 100755 index 00000000..c0c2023a --- /dev/null +++ b/tiny_mobile_exec.sh @@ -0,0 +1,27 @@ + +# target sparsity 0.5 + +# python main.py \ +# --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml \ +# --subfolder tiny_mobile_target_sparsity_50 \ +# --gpu 0 & + +# python main.py \ +# --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml \ +# --subfolder tiny_mobile_target_sparsity_5 \ +# --gpu 1 & + +python main.py \ +--config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml \ +--subfolder tiny_mobile_target_sparsity_2_5 \ +--gpu 0 & + +python main.py \ +--config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml \ +--subfolder tiny_mobile_target_sparsity_1_4 \ +--gpu 1 & + +# python main.py \ +# --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_tmp.yml \ +# --subfolder cifar_mobile_target_sparsity_1_4 \ +# --gpu 1 \ No newline at end of file diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index c32224f4..c9e39835 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -1,12 +1,119 @@ +# TinyImageNet, MobilenetV2 + +# Weight training (WT) +#python main.py --config configs/training/mobilenetV2/tiny_adam.yml #> log_tiny_mobile_wt_adam_0001_multi 2>&1 +#python main.py --config configs/training/mobilenetV2/tiny_sgd.yml > log_tiny_mobile_wt_sgd_01_multi 2>&1 + +# Renda +#:< "$subfolder" 2>&1 +#BLOCK + +# IMP +:< "$subfolder" 2>&1 +BLOCK + +# smart ratio (SR) +####### go to SR after getting the best result for WT +:< "$subfolder" 2>&1 +#python main.py --config configs/sr/tiny_sr_mobilenet.yml --gpu $gpu --smart_ratio 0.8 --subfolder "$subfolder" > "$subfolder" 2>&1 +BLOCK + +# Gem-Miner (GM) +:< "$subfolder" 2>&1 +BLOCK + + + +# Gem-Miner (GM) - Sanity checks! +:< "$subfolder" 2>&1 + +gpu=0 +subfolder="tiny_mobile_gm_sp_3_6_sanity" +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml \ + --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 + +gpu=1 +subfolder="tiny_mobile_gm_sp_1_4_sanity" +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml \ + --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 + +BLOCK + +gpu=3 +subfolder="tiny_mobile_gm_sp_20_invert" +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml \ + --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 + +subfolder="tiny_mobile_gm_sp_3_6_invert" +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml \ + --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 + +subfolder="tiny_mobile_gm_sp_1_4_invert" +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml \ + --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 + +#:< "$subfolder" 2>&1 +BLOCK + +:< "$subfolder" 2>&1 +BLOCK + +:< "$subfolder" 2>&1 +BLOCK + + + +#python main.py --config configs/hypercube/resnet18/resnet18_sparsity_1_4_adam_5lam6.yml # resnet18, check code + + + + # TinyImageNet, ResNet-50 # Weight training #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_001_multi.yml > log_tiny_res50_wt_adam_001_multi 2>&1 # this is current best #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_001_cosine.yml > log_tiny_res50_wt_adam_001_cosine 2>&1 # this is current best #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_0001_cosine.yml > log_tiny_res50_wt_adam_0001_cosine 2>&1 # this is current best -python main.py --config configs/training/resnet50/tiny_resnet50_training_sgd_multi.yml > log_tiny_res50_wt_sgd_multi 2>&1 # this is current best +#python main.py --config configs/training/resnet50/tiny_resnet50_training_sgd_multi.yml #> log_tiny_res50_wt_sgd_multi 2>&1 # this is current best diff --git a/utils/net_utils.py b/utils/net_utils.py index 46462981..63078413 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -26,7 +26,7 @@ def get_layers(arch='Conv4', model=None): model.convs[5], model.convs[7]] linear_layers = [model.linear[0], model.linear[2], model.linear[4]] - elif arch == 'MobileNetV2': + elif arch in ['MobileNetV2', 'tinyMobileNetV2']: conv_layers = [model.conv1] for i in range(len(model.layers)): conv_layers.append(model.layers[i].conv1)