FFCV imagenet serial

args_helper.py

@@ -282,8 +282,6 @@ def parse_arguments(self, jupyter_mode=False):
             default=1.0,
             help="portion of additional width compared with original width"
         )
-
-
         parser.add_argument(
             "--hidden-size",
             type=int,
@@ -899,28 +897,31 @@ def parse_arguments(self, jupyter_mode=False):
             default=False,
             help="Only run sanity checks on the files in specific directory or subdirectories"
         )
-
         parser.add_argument(
             "--invert-sanity-check",
             action="store_true",
             default=False,
             help="Enable this to run the inverted sanity check (for HC)"
         )
-
         parser.add_argument(
             "--sanity-folder",
             default=None,
             type=str,
             metavar="PATH",
             help="directory(s) to access for only sanity check",
         )
-
         parser.add_argument(
             "--sr-version",
             default=1,
             type=int,
             help="smart ratio version number (1, 2, ...)",
         )
+        parser.add_argument(
+            "--only-finetune",
+            action="store_true",
+            default=False,
+            help="Enable this to skip pruning and jump to finetune. Typically paired with --resume"
+        )
         parser.add_argument(
             "--use-full-data",
             action="store_true",

configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml

@@ -0,0 +1,66 @@
+subfolder: ffcv_imagenet_resnet18
+# trial_num: 1
+#lam_finetune_loss: 1
+#num_step_finetune: 5
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: ResNet18
+
+# ===== Dataset ===== #
+dataset: FfcvImageNet
+name: resnet18_ffcv
+data: /workspace/ffcv-imagenet/data/
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 50
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+mixed_precision: True
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 10
+# decide if you want to "unflag"
+unflag_before_finetune: False
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.000001 # 1e-4 #0.00005 # 5e-5
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/hypercube/resnet20/sreeniva_resnet20_1_4_best.yml

@@ -0,0 +1,64 @@
+# subfolder: sreeniva_hc_1_44_best
+# trial_num: 1
+#lam_finetune_loss: 1
+#num_step_finetune: 5
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: resnet20
+
+# ===== Dataset ===== #
+dataset: CIFAR10
+name: resnet20_quantized_iter_hc
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 150
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 1.44
+# decide if you want to "unflag"
+unflag_before_finetune: False
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.0001 # 1e-4 #0.00005 # 5e-5
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 2
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: True
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10_finetune_only.yml

@@ -0,0 +1,68 @@
+subfolder: ffcv_imagenet_resnet50_finetune_only
+# trial_num: 1
+#lam_finetune_loss: 1
+#num_step_finetune: 5
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: ResNet50
+
+# ===== Dataset ===== #
+dataset: FfcvImageNet
+name: resnet18_ffcv
+data: /workspace/ffcv-imagenet/data/
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.4 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 88
+wd: 0.0 
+momentum: 0.9
+batch_size: 128
+mixed_precision: True
+only_finetune: True
+resume: results/resnet50_ffcv_bad_finetune/model_before_finetune.pth
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 10
+# decide if you want to "unflag"
+unflag_before_finetune: False
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.000001 # 1e-4 #0.00005 # 5e-5
+
+# ===== Hardware setup ===== #
+workers: 6
+gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True

configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml

@@ -0,0 +1,67 @@
+subfolder: ffcv_imagenet_resnet50_sp20
+# trial_num: 1
+#lam_finetune_loss: 1
+#num_step_finetune: 5
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: ResNet50
+
+# ===== Dataset ===== #
+dataset: FfcvImageNet
+name: resnet50_ffcv_imagenet
+data: /workspace/ffcv-imagenet/data/
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.1 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.01
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 88
+wd: 0.0 
+momentum: 0.9
+batch_size: 256
+mixed_precision: True
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 20
+# decide if you want to "unflag"
+unflag_before_finetune: False
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+use_full_data: True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.000001 # 1e-4 #0.00005 # 5e-5
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 1
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: False

configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5.yml

@@ -0,0 +1,66 @@
+subfolder: ffcv_imagenet_resnet50
+# trial_num: 1
+#lam_finetune_loss: 1
+#num_step_finetune: 5
+
+# Hypercube optimization
+algo: 'hc_iter'
+iter_period: 5
+
+# Architecture
+arch: ResNet50
+
+# ===== Dataset ===== #
+dataset: FfcvImageNet
+name: resnet18_ffcv
+data: /workspace/ffcv-imagenet/data/
+
+# ===== Learning Rate Policy ======== #
+optimizer: sgd
+lr: 0.4 #0.01
+lr_policy: cosine_lr #constant_lr #multistep_lr
+fine_tune_lr: 0.001
+fine_tune_lr_policy: multistep_lr
+
+# ===== Network training config ===== #
+epochs: 88
+wd: 0.0 
+momentum: 0.9
+batch_size: 256
+mixed_precision: True
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+# enter target sparsity here
+target_sparsity: 5
+# decide if you want to "unflag"
+unflag_before_finetune: False
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False #True
+
+# ===== Rounding ===== #
+round: naive 
+noise: True
+noise_ratio: 0 
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+lmbda: 0.0000001 # 1e-4 #0.00005 # 5e-5
+
+# ===== Hardware setup ===== #
+workers: 8
+gpu: 0
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True