From 4d88d5872f149f40999d48c630ef40acdd5a63f4 Mon Sep 17 00:00:00 2001 From: Leiay Date: Sun, 27 Feb 2022 22:02:04 +0000 Subject: [PATCH 01/18] add mobilenet tiny exec file --- .../tinyImageNet/mobilenetV2/sparsity_1_4.yml | 61 +++++++++++++++++++ .../tinyImageNet/mobilenetV2/sparsity_2_5.yml | 61 +++++++++++++++++++ .../tinyImageNet/mobilenetV2/sparsity_5.yml | 61 +++++++++++++++++++ .../tinyImageNet/mobilenetV2/sparsity_50.yml | 61 +++++++++++++++++++ models/__init__.py | 2 + models/mobilenet.py | 3 + tiny_mobile_exec.sh | 23 +++++++ utils/net_utils.py | 2 +- 8 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml create mode 100755 tiny_mobile_exec.sh diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml new file mode 100644 index 00000000..2b65718f --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml @@ -0,0 +1,61 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml new file mode 100644 index 00000000..a5c49ded --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml @@ -0,0 +1,61 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_2_5 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 2.5 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml new file mode 100644 index 00000000..3c5327fc --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml @@ -0,0 +1,61 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_5 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml new file mode 100644 index 00000000..541cfc6e --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml @@ -0,0 +1,61 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_50 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 50.0 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/models/__init__.py b/models/__init__.py index 216a0bfb..f6160cc8 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -8,6 +8,7 @@ #### TODO: delete below ones (merge with above code) from models.resnet_cifar import cResNet18, cResNet50 from models.resnet_tiny import TinyResNet18 +from models.mobilenet import tinyMobileNetV2 __all__ = [ "tinyvgg16", @@ -20,6 +21,7 @@ "resnet20", "resnet32", "resnet32_double", + "tinyMobileNetV2", "MobileNetV2", "FC", "Conv2", diff --git a/models/mobilenet.py b/models/mobilenet.py index 54453d47..06e40e5e 100644 --- a/models/mobilenet.py +++ b/models/mobilenet.py @@ -108,6 +108,9 @@ def forward(self, x): def MobileNetV2(): return MobileNet_base(get_builder()) +def tinyMobileNetV2(): + return MobileNet_base(get_builder(), num_classes=200) + ''' class BlockNormal(nn.Module): def __init__(self, in_planes, out_planes, expansion, stride): diff --git a/tiny_mobile_exec.sh b/tiny_mobile_exec.sh new file mode 100755 index 00000000..6dca4e5d --- /dev/null +++ b/tiny_mobile_exec.sh @@ -0,0 +1,23 @@ + +# target sparsity 0.5 + +python main.py \ +--config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml \ +--subfolder tiny_mobile_target_sparsity_50 \ +--gpu 0 & + +python main.py \ +--config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml \ +--subfolder tiny_mobile_target_sparsity_5 \ +--gpu 1 & + +python main.py \ +--config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml \ +--subfolder tiny_mobile_target_sparsity_2_5 \ +--gpu 3 & + +python main.py \ +--config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml \ +--subfolder tiny_mobile_target_sparsity_1_4 \ +--gpu 3 & + diff --git a/utils/net_utils.py b/utils/net_utils.py index 46462981..63078413 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -26,7 +26,7 @@ def get_layers(arch='Conv4', model=None): model.convs[5], model.convs[7]] linear_layers = [model.linear[0], model.linear[2], model.linear[4]] - elif arch == 'MobileNetV2': + elif arch in ['MobileNetV2', 'tinyMobileNetV2']: conv_layers = [model.conv1] for i in range(len(model.layers)): conv_layers.append(model.layers[i].conv1) From b7679b4954cbf25efac47fec4f06d37112a02eb7 Mon Sep 17 00:00:00 2001 From: Leiay Date: Mon, 28 Feb 2022 08:03:47 +0000 Subject: [PATCH 02/18] modify imp setting for cresnet18 --- models/resnet_cifar.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/models/resnet_cifar.py b/models/resnet_cifar.py index 5b4f763e..2408f993 100644 --- a/models/resnet_cifar.py +++ b/models/resnet_cifar.py @@ -85,6 +85,21 @@ def __init__(self, builder, block, num_blocks): self.fc = nn.Conv2d(512 * block.expansion, 10, 1) else: self.fc = builder.conv1x1(512 * block.expansion, 10) + + self.prunable_layer_names, self.prunable_biases = self.get_prunable_param_names() + + def get_prunable_param_names(model): + prunable_weights = [name + '.weight' for name, module in model.named_modules() if + isinstance(module, torch.nn.modules.conv.Conv2d) or + isinstance(module, torch.nn.modules.linear.Linear)] + if parser_args.bias: + prunable_biases = [name + '.bias' for name, module in model.named_modules() if + isinstance(module, torch.nn.modules.conv.Conv2d) or + isinstance(module, torch.nn.modules.linear.Linear)] + else: + prunable_biases = [""] + + return prunable_weights, prunable_biases def _make_layer(self, block, planes, num_blocks, stride): strides = [stride] + [1] * (num_blocks - 1) From b34e87a27d959706df02aca7983fdd56d21218d6 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 1 Mar 2022 23:08:28 +0000 Subject: [PATCH 03/18] finetune the hyperparameter --- .../tinyImageNet/mobilenetV2/sparsity_1_4.yml | 9 +++---- .../tinyImageNet/mobilenetV2/sparsity_2_5.yml | 9 +++---- .../tinyImageNet/mobilenetV2/sparsity_5.yml | 5 ++-- main.py | 3 ++- main_utils.py | 9 +++---- tiny_mobile_exec.sh | 24 +++++++++++-------- 6 files changed, 34 insertions(+), 25 deletions(-) diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml index 2b65718f..6b1cafd3 100644 --- a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml @@ -13,8 +13,9 @@ dataset: TinyImageNet name: mobilenetV2_hc # ===== Learning Rate Policy ======== # -optimizer: sgd -lr: 0.1 #0.01 +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 lr_policy: cosine_lr #constant_lr #multistep_lr fine_tune_lr: 0.1 fine_tune_lr_policy: multistep_lr @@ -32,7 +33,7 @@ freeze_weights: True prune_type: BottomK # enter target sparsity here target_sparsity: 1.4 -#unflag_before_finetune: True +unflag_before_finetune: True init: signed_constant score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True @@ -48,7 +49,7 @@ quantize_threshold: 0.5 # ===== Regularization ===== # regularization: L2 -lmbda: 0 +lmbda: 0.000008 # ===== Hardware setup ===== # workers: 4 diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml index a5c49ded..d2cefdca 100644 --- a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml @@ -13,8 +13,9 @@ dataset: TinyImageNet name: mobilenetV2_hc # ===== Learning Rate Policy ======== # -optimizer: sgd -lr: 0.1 #0.01 +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 lr_policy: cosine_lr #constant_lr #multistep_lr fine_tune_lr: 0.1 fine_tune_lr_policy: multistep_lr @@ -32,7 +33,7 @@ freeze_weights: True prune_type: BottomK # enter target sparsity here target_sparsity: 2.5 -#unflag_before_finetune: True +unflag_before_finetune: True init: signed_constant score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True @@ -48,7 +49,7 @@ quantize_threshold: 0.5 # ===== Regularization ===== # regularization: L2 -lmbda: 0 +lmbda: 0.000005 # ===== Hardware setup ===== # workers: 4 diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml index 3c5327fc..7b54d82c 100644 --- a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml @@ -15,6 +15,7 @@ name: mobilenetV2_hc # ===== Learning Rate Policy ======== # optimizer: sgd lr: 0.1 #0.01 +wd: 0.0 lr_policy: cosine_lr #constant_lr #multistep_lr fine_tune_lr: 0.1 fine_tune_lr_policy: multistep_lr @@ -32,7 +33,7 @@ freeze_weights: True prune_type: BottomK # enter target sparsity here target_sparsity: 5.0 -#unflag_before_finetune: True +unflag_before_finetune: True init: signed_constant score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True @@ -48,7 +49,7 @@ quantize_threshold: 0.5 # ===== Regularization ===== # regularization: L2 -lmbda: 0 +lmbda: 0.00000001 # ===== Hardware setup ===== # workers: 4 diff --git a/main.py b/main.py index 4e6512a5..c4ad3a81 100644 --- a/main.py +++ b/main.py @@ -257,7 +257,8 @@ def main_worker(gpu, ngpus_per_node): parser_args.prune_rate = sum_pr / count writer.add_scalar("pr/average", parser_args.prune_rate, epoch) - writer.add_scalar("test/lr", cur_lr, epoch) + if writer is not None: + writer.add_scalar("test/lr", cur_lr, epoch) end_epoch = time.time() if parser_args.algo in ['hc', 'hc_iter']: diff --git a/main_utils.py b/main_utils.py index db0652ab..a0206794 100644 --- a/main_utils.py +++ b/main_utils.py @@ -11,7 +11,7 @@ import random import time import pandas as pd -from torch.utils.tensorboard import SummaryWriter +# from torch.utils.tensorboard import SummaryWriter import torch import torch.nn as nn import torch.nn.parallel @@ -340,7 +340,8 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b progress_overall.write_to_tensorboard( writer, prefix="diagnostics", global_step=epoch ) - writer.add_scalar("test/lr", cur_lr, epoch) + if writer is not None: + writer.add_scalar("test/lr", cur_lr, epoch) end_epoch = time.time() results_df = pd.DataFrame({'epoch': epoch_list, 'test_acc_before_rounding': test_acc_before_round_list, 'test_acc': test_acc_list, @@ -400,8 +401,8 @@ def get_settings(parser_args): run_base_dir, ckpt_base_dir, log_base_dir = get_directories(parser_args) parser_args.ckpt_base_dir = ckpt_base_dir - writer = SummaryWriter(log_dir=log_base_dir) - # writer = None + # writer = SummaryWriter(log_dir=log_base_dir) + writer = None epoch_time = AverageMeter("epoch_time", ":.4f", write_avg=False) validation_time = AverageMeter("validation_time", ":.4f", write_avg=False) train_time = AverageMeter("train_time", ":.4f", write_avg=False) diff --git a/tiny_mobile_exec.sh b/tiny_mobile_exec.sh index 6dca4e5d..c0c2023a 100755 --- a/tiny_mobile_exec.sh +++ b/tiny_mobile_exec.sh @@ -1,23 +1,27 @@ # target sparsity 0.5 -python main.py \ ---config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml \ ---subfolder tiny_mobile_target_sparsity_50 \ ---gpu 0 & +# python main.py \ +# --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_50.yml \ +# --subfolder tiny_mobile_target_sparsity_50 \ +# --gpu 0 & -python main.py \ ---config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml \ ---subfolder tiny_mobile_target_sparsity_5 \ ---gpu 1 & +# python main.py \ +# --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5.yml \ +# --subfolder tiny_mobile_target_sparsity_5 \ +# --gpu 1 & python main.py \ --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_2_5.yml \ --subfolder tiny_mobile_target_sparsity_2_5 \ ---gpu 3 & +--gpu 0 & python main.py \ --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml \ --subfolder tiny_mobile_target_sparsity_1_4 \ ---gpu 3 & +--gpu 1 & +# python main.py \ +# --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_tmp.yml \ +# --subfolder cifar_mobile_target_sparsity_1_4 \ +# --gpu 1 \ No newline at end of file From ef98645eb8a5506eb1070e8394d72a01901d8763 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Mar 2022 10:43:54 +0000 Subject: [PATCH 04/18] run mobilenet tinyimagenet GM --- .../mobilenetV2/sparsity_5_GD.yml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml new file mode 100644 index 00000000..69c42963 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml @@ -0,0 +1,62 @@ +subfolder: tiny_mobilenetV2_hc_sparsity_5 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From f79502faf0be190fb286fdb21be1c49b02ae6e99 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 13 Mar 2022 23:17:36 +0000 Subject: [PATCH 05/18] tinyimagenet, mobielnetV2, sgd, weight training --- configs/training/mobilenetV2/tiny_sgd.yml | 40 +++++++++++++++++++++++ tinyimagenet_exec_GD.sh | 36 +++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 configs/training/mobilenetV2/tiny_sgd.yml diff --git a/configs/training/mobilenetV2/tiny_sgd.yml b/configs/training/mobilenetV2/tiny_sgd.yml new file mode 100644 index 00000000..90b42d84 --- /dev/null +++ b/configs/training/mobilenetV2/tiny_sgd.yml @@ -0,0 +1,40 @@ + +subfolder: tiny_mobile_weight_training_sgd +trial_num: 1 + + +# algorithm +algo: 'hc_iter' + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_mobile_training + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: multistep_lr #cosine_lr #constant_lr + +# ===== Network training config ===== # +epochs: 300 +wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +weight_training: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_rate: -1 +init: kaiming_normal +scale_fan: True + +# ===== Hardware setup ===== # +workers: 4 +gpu: 1 + diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index c32224f4..295df638 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -1,12 +1,46 @@ +# TinyImageNet, MobilenetV2 + + +# Weight training (WT) +#python main.py --config configs/training/mobilenetV2/tiny_adam.yml #> log_tiny_mobile_wt_adam_0001_multi 2>&1 +python main.py --config configs/training/mobilenetV2/tiny_sgd.yml #> log_tiny_mobile_wt_sgd_01_multi 2>&1 + +# smart ratio (SR) +####### go to SR after getting the best result for WT +#python main.py --config configs/sr/tiny_sr_mobilenet.yml --smart_ratio 0.95 #> log_tiny_mobile_sr 2>&1 + + +# Gem-Miner (GM) +:< "$subfolder" 2>&1 + +BLOCK +#python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml > tiny_mobile_1_4_lam_8e5 2>&1 +#python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml > tiny_mobile_5 2>&1 + + + + +#python main.py --config configs/hypercube/resnet18/resnet18_sparsity_1_4_adam_5lam6.yml # resnet18, check code + + + + # TinyImageNet, ResNet-50 # Weight training #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_001_multi.yml > log_tiny_res50_wt_adam_001_multi 2>&1 # this is current best #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_001_cosine.yml > log_tiny_res50_wt_adam_001_cosine 2>&1 # this is current best #python main.py --config configs/training/resnet50/tiny_resnet50_training_adam_0001_cosine.yml > log_tiny_res50_wt_adam_0001_cosine 2>&1 # this is current best -python main.py --config configs/training/resnet50/tiny_resnet50_training_sgd_multi.yml > log_tiny_res50_wt_sgd_multi 2>&1 # this is current best +#python main.py --config configs/training/resnet50/tiny_resnet50_training_sgd_multi.yml #> log_tiny_res50_wt_sgd_multi 2>&1 # this is current best From a2782121a06f4744c1d2b64361fbb43dd2227b5e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 15 Mar 2022 12:07:18 +0000 Subject: [PATCH 06/18] mobilenetV2, tinyimgnet, weight training --- configs/training/mobilenetV2/tiny_sgd.yml | 4 ++++ tinyimagenet_exec_GD.sh | 15 +++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/configs/training/mobilenetV2/tiny_sgd.yml b/configs/training/mobilenetV2/tiny_sgd.yml index 90b42d84..0144d997 100644 --- a/configs/training/mobilenetV2/tiny_sgd.yml +++ b/configs/training/mobilenetV2/tiny_sgd.yml @@ -38,3 +38,7 @@ scale_fan: True workers: 4 gpu: 1 +# ==== sanity check ==== # +skip_fine_tune: True +skip_sanity_checks: True + diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 295df638..026c0c2c 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -2,10 +2,9 @@ # TinyImageNet, MobilenetV2 - # Weight training (WT) #python main.py --config configs/training/mobilenetV2/tiny_adam.yml #> log_tiny_mobile_wt_adam_0001_multi 2>&1 -python main.py --config configs/training/mobilenetV2/tiny_sgd.yml #> log_tiny_mobile_wt_sgd_01_multi 2>&1 +#python main.py --config configs/training/mobilenetV2/tiny_sgd.yml > log_tiny_mobile_wt_sgd_01_multi 2>&1 # smart ratio (SR) ####### go to SR after getting the best result for WT @@ -13,16 +12,16 @@ python main.py --config configs/training/mobilenetV2/tiny_sgd.yml #> log_tiny_mo # Gem-Miner (GM) -:< "$subfolder" 2>&1 -BLOCK +#BLOCK #python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml > tiny_mobile_1_4_lam_8e5 2>&1 #python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml > tiny_mobile_5 2>&1 From 67b130f624e65a8a65ce4e3b681b9de99f2b11bd Mon Sep 17 00:00:00 2001 From: root Date: Thu, 17 Mar 2022 02:25:16 +0000 Subject: [PATCH 07/18] hypercube mobilenet tinyimgnet yml file --- .../tinyImageNet/mobilenetV2/sgd.yml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sgd.yml diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sgd.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sgd.yml new file mode 100644 index 00000000..ac3abbe3 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sgd.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +#target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +#lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From 66f140ae3948008b0f4ebf1f2667ebfb1d12322e Mon Sep 17 00:00:00 2001 From: root Date: Thu, 17 Mar 2022 02:34:02 +0000 Subject: [PATCH 08/18] run GM/SR for tinyimgnet, mobilenetv2 --- configs/sr/tiny_sr_mobilenet.yml | 46 ++++++++++++++++++++++++++++++++ tinyimagenet_exec_GD.sh | 33 ++++++++++++++++++----- 2 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 configs/sr/tiny_sr_mobilenet.yml diff --git a/configs/sr/tiny_sr_mobilenet.yml b/configs/sr/tiny_sr_mobilenet.yml new file mode 100644 index 00000000..3e885a56 --- /dev/null +++ b/configs/sr/tiny_sr_mobilenet.yml @@ -0,0 +1,46 @@ +# subfolder: sr +algo: 'hc_iter' +random_subnet: True +# smart_ratio: 0.98 # 2% sparsity + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: tiny_mobilenetV2_sr + +# ===== Learning Rate Policy ======== # +optimizer: sgd +# doesn't matter +# lr: 0.1 +# lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr # cosine_lr +fine_tune_wd: 0.00004 + +# ===== Network training config ===== # +epochs: 300 #150 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True + +init: kaiming_normal # signed_constant +mode: fan_in +nonlinearity: relu +# prune_rate: 0.5 +scale_fan: True + +# ===== Hardware setup ===== # +workers: 4 +# gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 026c0c2c..f3fdf024 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -8,24 +8,45 @@ # smart ratio (SR) ####### go to SR after getting the best result for WT -#python main.py --config configs/sr/tiny_sr_mobilenet.yml --smart_ratio 0.95 #> log_tiny_mobile_sr 2>&1 +gpu=1 +python main.py --config configs/sr/tiny_sr_mobilenet.yml --gpu $gpu --smart_ratio 0.95 #> log_tiny_mobile_sr 2>&1 # Gem-Miner (GM) #:< "$subfolder" 2>&1 #BLOCK -#python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4.yml > tiny_mobile_1_4_lam_8e5 2>&1 -#python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_5_GD.yml > tiny_mobile_5 2>&1 +# Edge-Popup (EP) +:< "$subfolder" 2>&1 +BLOCK + +:< "$subfolder" 2>&1 +BLOCK + +:< "$subfolder" 2>&1 +BLOCK + #python main.py --config configs/hypercube/resnet18/resnet18_sparsity_1_4_adam_5lam6.yml # resnet18, check code From 004f246d7ccde1f8cfda046c5e50b6059e362245 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 21 Mar 2022 00:39:28 +0000 Subject: [PATCH 09/18] add arguments for handling fine-tune-lr-gamma, let fine-tune-gamma applied to the get_scheduler() --- args_helper.py | 6 ++++++ main.py | 2 +- main_utils.py | 2 +- tinyimagenet_exec_GD.sh | 20 +++++++++++--------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/args_helper.py b/args_helper.py index b03fbb95..72ba940b 100644 --- a/args_helper.py +++ b/args_helper.py @@ -742,6 +742,12 @@ def parse_arguments(self, jupyter_mode=False): metavar="LR", help="Learning rate for fine-tuning weights" ) + parser.add_argument( + "--fine-tune-lr-gamma", + type=float, + default=0.1, + help="Learning rate decay for multi-step" + ) parser.add_argument( "--fine-tune-wd", type=float, diff --git a/main.py b/main.py index c4ad3a81..2fe01432 100644 --- a/main.py +++ b/main.py @@ -64,7 +64,7 @@ def main_worker(gpu, ngpus_per_node): model2 = None optimizer = get_optimizer(parser_args, model) data = get_dataset(parser_args) - scheduler = get_scheduler(optimizer, parser_args.lr_policy) + scheduler = get_scheduler(optimizer, policy=parser_args.lr_policy, gamma=parser_args.gamma) #lr_policy = get_policy(parser_args.lr_policy)(optimizer, parser_args) if parser_args.label_smoothing is None: criterion = nn.CrossEntropyLoss().cuda() diff --git a/main_utils.py b/main_utils.py index a0206794..e403424e 100644 --- a/main_utils.py +++ b/main_utils.py @@ -272,7 +272,7 @@ def finetune(model, parser_args, data, criterion, old_epoch_list, old_test_acc_b parser_args) optimizer = get_optimizer(parser_args, model, finetune_flag=True) - scheduler = get_scheduler(optimizer, policy=parser_args.fine_tune_lr_policy) + scheduler = get_scheduler(optimizer, policy=parser_args.fine_tune_lr_policy, gamma=parser_args.fine_tune_lr_gamma) ''' if parser_args.epochs == 150: scheduler = get_scheduler(optimizer, parser_args.fine_tune_lr_policy, milestones=[ diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index f3fdf024..5f9962bf 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -8,21 +8,23 @@ # smart ratio (SR) ####### go to SR after getting the best result for WT -gpu=1 -python main.py --config configs/sr/tiny_sr_mobilenet.yml --gpu $gpu --smart_ratio 0.95 #> log_tiny_mobile_sr 2>&1 - +#:< "$subfolder" 2>&1 +#python main.py --config configs/sr/tiny_sr_mobilenet.yml --gpu $gpu --smart_ratio 0.8 --subfolder "$subfolder" > "$subfolder" 2>&1 +#BLOCK # Gem-Miner (GM) -#:< "$subfolder" 2>&1 - -#BLOCK +BLOCK # Edge-Popup (EP) From b12e393c6f46223c7943badeb07ff9dbda9f28b9 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 21 Mar 2022 02:33:32 +0000 Subject: [PATCH 10/18] run tiny, mobile for imp/renda/sr/ep/hc --- .../tinyImageNet/mobilenetV2/sparsity_1_4.yml | 43 +++++++++++++ .../tinyImageNet/mobilenetV2/sparsity_20.yml | 43 +++++++++++++ .../tinyImageNet/mobilenetV2/sparsity_5.yml | 43 +++++++++++++ .../mobilenetV2/sparsity_5_adam.yml | 43 +++++++++++++ .../tinyImageNet/mobilenetV2/adam.yml | 62 +++++++++++++++++++ configs/imp/tiny_mobilenet.yml | 36 +++++++++++ configs/sr/tiny_sr.yml | 6 +- tinyimagenet_exec_GD.sh | 18 +++++- 8 files changed, 289 insertions(+), 5 deletions(-) create mode 100644 configs/ep/tinyImageNet/mobilenetV2/sparsity_1_4.yml create mode 100644 configs/ep/tinyImageNet/mobilenetV2/sparsity_20.yml create mode 100644 configs/ep/tinyImageNet/mobilenetV2/sparsity_5.yml create mode 100644 configs/ep/tinyImageNet/mobilenetV2/sparsity_5_adam.yml create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/adam.yml create mode 100644 configs/imp/tiny_mobilenet.yml diff --git a/configs/ep/tinyImageNet/mobilenetV2/sparsity_1_4.yml b/configs/ep/tinyImageNet/mobilenetV2/sparsity_1_4.yml new file mode 100644 index 00000000..40ff568b --- /dev/null +++ b/configs/ep/tinyImageNet/mobilenetV2/sparsity_1_4.yml @@ -0,0 +1,43 @@ +#subfolder: EP_sparsity_1_4 +algo: 'ep' +override_prune_rate: True + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobileV2_tiny + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 #256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +prune_rate: 0.014 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/ep/tinyImageNet/mobilenetV2/sparsity_20.yml b/configs/ep/tinyImageNet/mobilenetV2/sparsity_20.yml new file mode 100644 index 00000000..3d32df4c --- /dev/null +++ b/configs/ep/tinyImageNet/mobilenetV2/sparsity_20.yml @@ -0,0 +1,43 @@ +#subfolder: EP_sparsity_20 +algo: 'ep' +override_prune_rate: True + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobileV2_tiny + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 #256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +prune_rate: 0.20 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/ep/tinyImageNet/mobilenetV2/sparsity_5.yml b/configs/ep/tinyImageNet/mobilenetV2/sparsity_5.yml new file mode 100644 index 00000000..0a8013c1 --- /dev/null +++ b/configs/ep/tinyImageNet/mobilenetV2/sparsity_5.yml @@ -0,0 +1,43 @@ +#subfolder: EP_sparsity_5 +algo: 'ep' +override_prune_rate: True + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobileV2_tiny + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 #256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +prune_rate: 0.05 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/ep/tinyImageNet/mobilenetV2/sparsity_5_adam.yml b/configs/ep/tinyImageNet/mobilenetV2/sparsity_5_adam.yml new file mode 100644 index 00000000..6ec42381 --- /dev/null +++ b/configs/ep/tinyImageNet/mobilenetV2/sparsity_5_adam.yml @@ -0,0 +1,43 @@ +#subfolder: EP_sparsity_5 +algo: 'ep' +override_prune_rate: True + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobileV2_tiny + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 +lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 #256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +prune_rate: 0.05 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/adam.yml b/configs/hypercube/tinyImageNet/mobilenetV2/adam.yml new file mode 100644 index 00000000..d712c7f0 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/adam.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +#target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +#lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/imp/tiny_mobilenet.yml b/configs/imp/tiny_mobilenet.yml new file mode 100644 index 00000000..b176907b --- /dev/null +++ b/configs/imp/tiny_mobilenet.yml @@ -0,0 +1,36 @@ +# IMP algorithm +algo: 'imp' +seed: 42 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: multistep_lr +lr_gamma: 0.1 + +# ===== Network training config ===== # +# epochs: 300 +wd: 0.00004 +momentum: 0.9 +batch_size: 128 +bias: False + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +prune_rate: 0.2 +init: kaiming_normal +iter_period: 300 # 5 +imp_rewind_iter: 1000 + + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 2 + diff --git a/configs/sr/tiny_sr.yml b/configs/sr/tiny_sr.yml index 3cfd00a4..c0901d0b 100644 --- a/configs/sr/tiny_sr.yml +++ b/configs/sr/tiny_sr.yml @@ -20,9 +20,9 @@ fine_tune_lr_policy: multistep_lr # cosine_lr fine_tune_wd: 0.0001 # ===== Network training config ===== # -epochs: 200 #150 +epochs: 300 #150 momentum: 0.9 -batch_size: 256 +batch_size: 128 bias: False # ===== Sparsity =========== # @@ -40,7 +40,7 @@ workers: 4 # gpu: 3 # ===== Checkpointing ===== # -checkpoint_at_prune: True +checkpoint_at_prune: False # ==== sanity check ==== # skip_sanity_checks: True diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 5f9962bf..4159b235 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -6,14 +6,28 @@ #python main.py --config configs/training/mobilenetV2/tiny_adam.yml #> log_tiny_mobile_wt_adam_0001_multi 2>&1 #python main.py --config configs/training/mobilenetV2/tiny_sgd.yml > log_tiny_mobile_wt_sgd_01_multi 2>&1 +# IMP +#:< "$subfolder" 2>&1 +#BLOCK + +# IMP +:< "$subfolder" 2>&1 +BLOCK + # smart ratio (SR) ####### go to SR after getting the best result for WT -#:< "$subfolder" 2>&1 #python main.py --config configs/sr/tiny_sr_mobilenet.yml --gpu $gpu --smart_ratio 0.8 --subfolder "$subfolder" > "$subfolder" 2>&1 -#BLOCK +BLOCK # Gem-Miner (GM) :< Date: Tue, 29 Mar 2022 04:31:32 +0000 Subject: [PATCH 11/18] run sanity check for 1.4% GM, tiny, mobile --- .../mobilenetV2/sparsity_1_4_sgd_w_sanity.yml | 62 +++++++++++++++++++ tinyimagenet_exec_GD.sh | 28 ++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml new file mode 100644 index 00000000..1c6d2d27 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000009 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 4159b235..72c063b5 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -7,11 +7,11 @@ #python main.py --config configs/training/mobilenetV2/tiny_sgd.yml > log_tiny_mobile_wt_sgd_01_multi 2>&1 # IMP -#:< "$subfolder" 2>&1 -#BLOCK +BLOCK # IMP :< "$subfolder" 2>&1 + +gpu=0 +subfolder="tiny_mobile_gm_sp_3_6_sanity" + +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml \ + --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 + +BLOCK + +gpu=0 +subfolder="tiny_mobile_gm_sp_1_4_sanity" + +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml \ + --gpu $gpu --subfolder "$subfolder" #> "$subfolder" 2>&1 + + # Edge-Popup (EP) :< Date: Tue, 29 Mar 2022 04:39:05 +0000 Subject: [PATCH 12/18] run sr for tiny, mobilenet --- configs/sr/tiny_sr_mobilenet.yml | 4 ++-- tinyimagenet_exec_GD.sh | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configs/sr/tiny_sr_mobilenet.yml b/configs/sr/tiny_sr_mobilenet.yml index 3e885a56..b68bfdad 100644 --- a/configs/sr/tiny_sr_mobilenet.yml +++ b/configs/sr/tiny_sr_mobilenet.yml @@ -16,8 +16,8 @@ optimizer: sgd # lr: 0.1 # lr_policy: cosine_lr fine_tune_lr: 0.1 -fine_tune_lr_policy: multistep_lr # cosine_lr -fine_tune_wd: 0.00004 +fine_tune_lr_policy: cosine_lr #multistep_lr # cosine_lr +fine_tune_wd: 0 #0.00004 # ===== Network training config ===== # epochs: 300 #150 diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 5f9962bf..b9fbb11c 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -8,23 +8,23 @@ # smart ratio (SR) ####### go to SR after getting the best result for WT -#:< "$subfolder" 2>&1 +subfolder="tiny_mobile_sr_20_cosien" +python main.py --config configs/sr/tiny_sr_mobilenet.yml --gpu $gpu --smart_ratio 0.8 --subfolder "$subfolder" > "$subfolder" 2>&1 #python main.py --config configs/sr/tiny_sr_mobilenet.yml --gpu $gpu --smart_ratio 0.8 --subfolder "$subfolder" > "$subfolder" 2>&1 -#BLOCK +BLOCK # Gem-Miner (GM) -:< "$subfolder" 2>&1 -BLOCK +#BLOCK # Edge-Popup (EP) From 17447b4548718c7f43930f11f2773349ced7286a Mon Sep 17 00:00:00 2001 From: root Date: Tue, 29 Mar 2022 04:44:49 +0000 Subject: [PATCH 13/18] sanity checks for tiny, mobilenet, gm --- tinyimagenet_exec_GD.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index fa0281ce..4b547558 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -30,7 +30,7 @@ python main.py --config configs/sr/tiny_sr_mobilenet.yml --gpu $gpu --smart_rati BLOCK # Gem-Miner (GM) -#:< "$subfolder" 2>&1 -#BLOCK +BLOCK @@ -46,23 +46,24 @@ python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sgd_unflag_F. :< "$subfolder" 2>&1 gpu=0 subfolder="tiny_mobile_gm_sp_3_6_sanity" - python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml \ --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 +gpu=1 +subfolder="tiny_mobile_gm_sp_1_4_sanity" +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml \ + --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 + BLOCK -gpu=0 -subfolder="tiny_mobile_gm_sp_1_4_sanity" -python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_sanity.yml \ - --gpu $gpu --subfolder "$subfolder" #> "$subfolder" 2>&1 + + # Edge-Popup (EP) From 8d1b80dea3d951d0941a4532acba37fda1f1a49f Mon Sep 17 00:00:00 2001 From: root Date: Tue, 29 Mar 2022 04:47:05 +0000 Subject: [PATCH 14/18] sanity check yml, tiny mobile --- .../mobilenetV2/sparsity_20_sgd_w_sanity.yml | 63 +++++++++++++++++++ .../sparsity_3_6_adam_w_sanity.yml | 62 ++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_sanity.yml create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_sanity.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_sanity.yml new file mode 100644 index 00000000..b480b3ed --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_sanity.yml @@ -0,0 +1,63 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000003 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False + diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml new file mode 100644 index 00000000..b7ff3cf7 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_sanity.yml @@ -0,0 +1,62 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: False From d3337444b1b8a27cca7da4170c31445aecfb83bb Mon Sep 17 00:00:00 2001 From: root Date: Tue, 29 Mar 2022 04:58:42 +0000 Subject: [PATCH 15/18] invert, tiny, mobile, gm --- .../mobilenetV2/sparsity_1_4_sgd_w_invert.yml | 63 ++++++++++++++++++ .../mobilenetV2/sparsity_20_sgd_w_invert.yml | 64 +++++++++++++++++++ .../sparsity_3_6_adam_w_invert.yml | 63 ++++++++++++++++++ tinyimagenet_exec_GD.sh | 14 ++++ 4 files changed, 204 insertions(+) create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml create mode 100644 configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml new file mode 100644 index 00000000..a58dae7b --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml @@ -0,0 +1,63 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 +invert_sanity_check: True + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000009 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml new file mode 100644 index 00000000..a7e4b9a8 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml @@ -0,0 +1,64 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 +invert_sanity_check: True + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000003 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True + diff --git a/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml new file mode 100644 index 00000000..1b67b5d2 --- /dev/null +++ b/configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml @@ -0,0 +1,63 @@ +#subfolder: tiny_mobilenetV2_hc_sparsity_1_4 +trial_num: 1 +invert_sanity_check: True + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: tinyMobileNetV2 + +# ===== Dataset ===== # +dataset: TinyImageNet +name: mobilenetV2_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.001 #0.01 +wd: 0.0 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 300 #300 +fine_tune_wd: 0.00004 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000008 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 4b547558..0ba13508 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -61,9 +61,23 @@ python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_ BLOCK +gpu=0 +subfolder="tiny_mobile_gm_sp_20_invert" +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml \ + --gpu $gpu --subfolder "$subfolder" #> "$subfolder" 2>&1 +:< "$subfolder" 2>&1 +gpu=1 +subfolder="tiny_mobile_gm_sp_1_4_invert" +python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml \ + --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 +BLOCK # Edge-Popup (EP) From c905c466ee7884dfb95deadbf510fce08f8e2392 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 31 Mar 2022 06:51:23 +0000 Subject: [PATCH 16/18] debugging tiny, mobile --- tinyimagenet_exec_GD.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 4b547558..e97b0dfa 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -6,12 +6,12 @@ #python main.py --config configs/training/mobilenetV2/tiny_adam.yml #> log_tiny_mobile_wt_adam_0001_multi 2>&1 #python main.py --config configs/training/mobilenetV2/tiny_sgd.yml > log_tiny_mobile_wt_sgd_01_multi 2>&1 -# IMP -:< "$subfolder" 2>&1 -BLOCK +subfolder="tiny_mobile_renda_new" +python imp_main.py --config configs/imp/tiny_mobilenet.yml --imp-rounds 20 --imp-no-rewind --gpu $gpu --subfolder "$subfolder" #> "$subfolder" 2>&1 +#BLOCK # IMP :< Date: Thu, 21 Apr 2022 03:49:29 +0000 Subject: [PATCH 17/18] renda, tiny, mobilenet --- tinyimagenet_exec_GD.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 6e2d7a56..f80919ea 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -9,8 +9,8 @@ # Renda #:< "$subfolder" 2>&1 +subfolder="tiny_mobile_renda_updated" +python imp_main.py --config configs/imp/tiny_mobilenet.yml --imp-rounds 20 --imp-no-rewind --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 #BLOCK # IMP From 078e65a24caca92ae403bb6438454ad306787369 Mon Sep 17 00:00:00 2001 From: "jysohn1108@gmail.com" Date: Wed, 27 Apr 2022 16:03:19 +0000 Subject: [PATCH 18/18] comment --- tinyimagenet_exec_GD.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tinyimagenet_exec_GD.sh b/tinyimagenet_exec_GD.sh index 0ba13508..7c44ba4d 100755 --- a/tinyimagenet_exec_GD.sh +++ b/tinyimagenet_exec_GD.sh @@ -61,23 +61,21 @@ python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_ BLOCK -gpu=0 +gpu=3 subfolder="tiny_mobile_gm_sp_20_invert" python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_20_sgd_w_invert.yml \ - --gpu $gpu --subfolder "$subfolder" #> "$subfolder" 2>&1 -:< "$subfolder" 2>&1 -gpu=0 subfolder="tiny_mobile_gm_sp_3_6_invert" python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_3_6_adam_w_invert.yml \ --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 -gpu=1 subfolder="tiny_mobile_gm_sp_1_4_invert" python main.py --config configs/hypercube/tinyImageNet/mobilenetV2/sparsity_1_4_sgd_w_invert.yml \ --gpu $gpu --subfolder "$subfolder" > "$subfolder" 2>&1 -BLOCK +#:<