From c5e7fa6e970e0c308b91407282dac79f9db1f224 Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Fri, 4 Feb 2022 01:19:35 -0600 Subject: [PATCH 01/30] smart ratio v3 init --- SmartRatio.py | 21 +++++---- cifar_exec_GD.sh | 27 +++++++++--- configs/sr/resnet20/resnet20_find_srV3.yml | 51 ++++++++++++++++++++++ configs/sr/resnet20/resnet20_srV3.yml | 47 ++++++++++++++++++++ main.py | 2 +- utils/conv_type.py | 3 ++ 6 files changed, 136 insertions(+), 15 deletions(-) create mode 100644 configs/sr/resnet20/resnet20_find_srV3.yml create mode 100644 configs/sr/resnet20/resnet20_srV3.yml diff --git a/SmartRatio.py b/SmartRatio.py index eedc9c3b..8f751df0 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -90,19 +90,22 @@ def SmartRatio(model, sr_args, parser_args): print(sum(p_arr)) # if we use modified version of smart ratio - if parser_args.sr_version == 2: + if parser_args.sr_version >= 2: if parser_args.arch.lower() != 'resnet20': raise NotImplementedError - - # followed the result in https://github.com/ksreenivasan/results_repo_pruning/blob/master/per_layer_sparsity_resnet20/hc_iter.csv - if parser_args.smart_ratio == 0.9856: # 1.44% sparsity - p_arr[0], p_arr[-1] = 0.3449074074074074, 0.259375 - - elif parser_args.smart_ratio == 0.9628: # 3.72% sparsity - p_arr[0], p_arr[-1] = 0.4143518518518518, 0.546875 - + if parser_args.sr_version == 2: + # followed the result in https://github.com/ksreenivasan/results_repo_pruning/blob/master/per_layer_sparsity_resnet20/hc_iter.csv + if parser_args.smart_ratio == 0.9856: # 1.44% sparsity + p_arr[0], p_arr[-1] = 0.3449074074074074, 0.259375 + elif parser_args.smart_ratio == 0.9628: # 3.72% sparsity + p_arr[0], p_arr[-1] = 0.4143518518518518, 0.546875 + else: + raise NotImplementedError + elif parser_args.sr_version == 3: + raise NotImplementedError else: raise NotImplementedError + print("p_arr", p_arr) print(sum(p_arr)) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 3058c1c2..e00cb2a8 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -3,29 +3,46 @@ #config_file="configs/sr/resnet20/resnet20_sr.yml" #subfolder=tmp -# SRv2, SRv3 +# SRv2 +:< Date: Fri, 4 Feb 2022 02:18:38 -0600 Subject: [PATCH 02/30] srV3 implement (for tuning sr) --- SmartRatio.py | 8 ++++---- args_helper.py | 16 ++++++++++++++++ main.py | 2 ++ main_utils.py | 19 +++++++++++++++++++ utils/conv_type.py | 36 +++++++++++++++++++++++++++--------- 5 files changed, 68 insertions(+), 13 deletions(-) diff --git a/SmartRatio.py b/SmartRatio.py index 8f751df0..dbed38f4 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -86,8 +86,8 @@ def SmartRatio(model, sr_args, parser_args): print("conv_term:", conv_term, "lin_term:", lin_term) scale = (num_weights * keep_ratio - lin_term) / conv_term p_arr[:-1] = scale * np.array(p_arr[:-1]) - print("p_arr", p_arr) - print(sum(p_arr)) + #print("p_arr", p_arr) + #print(sum(p_arr)) # if we use modified version of smart ratio if parser_args.sr_version >= 2: @@ -107,8 +107,8 @@ def SmartRatio(model, sr_args, parser_args): raise NotImplementedError - print("p_arr", p_arr) - print(sum(p_arr)) + #print("p_arr", p_arr) + #print(sum(p_arr)) # sometimes, if the prune_ratio is too small, some layer's keep ratio may be larger than 1 ExtraNum = 0 diff --git a/args_helper.py b/args_helper.py index 0b55ee90..3f794c26 100644 --- a/args_helper.py +++ b/args_helper.py @@ -904,6 +904,22 @@ def parse_arguments(self, jupyter_mode=False): type=int, help="smart ratio version number (1, 2, ...)", ) + parser.add_argument( + "--current-layer", + default=0, + type=int, + help="current layer index", + ) + # parser.add_argument( + # "--pt-sr-init", + # type=list, + # default=[], + # nargs="+", + # metavar="M", + # help="List of initial sparsity pattern for pt-sr (resnet20)" + # ) + + if jupyter_mode: args = parser.parse_args("") diff --git a/main.py b/main.py index b569db08..a430678b 100644 --- a/main.py +++ b/main.py @@ -46,6 +46,8 @@ def main_worker(gpu, ngpus_per_node): if not os.path.isdir(result_root): os.mkdir(result_root) + if parser_args.algo == 'pt_sr': + init_smart_ratio(parser_args) model = get_model(parser_args) print_model(model, parser_args) diff --git a/main_utils.py b/main_utils.py index db0652ab..70d8324b 100644 --- a/main_utils.py +++ b/main_utils.py @@ -1131,3 +1131,22 @@ def print_num_dataset(data): num_test += label.size()[0] print(num_train, num_val, num_test) + + +def init_smart_ratio(parser_args): + if parser_args.arch.lower() == 'resnet20': + if parser_args.target_sparsity == 3.72: + parser_args.init_sr = [41.43518518518518, 24.305555555555557, 21.875, 19.57465277777778, 17.36111111111111, + 15.321180555555555, 13.411458333333334, 11.631944444444445, 9.971788194444445, 8.441840277777779, + 7.03125, 5.750868055555555, 4.600694444444445, 3.5807291666666665, 2.685546875, + 1.9178602430555556, 1.2776692708333333, 0.7676866319444444, 0.3824869791666667, 54.6875]/100 + elif parser_args.target_sparsity == 1.44: + parser_args.init_sr = [34.49074074074074, 9.114583333333334, 8.203125, 7.335069444444445, 6.510416666666667, + 5.729166666666667, 5.034722222222222, 4.361979166666667, 3.7434895833333335, 3.1575520833333335, + 2.63671875, 2.1592881944444446, 1.7252604166666667, 1.3400607638888888, 1.0064019097222223, + 0.7188585069444444, 0.4774305555555556, 0.2875434027777778, 0.1437717013888889, 25.9375]/100 + else: + raise NotImplementedError + else: + raise NotImplementedError + diff --git a/utils/conv_type.py b/utils/conv_type.py index d548d294..a55d873e 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -66,6 +66,12 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc out = torch.bernoulli(scores) bias_out = torch.bernoulli(bias_scores) + elif parser_args.algo == 'pt_sr': + scores = torch.clamp(scores, 0, 1) + bias_scores = torch.clamp(bias_scores, 0, 1) + out = torch.bernoulli(scores) + bias_out = torch.bernoulli(bias_scores) + elif parser_args.algo in ['hc', 'hc_iter']: # round scores to {0, 1} # NOTE: doing this EP style where the scores are unchanged, but mask is computed @@ -103,16 +109,22 @@ def __init__(self, *args, **kwargs): # dummy variable just so other things don't break self.bias_flag = nn.Parameter(torch.Tensor(1)) - # initialize the scores - self.scores = nn.Parameter(torch.Tensor(self.weight.size())) - if parser_args.bias: - self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) + # initialize the layer_score (scalar) or score vector + if parser_args.algo == 'pt_sr': + if parser_args.arch.lower() == 'resnet20': + self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]])) # load pre-defined initial smart ratio + parser_args.current_layer += 1 + else: + raise NotImplementedError else: - # dummy variable just so other things don't break - self.bias_scores = nn.Parameter(torch.Tensor(1)) + self.scores = nn.Parameter(torch.Tensor(self.weight.size())) + if parser_args.bias: + self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_scores = nn.Parameter(torch.Tensor(1)) + - # initialize the random - # prune scores below this for global EP in bottom-k self.scores_prune_threshold = -np.inf @@ -141,6 +153,12 @@ def __init__(self, *args, **kwargs): self.scores.data = m.sample() m = Beta(torch.ones_like(self.bias_scores.data)*alpha, torch.ones_like(self.bias_scores.data)*beta) self.bias_scores.data = m.sample() + elif parser_args.algo in ['pt_sr']: + self.scores = self.layer_score * torch.ones_like(self.weight) + if parser_args.bias: + self.bias_scores = self.layer_score * torch.ones_like(self.bias) + else: + self.bias_scores = torch.Tensor(1) else: nn.init.kaiming_uniform_(self.scores, a=math.sqrt(5)) nn.init.uniform_(self.bias_scores, a=-1.0, b=1.0) # can't do kaiming here. picking U[-1, 1] for no real reason @@ -185,7 +203,7 @@ def forward(self, x): elif parser_args.algo in ['global_ep', 'global_ep_iter']: subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) else: - # ep, global_ep, global_ep_iter, pt etc + # ep, global_ep, global_ep_iter, pt, pt_sr etc subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), parser_args.prune_rate) if parser_args.algo in ['imp']: From 542ffb2df7acea1e8c53a4cdf67ad28b34d25b86 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 4 Feb 2022 08:58:32 +0000 Subject: [PATCH 03/30] srV3 tuning pruning ratio --- cifar_exec_GD.sh | 2 +- main_utils.py | 9 +++++---- utils/conv_type.py | 20 ++++++++++++++------ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index e00cb2a8..62a96f58 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -31,7 +31,7 @@ n_gpu=2 subfolder=find_SRv3_sp_1_44 python main.py \ --config $config_file \ - --target_sparsity 1.44 \ + --target-sparsity 1.44 \ --subfolder $subfolder \ --gpu $n_gpu diff --git a/main_utils.py b/main_utils.py index 70d8324b..ada81bc7 100644 --- a/main_utils.py +++ b/main_utils.py @@ -1136,17 +1136,18 @@ def print_num_dataset(data): def init_smart_ratio(parser_args): if parser_args.arch.lower() == 'resnet20': if parser_args.target_sparsity == 3.72: - parser_args.init_sr = [41.43518518518518, 24.305555555555557, 21.875, 19.57465277777778, 17.36111111111111, + parser_args.init_sr = np.array([41.43518518518518, 24.305555555555557, 21.875, 19.57465277777778, 17.36111111111111, 15.321180555555555, 13.411458333333334, 11.631944444444445, 9.971788194444445, 8.441840277777779, 7.03125, 5.750868055555555, 4.600694444444445, 3.5807291666666665, 2.685546875, - 1.9178602430555556, 1.2776692708333333, 0.7676866319444444, 0.3824869791666667, 54.6875]/100 + 1.9178602430555556, 1.2776692708333333, 0.7676866319444444, 0.3824869791666667, 54.6875])/100 elif parser_args.target_sparsity == 1.44: - parser_args.init_sr = [34.49074074074074, 9.114583333333334, 8.203125, 7.335069444444445, 6.510416666666667, + parser_args.init_sr = np.array([34.49074074074074, 9.114583333333334, 8.203125, 7.335069444444445, 6.510416666666667, 5.729166666666667, 5.034722222222222, 4.361979166666667, 3.7434895833333335, 3.1575520833333335, 2.63671875, 2.1592881944444446, 1.7252604166666667, 1.3400607638888888, 1.0064019097222223, - 0.7188585069444444, 0.4774305555555556, 0.2875434027777778, 0.1437717013888889, 25.9375]/100 + 0.7188585069444444, 0.4774305555555556, 0.2875434027777778, 0.1437717013888889, 25.9375])/100 else: raise NotImplementedError + #parser_args.init_sr = torch.from_numpy(parser_args.init_sr).cuda() else: raise NotImplementedError diff --git a/utils/conv_type.py b/utils/conv_type.py index a55d873e..e5151b69 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -67,6 +67,7 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc bias_out = torch.bernoulli(bias_scores) elif parser_args.algo == 'pt_sr': + #import pdb; pdb.set_trace() scores = torch.clamp(scores, 0, 1) bias_scores = torch.clamp(bias_scores, 0, 1) out = torch.bernoulli(scores) @@ -111,11 +112,19 @@ def __init__(self, *args, **kwargs): # initialize the layer_score (scalar) or score vector if parser_args.algo == 'pt_sr': + #import pdb; pdb.set_trace() if parser_args.arch.lower() == 'resnet20': - self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]])) # load pre-defined initial smart ratio + #self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]])) + self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]]).to("cuda:{}".format(parser_args.gpu))) # load pre-defined initial smart ratio parser_args.current_layer += 1 else: raise NotImplementedError + self.scores = torch.Tensor(self.weight.size()) + if parser_args.bias: + self.bias_scores = torch.Tensor(self.bias.size()) + else: + # dummy variable just so other things don't break + self.bias_scores = torch.Tensor(1) else: self.scores = nn.Parameter(torch.Tensor(self.weight.size())) if parser_args.bias: @@ -124,8 +133,6 @@ def __init__(self, *args, **kwargs): # dummy variable just so other things don't break self.bias_scores = nn.Parameter(torch.Tensor(1)) - - # prune scores below this for global EP in bottom-k self.scores_prune_threshold = -np.inf self.bias_scores_prune_threshold = -np.inf @@ -154,11 +161,12 @@ def __init__(self, *args, **kwargs): m = Beta(torch.ones_like(self.bias_scores.data)*alpha, torch.ones_like(self.bias_scores.data)*beta) self.bias_scores.data = m.sample() elif parser_args.algo in ['pt_sr']: - self.scores = self.layer_score * torch.ones_like(self.weight) + #self.scores.data = self.layer_score.data * torch.ones_like(self.weight.to("cuda:{}".format(parser_args.gpu))) + self.scores = self.layer_score * torch.ones_like(self.weight.to("cuda:{}".format(parser_args.gpu))) if parser_args.bias: - self.bias_scores = self.layer_score * torch.ones_like(self.bias) + self.bias_scores.data = self.layer_score.data * torch.ones_like(self.bias) else: - self.bias_scores = torch.Tensor(1) + self.bias_scores.data = self.layer_score.data else: nn.init.kaiming_uniform_(self.scores, a=math.sqrt(5)) nn.init.uniform_(self.bias_scores, a=-1.0, b=1.0) # can't do kaiming here. picking U[-1, 1] for no real reason From 940211ca6ee8627f39364c2dea73cb93945e89b6 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 4 Feb 2022 15:29:38 -0600 Subject: [PATCH 04/30] fixing pt_sr i'm sure i broke a few things, but this should be more amenable to autodiff --- utils/conv_type.py | 69 ++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/utils/conv_type.py b/utils/conv_type.py index e5151b69..f762d7ff 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -66,12 +66,12 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc out = torch.bernoulli(scores) bias_out = torch.bernoulli(bias_scores) - elif parser_args.algo == 'pt_sr': - #import pdb; pdb.set_trace() - scores = torch.clamp(scores, 0, 1) - bias_scores = torch.clamp(bias_scores, 0, 1) - out = torch.bernoulli(scores) - bias_out = torch.bernoulli(bias_scores) + # elif parser_args.algo == 'pt_sr': + # #import pdb; pdb.set_trace() + # scores = torch.clamp(scores, 0, 1) + # bias_scores = torch.clamp(bias_scores, 0, 1) + # out = torch.bernoulli(scores) + # bias_out = torch.bernoulli(bias_scores) elif parser_args.algo in ['hc', 'hc_iter']: # round scores to {0, 1} @@ -97,6 +97,20 @@ def backward(ctx, g_1, g_2): return g_1, g_2, None, None, None +class GetRandomSubnet(autograd.Function): + @staticmethod + def forward(ctx, layer_weight_ratio, layer_bias_ratio, weights): + out = torch.bernoulli(layer_weight_ratio * torch.ones_like(weights)) + bias_out = torch.bernoulli(layer_bias_ratio * torch.ones_like(weights)) + + return out, bias_out + + @staticmethod + def backward(ctx, g_1, g_2): + # send the gradient g straight-through on the backward pass. + return g_1, g_2, None + + # Not learning weights, finding subnet class SubnetConv(nn.Conv2d): def __init__(self, *args, **kwargs): @@ -110,28 +124,35 @@ def __init__(self, *args, **kwargs): # dummy variable just so other things don't break self.bias_flag = nn.Parameter(torch.Tensor(1)) + self.scores = nn.Parameter(torch.Tensor(self.weight.size())) + if parser_args.bias: + self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) + else: + # dummy variable just so other things don't break + self.bias_scores = nn.Parameter(torch.Tensor(1)) + # initialize the layer_score (scalar) or score vector if parser_args.algo == 'pt_sr': - #import pdb; pdb.set_trace() - if parser_args.arch.lower() == 'resnet20': - #self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]])) - self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]]).to("cuda:{}".format(parser_args.gpu))) # load pre-defined initial smart ratio - parser_args.current_layer += 1 - else: - raise NotImplementedError - self.scores = torch.Tensor(self.weight.size()) + #self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]])) + self.layer_weight_ratio = nn.Parameter(torch.Tensor(1)) + self.layer_weight_ratio = 0.5 + # do this in a better way. until then, let it be 0.5 + # self.layer_weight_ratio.data = parser_args.init_sr[parser_args.current_layer] + # parser_args.current_layer += 1 + if parser_args.bias: - self.bias_scores = torch.Tensor(self.bias.size()) + self.layer_bias_ratio = nn.Parameter(torch.Tensor(1)) + self.layer_bias_ratio = 0.5 else: # dummy variable just so other things don't break - self.bias_scores = torch.Tensor(1) - else: - self.scores = nn.Parameter(torch.Tensor(self.weight.size())) + self.layer_bias_ratio = torch.Tensor(1) + + # no gradients for scores anymore + self.scores.data = torch.ones_like(self.scores) + self.bias_scores = torch.ones_like(self.bias_scores) + self.scores.requires_grad = False if parser_args.bias: - self.bias_scores = nn.Parameter(torch.Tensor(self.bias.size())) - else: - # dummy variable just so other things don't break - self.bias_scores = nn.Parameter(torch.Tensor(1)) + self.bias_scores.requires_grad = False # prune scores below this for global EP in bottom-k self.scores_prune_threshold = -np.inf @@ -210,10 +231,12 @@ def forward(self, x): pass elif parser_args.algo in ['global_ep', 'global_ep_iter']: subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) + elif parser_args.algo in ['pt_sr']: + subnet, bias_subnet = GetRandomSubnet.apply(self.layer_weight_ratio, self.layer_bias_ratio, self.weight) else: # ep, global_ep, global_ep_iter, pt, pt_sr etc subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), parser_args.prune_rate) - + if parser_args.algo in ['imp']: # no STE, no subnet. Mask is handled outside w = self.weight From a2072b282d7bbd7fef5170e2342121cc45a897f2 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 4 Feb 2022 15:58:38 -0600 Subject: [PATCH 05/30] minor bugfixes --- cifar_exec.sh | 35 +++++++++++----------- configs/sr/resnet20/resnet20_find_srV3.yml | 2 +- trainers/default.py | 1 + utils/conv_type.py | 23 ++++++-------- 4 files changed, 29 insertions(+), 32 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index 30d57a9b..da9ec42f 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -64,21 +64,22 @@ BLOCK # Running trials in parallel # NOTE: make sure to delete/comment subfolder from the config file or else it may not work -conf_file="configs/hypercube/wideresnet28/wideresnet28_weight_training.yml" -log_root="wideresnet28_" +conf_file="configs/sr/resnet20/resnet20_find_srV3.yml" +log_root="pt_sr_debug_" log_end="_log" -subfolder_root="wideresnet28_results_trial_" - -for trial in 2 3 -do - python main.py \ - --config "$conf_file" \ - --trial-num $trial \ - --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 & - - python main.py \ - --config "$conf_file" \ - --trial-num $trial \ - --invert-sanity-check \ - --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & -done \ No newline at end of file +subfolder_root="pt_sr_debug_" + +#for trial in 1 +#do +trial=1 +python main.py \ +--config "$conf_file" \ +--trial-num $trial \ +--subfolder "$subfolder_root$trial" # > "$log_root$trial$log_end" 2>&1 + +# python main.py \ +# --config "$conf_file" \ +# --trial-num $trial \ +# --invert-sanity-check \ +# --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +#done diff --git a/configs/sr/resnet20/resnet20_find_srV3.yml b/configs/sr/resnet20/resnet20_find_srV3.yml index 885da2d3..ca7524fa 100644 --- a/configs/sr/resnet20/resnet20_find_srV3.yml +++ b/configs/sr/resnet20/resnet20_find_srV3.yml @@ -33,7 +33,7 @@ mode: fan_in nonlinearity: relu scale_fan: False freeze_weights: True -#target_sparsity: 1.44 +target_sparsity: 1.44 # ===== Hardware setup ===== # workers: 4 diff --git a/trainers/default.py b/trainers/default.py index 6597353f..effe480c 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -79,6 +79,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler loss += regularization_loss # measure accuracy and record loss + # import ipdb; ipdb.set_trace() acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) losses.update(loss.item(), images.size(0)) top1.update(acc1.item(), images.size(0)) diff --git a/utils/conv_type.py b/utils/conv_type.py index f762d7ff..610ad1a2 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -99,16 +99,19 @@ def backward(ctx, g_1, g_2): class GetRandomSubnet(autograd.Function): @staticmethod - def forward(ctx, layer_weight_ratio, layer_bias_ratio, weights): + def forward(ctx, layer_weight_ratio, layer_bias_ratio, weights, bias): out = torch.bernoulli(layer_weight_ratio * torch.ones_like(weights)) - bias_out = torch.bernoulli(layer_bias_ratio * torch.ones_like(weights)) + if parser_args.bias: + bias_out = torch.bernoulli(layer_bias_ratio * torch.ones_like(bias)) + else: + bias_out = torch.Tensor([0]) return out, bias_out @staticmethod def backward(ctx, g_1, g_2): # send the gradient g straight-through on the backward pass. - return g_1, g_2, None + return g_1, g_2, None, None # Not learning weights, finding subnet @@ -135,7 +138,7 @@ def __init__(self, *args, **kwargs): if parser_args.algo == 'pt_sr': #self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]])) self.layer_weight_ratio = nn.Parameter(torch.Tensor(1)) - self.layer_weight_ratio = 0.5 + self.layer_weight_ratio.data = torch.Tensor([0.5]) # do this in a better way. until then, let it be 0.5 # self.layer_weight_ratio.data = parser_args.init_sr[parser_args.current_layer] # parser_args.current_layer += 1 @@ -149,9 +152,9 @@ def __init__(self, *args, **kwargs): # no gradients for scores anymore self.scores.data = torch.ones_like(self.scores) - self.bias_scores = torch.ones_like(self.bias_scores) self.scores.requires_grad = False if parser_args.bias: + self.bias_scores = torch.ones_like(self.bias_scores) self.bias_scores.requires_grad = False # prune scores below this for global EP in bottom-k @@ -181,13 +184,6 @@ def __init__(self, *args, **kwargs): self.scores.data = m.sample() m = Beta(torch.ones_like(self.bias_scores.data)*alpha, torch.ones_like(self.bias_scores.data)*beta) self.bias_scores.data = m.sample() - elif parser_args.algo in ['pt_sr']: - #self.scores.data = self.layer_score.data * torch.ones_like(self.weight.to("cuda:{}".format(parser_args.gpu))) - self.scores = self.layer_score * torch.ones_like(self.weight.to("cuda:{}".format(parser_args.gpu))) - if parser_args.bias: - self.bias_scores.data = self.layer_score.data * torch.ones_like(self.bias) - else: - self.bias_scores.data = self.layer_score.data else: nn.init.kaiming_uniform_(self.scores, a=math.sqrt(5)) nn.init.uniform_(self.bias_scores, a=-1.0, b=1.0) # can't do kaiming here. picking U[-1, 1] for no real reason @@ -232,7 +228,7 @@ def forward(self, x): elif parser_args.algo in ['global_ep', 'global_ep_iter']: subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), 0, self.scores_prune_threshold, self.bias_scores_prune_threshold) elif parser_args.algo in ['pt_sr']: - subnet, bias_subnet = GetRandomSubnet.apply(self.layer_weight_ratio, self.layer_bias_ratio, self.weight) + subnet, bias_subnet = GetRandomSubnet.apply(self.layer_weight_ratio, self.layer_bias_ratio, self.weight, self.bias) else: # ep, global_ep, global_ep_iter, pt, pt_sr etc subnet, bias_subnet = GetSubnet.apply(self.scores.abs(), self.bias_scores.abs(), parser_args.prune_rate) @@ -247,7 +243,6 @@ def forward(self, x): b = self.bias * bias_subnet else: b = self.bias - x = F.conv2d( x, w, b, self.stride, self.padding, self.dilation, self.groups ) From ab7b5db7dd574761e4dedcb998637a746b7cc3cb Mon Sep 17 00:00:00 2001 From: Kartik Sreenivasan Date: Fri, 4 Feb 2022 16:39:54 -0600 Subject: [PATCH 06/30] added projection steps turns out the device_error was due to this, hiding under the pretense of multithreading --- trainers/default.py | 9 ++++----- utils/eval_utils.py | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/trainers/default.py b/trainers/default.py index effe480c..89bdb511 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -49,9 +49,9 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler if args.algo in ['global_ep', 'global_ep_iter']: prune(model, update_thresholds_only=True) - if args.algo in ['hc', 'hc_iter', 'pt'] and i % args.project_freq == 0 and not args.differentiate_clamp: + if args.algo in ['hc', 'hc_iter', 'pt', 'pt_sr'] and i % args.project_freq == 0 and not args.differentiate_clamp: for name, params in model.named_parameters(): - if "score" in name: + if "score" in name or "layer_weight_ratio" in name or "layer_bias_ratio" in name: scores = params with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) @@ -79,7 +79,6 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler loss += regularization_loss # measure accuracy and record loss - # import ipdb; ipdb.set_trace() acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) losses.update(loss.item(), images.size(0)) top1.update(acc1.item(), images.size(0)) @@ -110,9 +109,9 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler # update score thresholds for global ep if args.algo in ['global_ep', 'global_ep_iter']: prune(model, update_thresholds_only=True) - if args.algo in ['hc', 'hc_iter', 'pt'] and not args.differentiate_clamp: + if args.algo in ['hc', 'hc_iter', 'pt', 'pt_sr'] and not args.differentiate_clamp: for name, params in model.named_parameters(): - if "score" in name: + if "score" in name or "layer_weight_ratio" in name or "layer_bias_ratio" in name: scores = params with torch.no_grad(): scores.data = torch.clamp(scores.data, 0.0, 1.0) diff --git a/utils/eval_utils.py b/utils/eval_utils.py index 1147e6f8..1027d193 100644 --- a/utils/eval_utils.py +++ b/utils/eval_utils.py @@ -1,6 +1,5 @@ import torch - def accuracy(output, target, topk=(1,)): """Computes the accuracy over the k top predictions for the specified values of k""" with torch.no_grad(): From eeb828aadaa81889f22efe00a596ab42e7356209 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 4 Feb 2022 16:43:10 -0600 Subject: [PATCH 07/30] fixing up the exec script again --- cifar_exec.sh | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/cifar_exec.sh b/cifar_exec.sh index da9ec42f..d06afe16 100755 --- a/cifar_exec.sh +++ b/cifar_exec.sh @@ -69,17 +69,16 @@ log_root="pt_sr_debug_" log_end="_log" subfolder_root="pt_sr_debug_" -#for trial in 1 -#do -trial=1 -python main.py \ ---config "$conf_file" \ ---trial-num $trial \ ---subfolder "$subfolder_root$trial" # > "$log_root$trial$log_end" 2>&1 - -# python main.py \ -# --config "$conf_file" \ -# --trial-num $trial \ -# --invert-sanity-check \ -# --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & -#done +for trial in 1 +do + python main.py \ + --config "$conf_file" \ + --trial-num $trial \ + --subfolder "$subfolder_root$trial" > "$log_root$trial$log_end" 2>&1 + + # python main.py \ + # --config "$conf_file" \ + # --trial-num $trial \ + # --invert-sanity-check \ + # --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +done From 879f7be56e220e0c1973b02ce0488154072b6995 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 7 Feb 2022 20:53:55 +0000 Subject: [PATCH 08/30] debug --- trainers/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trainers/default.py b/trainers/default.py index 6597353f..cfd8992b 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -85,6 +85,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler top5.update(acc5.item(), images.size(0)) top10.update(acc10.item(), images.size(0)) + print("I am here") # compute gradient and do SGD step optimizer.zero_grad() if scaler is None: From f1ce43e0997f3f57dea2370d4647872dce72fb36 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 8 Feb 2022 08:40:51 +0000 Subject: [PATCH 09/30] SRv3 training p & running the tuned ratio --- SmartRatio.py | 181 +++------------ args_helper.py | 9 + cifar_exec_GD.sh | 255 +++------------------ configs/sr/resnet20/resnet20_find_srV3.yml | 4 +- configs/sr/resnet20/resnet20_srV3.yml | 2 +- main.py | 6 +- main_utils.py | 37 ++- trainers/default.py | 2 +- utils/conv_type.py | 4 - utils/net_utils.py | 30 ++- 10 files changed, 129 insertions(+), 401 deletions(-) diff --git a/SmartRatio.py b/SmartRatio.py index dbed38f4..554e79c7 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -4,6 +4,7 @@ import torch.nn.functional as F import math import numpy as np +import pandas as pd import copy import types import math @@ -86,29 +87,6 @@ def SmartRatio(model, sr_args, parser_args): print("conv_term:", conv_term, "lin_term:", lin_term) scale = (num_weights * keep_ratio - lin_term) / conv_term p_arr[:-1] = scale * np.array(p_arr[:-1]) - #print("p_arr", p_arr) - #print(sum(p_arr)) - - # if we use modified version of smart ratio - if parser_args.sr_version >= 2: - if parser_args.arch.lower() != 'resnet20': - raise NotImplementedError - if parser_args.sr_version == 2: - # followed the result in https://github.com/ksreenivasan/results_repo_pruning/blob/master/per_layer_sparsity_resnet20/hc_iter.csv - if parser_args.smart_ratio == 0.9856: # 1.44% sparsity - p_arr[0], p_arr[-1] = 0.3449074074074074, 0.259375 - elif parser_args.smart_ratio == 0.9628: # 3.72% sparsity - p_arr[0], p_arr[-1] = 0.4143518518518518, 0.546875 - else: - raise NotImplementedError - elif parser_args.sr_version == 3: - raise NotImplementedError - else: - raise NotImplementedError - - - #print("p_arr", p_arr) - #print(sum(p_arr)) # sometimes, if the prune_ratio is too small, some layer's keep ratio may be larger than 1 ExtraNum = 0 @@ -129,8 +107,34 @@ def SmartRatio(model, sr_args, parser_args): if ExtraNum == 0: break + # if we use modified version of smart ratio + if parser_args.sr_version >= 2: + if parser_args.arch.lower() != 'resnet20': + raise NotImplementedError + + root = 'per_layer_sparsity_resnet20/' + if parser_args.sr_version == 2: + hc = pd.read_csv(root + 'hc_iter.csv') + if parser_args.smart_ratio == 0.9856: # 1.44% sparsity + p_arr[0], p_arr[-1] = hc['1_4'].array[0]/100, hc['1_4'].array[-1]/100 + elif parser_args.smart_ratio == 0.9628: # 3.72% sparsity + p_arr[0], p_arr[-1] = hc['3_72'].array[0]/100, hc['3_72'].array[-1]/100 + else: + raise NotImplementedError + elif parser_args.sr_version == 3: + srV3 = pd.read_csv(root + 'smart_ratio_v3_lr1e-8_manual.csv') + if parser_args.smart_ratio == 0.9856: # 1.44% sparsity + #import pdb; pdb.set_trace() + p_arr = srV3['{}.0'.format(parser_args.srV3_epoch)].tolist() + else: + raise NotImplementedError + else: + raise NotImplementedError + + + print("p_arr", p_arr) - print(sum(p_arr)) + #print(sum(p_arr)) # 4. Randomly set the mask of each layer """ @@ -165,132 +169,3 @@ def SmartRatio(model, sr_args, parser_args): return model - ''' - # ========== calculate the sparsity using order statistics ============ - CNT = 0 - Num = [] - # ========== calculate the number of layers and the corresponding number of weights ============ - for idx, m in enumerate(net.modules()): - if isinstance(m, nn.Conv2d) or isinstance(m,nn.Linear): - Num.append(m.weight.data.view(-1).size()[0]) - CNT = CNT + 1 - - Num = torch.from_numpy(np.array(Num)).float() - - # ========== set ratio ============ - n = CNT - Ratio = torch.rand(1,CNT) - for i in range(CNT): - k = i + 1 # 1~CNT - Ratio[0][n-k] = (k)**2 + k - if args.linear_decay != 0: - Ratio[0][n-k] = k - if args.ascend != 0: - Ratio[0][n-k] = (n-k+1)**2 + (n-k+1) - if args.cubic != 0: - Ratio[0][n-k] = (k)**3 - - Ratio = Ratio[0] - - num_now = 0 - total_num = 0 - linear_num = 0 - - - # ========== calculation and scaling ============ - i = 0 - TEST = 0 - for m in net.modules(): - if isinstance(m,nn.Linear) or isinstance(m,nn.Conv2d): - if not isinstance(m,nn.Linear): - num_now = num_now + int((Ratio[i])*Num[i]) - if args.arch != 'resnet': - TEST = TEST + int(Num[i]*Ratio[i]/(i+1)**2) - else: - TEST = TEST + int(Num[i]*Ratio[i]) - else: - linear_num = linear_num + Num[i] - total_num = total_num + Num[i] - i = i + 1 - - goal_num = int(total_num * (1-args.init_prune_ratio)) - int(linear_num*linear_keep_ratio) - # ========== since the #linear_num is much lesser than that of total_num ============ - # ========== one can just easily set balance_ratio = 1 - init_prune_ratio without hurting the performance ============ - balance_ratio = goal_num / (total_num - linear_num) - # TEST - k = (goal_num) / TEST - i = 0 - for m in net.modules(): - if isinstance(m,nn.Conv2d): - if args.arch != 'resnet': - Ratio[i] = Ratio[i] * k / (i+1)**2 - else: - Ratio[i] = Ratio[i] * k - i = i + 1 - - - # ========== if the prune-ratio is too small, then some keep_ratio will > 1 ============ - # ========== the easy modification ============ - ExtraNum = 0 - i = 0 - for m in net.modules(): - size = Num[i] - if isinstance(m,nn.Linear) or isinstance(m,nn.Conv2d): - if not isinstance(m,nn.Linear): - if Ratio[i] >= 1: - ExtraNum = ExtraNum + int((Ratio[i]-1)*size) - Ratio[i] = 1 - else: - RestNum = int((1-Ratio[i])*Num[i]) - if RestNum >= ExtraNum: - Ratio[i] = Ratio[i] + ExtraNum/Num[i] - ExtraNum = 0 - else: - ExtraNum = ExtraNum - RestNum - Ratio[i] = 1 - if ExtraNum == 0: - break - i = i + 1 - - # ========== set the smart-ratio masks ============ - keep_masks = [] - CNT = 0 - - for m in net.modules(): - if isinstance(m,nn.Conv2d) or isinstance(m,nn.Linear): - mask = m.weight.data.abs().clone().float().cuda() - Size = mask.size() - mask = mask.view(-1) - keep_ratio = Ratio[CNT] - num_keep = int((keep_ratio)*Num[CNT]) - if Ratio[CNT] >= 1: - num_keep = int(Num[CNT]) - if args.uniform != 0: - Ratio[CNT] = balance_ratio - num_keep = int(Ratio[CNT]*Num[CNT]) - if isinstance(m,nn.Linear): - num_keep = int(linear_keep_ratio*Num[CNT]) - # ========== this judgement is for our hybrid ticket ============ - # ========== if specify the hybrid method, our smart ratio will combine the magnitude-based pruning ============ - if args.hybrid != 0: - print("################### DEBUG PRINT : USING HYBRID TICKET ###################") - value,idx = torch.topk(mask,num_keep) - temp = torch.zeros(int(Num[CNT])) - temp[idx] = 1.0 - mask = temp.clone().float().cuda() - - else: - temp = torch.ones(1,num_keep) - mask[0:num_keep] = temp - temp = torch.zeros(1,int(Num[CNT].item()-num_keep)) - mask[num_keep:] = temp - mask = mask.view(-1)[torch.randperm(mask.nelement())].view(mask.size()) - - - - CNT = CNT + 1 - keep_masks.append(mask.view(Size)) - - - return keep_masks - ''' diff --git a/args_helper.py b/args_helper.py index 3f794c26..65c0fc15 100644 --- a/args_helper.py +++ b/args_helper.py @@ -904,12 +904,21 @@ def parse_arguments(self, jupyter_mode=False): type=int, help="smart ratio version number (1, 2, ...)", ) + parser.add_argument( + "--srV3-epoch", + default=160, + type=int, + help="number of intermediate epochs used for loading srV3", + ) + + ''' parser.add_argument( "--current-layer", default=0, type=int, help="current layer index", ) + ''' # parser.add_argument( # "--pt-sr-init", # type=list, diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 62a96f58..1456c37d 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -1,4 +1,3 @@ - # SRv1 #config_file="configs/sr/resnet20/resnet20_sr.yml" #subfolder=tmp @@ -7,258 +6,60 @@ :< "$log_root$trial$log_end" 2>&1 #& - -# python main.py \ -# --config "$conf_file" --subfolder "invert_$subfolder_root$trial" \ -# --trial-num "$trial" --invert-sanity-check --skip-sanity-checks > "invert_$log_root$trial$log_end" 2>&1 & -done +config_file="configs/sr/resnet20/resnet20_srV3.yml" +n_gpu=1 +subfolder=SRv3_sp_1_44_lr_1e-8_manual +python main.py \ + --config $config_file \ + --smart_ratio 0.9856 \ + --subfolder $subfolder \ + --gpu $n_gpu BLOCK - -:< "$log_root$trial$log_end" 2>&1 #& -done -BLOCK - - - - - - - - - - - - - -### MobileNetV2 -#####python main.py --config configs/training/mobilenetV2/cifar10_mobileV2_training.yml #> mobilenet_cifar10_wt - - - -#python main.py --config configs/training/mobilenetV2/cifar10_mobileV2_training_check.yml #> mobilenet_cifar10_wt_check - -:< log_mobilev2_EP_sparsity_50 2>&1 -#python main.py --config configs/ep/mobilenetV2/cifar10_mobileV2_ep_sparsity_5.yml > log_mobilev2_EP_sparsity_5 2>&1 -#python main.py --config configs/ep/mobilenetV2/cifar10_mobileV2_ep_sparsity_20.yml > log_mobilev2_EP_sparsity_20 2>&1 -#python main.py --config configs/ep/mobilenetV2/cifar10_mobileV2_ep_sparsity_1_7.yml > log_mobilev2_EP_sparsity_1_7 2>&1 -BLOCK - -:< log_mobilev2_HC_sparsity_50 2>&1 -python main.py --config configs/hypercube/mobilenetV2/sparsity_5.yml > log_mobilev2_HC_sparsity_5 2>&1 -python main.py --config configs/hypercube/mobilenetV2/sparsity_20.yml > log_mobilev2_HC_sparsity_20 2>&1 -BLOCK -#python main.py --config configs/hypercube/mobilenetV2/sparsity_5_7lam6.yml > log_mobilev2_HC_sparsity_5_7lam6 2>&1 -#python main.py --config configs/hypercube/mobilenetV2/sparsity_1_4.yml > log_mobilev2_HC_sparsity_1_4 2>&1 -python main.py --config configs/hypercube/mobilenetV2/sparsity_20_3lam6.yml > log_mobilev2_HC_sparsity_20_3lam6 2>&1 - -#python main.py --config configs/hypercube/mobilenetV2/sparsity_20_1lam6.yml > log_mobilev2_HC_sparsity_20_1lam6 2>&1 -#python main.py --config configs/hypercube/mobilenetV2/sparsity_20_3lam6.yml > log_mobilev2_HC_sparsity_20_3lam6 2>&1 - - -#### ResNet-18 -#python main.py --config configs/hypercube/resnet18/resnet18_sc_hypercube_reg.yml # 93.17% at 150 epoch -#python main.py --config configs/hypercube/resnet18/resnet18_sc_hypercube_iter_reg.yml -#python main.py --config configs/hypercube/resnet18/resnet18_sc_hypercube_iter_reg_v2.yml -#python main.py --config configs/hypercube/resnet18/resnet18_sc_hypercube_noreg.yml -#python main.py --config configs/ep/resnet18/resnet18_sc_ep.yml -#python main.py --config configs/hypercube/resnet18/resnet18_sc_hypercube_iter_reg_evaluate.yml - - -### ResNet-32 -#python main.py --config configs/training/resnet32/cifar10_resnet32_training.yml - - -### ResNet-20 -#python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml -#TODO: add global_ep yml here -#python main.py --config configs/ep/resnet20/resnet20_global_ep_iter.yml #> cifar_log 2>&1 -#python main.py --config configs/ep/resnet20/resnet20_global_ep_iter_adam.yml #> cifar_log 2>&1 - -# Smart Ratio -#python main.py --config configs/sr/resnet20/resnet20_sr.yml # ResNet-20 -#python main.py --config configs/sr/resnet32/resnet32_sr.yml # ResNet-32 -#python main.py --config configs/training/resnet20/cifar10_resnet20_training.yml -#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t1.yml #> log_hc_sparsity_3_72_t1 2>&1 - - -# check mixed precision -#python main.py --config configs/hypercube/resnet20/mixed_precision/resnet20_sparsity_3_72_t1_with_MP.yml #> log_hc_sparsity_3_72_t1 2>&1 - -# old pruning schedule -:< log_hc_old_prune_schedule_sparsity_1_8_t1 2>&1 -python main.py --config configs/hypercube/resnet20/old_prune_schedule/resnet20_sparsity_0_15_t1.yml > log_hc_old_prune_schedule_sparsity_0_15_t1 2>&1 -python main.py --config configs/hypercube/resnet20/old_prune_schedule/resnet20_sparsity_0_15_t2.yml > log_hc_old_prune_schedule_sparsity_0_15_t2 2>&1 -python main.py --config configs/hypercube/resnet20/old_prune_schedule/resnet20_sparsity_0_15_t3.yml > log_hc_old_prune_schedule_sparsity_0_15_t3 2>&1 -python main.py --config configs/hypercube/resnet20/old_prune_schedule/resnet20_sparsity_0_15_t4.yml > log_hc_old_prune_schedule_sparsity_0_15_t4 2>&1 -python main.py --config configs/hypercube/resnet20/old_prune_schedule/resnet20_sparsity_0_15_t5.yml > log_hc_old_prune_schedule_sparsity_0_15_t5 2>&1 -BLOCK -#python main.py --config configs/hypercube/resnet20/old_prune_schedule/resnet20_target_sparsity_0_15_t1.yml > log_hc_old_prune_schedule_target_sparsity_0_15_t1 2>&1 -#python main.py --config configs/hypercube/resnet20/old_prune_schedule/resnet20_target_sparsity_0_15_t1.yml > log_hc_old_prune_schedule_target_sparsity_0_15_t1_real 2>&1 - -# HC for multiple trials -#:< log_hc_sparsity_3_72_t2 2>&1 -#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t3.yml > log_hc_sparsity_3_72_t3 2>&1 -#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t4.yml > log_hc_sparsity_3_72_t4 2>&1 -#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t5.yml > log_hc_sparsity_3_72_t5 2>&1 -#BLOCK - -:< log_hc_sparsity_0_59_t1 2>&1 -python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t2.yml > log_hc_sparsity_0_59_t2 2>&1 -python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t3.yml > log_hc_sparsity_0_59_t3 2>&1 -python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t4.yml > log_hc_sparsity_0_59_t4 2>&1 -python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t5.yml > log_hc_sparsity_0_59_t5 2>&1 -BLOCK - - - - - - -# EP -#python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_50.yml #> log_EP_sparsity_50 2>&1 -#python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_13_34.yml > log_EP_sparsity_13_34 2>&1 -#python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_3_72.yml > log_EP_sparsity_3_72 2>&1 -#python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_1_44.yml > log_EP_sparsity_1_44 2>&1 -#python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_0_59.yml > log_EP_sparsity_0_59 2>&1 -#python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_0_15.yml > log_EP_sparsity_0_15 2>&1 -#python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml - - - -## HC for denser models -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5.yml > log_target_sparsity_5_lam_5e-6 2>&1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20.yml > log_target_sparsity_20_lam_1e-6 2>&1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50.yml > log_target_sparsity_50_lam_0 2>&1 - -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5.yml > log_target_sparsity_5_lam_3e-5 2>&1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20.yml > log_target_sparsity_20_lam_1e-5 2>&1 - -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag.yml > log_target_sparsity_5_without_unflag_lam_3e-5 2>&1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag.yml > log_target_sparsity_20_without_unflag_lam_1e-5 2>&1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag.yml > log_target_sparsity_50_without_flag_lam_0 2>&1 - -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1.yml --run_idx 1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1e-2.yml --run_idx 1 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_1e-4.yml --run_idx 2 -#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_0_5_MAML_0.yml --run_idx 3 - - - - -#python main.py --config config10.yml --run_idx 10 #> log_config$r 2>&1 -:< log_config$r 2>&1 -done -BLOCK - -#python main.py --config config1.yml --run_idx 1 -#python main.py --config config2.yml --run_idx 2 -#python main.py --config config3.yml --run_idx 3 -#python main.py --config config4.yml --run_idx 4 - -#python main.py --config config5.yml --run_idx 5 -#python main.py --config config6.yml --run_idx 6 -#python main.py --config config7.yml --run_idx 7 -#python main.py --config config8.yml --run_idx 8 - -#python main.py --config config9.yml --run_idx 9 -#python main.py --config config10.yml --run_idx 10 - -#python main.py --config config11.yml --run_idx 11 -#python main.py --config config12.yml --run_idx 12 -#python main.py --config config13.yml --run_idx 13 - - - - -:< log_config$r 2>&1 + --config "$conf_file" \ + --smart_ratio 0.9856 \ + --srV3-epoch $epoch \ + --subfolder "$subfolder_root$epoch" > "$log_root$epoch$log_end" 2>&1 & done -BLOCK -# To run: nohup bash cifar_exec_GD.sh & -# To view log: tail -f log_config_i diff --git a/configs/sr/resnet20/resnet20_find_srV3.yml b/configs/sr/resnet20/resnet20_find_srV3.yml index ca7524fa..606c4046 100644 --- a/configs/sr/resnet20/resnet20_find_srV3.yml +++ b/configs/sr/resnet20/resnet20_find_srV3.yml @@ -13,14 +13,14 @@ name: resnet20_cifar10_sc # ===== Learning Rate Policy ======== # optimizer: sgd -lr: 0.1 +lr: 0.0000001 lr_policy: cosine_lr #fine_tune_lr: 0.1 #fine_tune_lr_policy: multistep_lr # cosine_lr #fine_tune_wd: 0.0001 # ===== Network training config ===== # -epochs: 160 #150 +epochs: 150 #150 momentum: 0.9 batch_size: 128 # 256 diff --git a/configs/sr/resnet20/resnet20_srV3.yml b/configs/sr/resnet20/resnet20_srV3.yml index 98adedaa..8cf42580 100644 --- a/configs/sr/resnet20/resnet20_srV3.yml +++ b/configs/sr/resnet20/resnet20_srV3.yml @@ -38,7 +38,7 @@ freeze_weights: True # ===== Hardware setup ===== # workers: 4 -# gpu: 3 +gpu: 2 # ===== Checkpointing ===== # checkpoint_at_prune: True diff --git a/main.py b/main.py index a430678b..029aeef5 100644 --- a/main.py +++ b/main.py @@ -46,9 +46,9 @@ def main_worker(gpu, ngpus_per_node): if not os.path.isdir(result_root): os.mkdir(result_root) - if parser_args.algo == 'pt_sr': - init_smart_ratio(parser_args) model = get_model(parser_args) + if parser_args.algo == 'pt_sr': + init_layer_weight_ratio(model, parser_args) print_model(model, parser_args) if parser_args.weight_training: @@ -162,6 +162,8 @@ def main_worker(gpu, ngpus_per_node): train_time.update((time.time() - start_train) / 60) scheduler.step() + + # evaluate on validation set start_validation = time.time() if parser_args.algo in ['hc', 'hc_iter']: diff --git a/main_utils.py b/main_utils.py index ada81bc7..3e2ce539 100644 --- a/main_utils.py +++ b/main_utils.py @@ -58,6 +58,7 @@ def print_layers(parser_args, model): i = 0 for layer in [*conv_layers, *linear_layers]: i += 1 + #print(i, layer, layer.layer_weight_ratio.data) print(i, layer) @@ -1133,21 +1134,37 @@ def print_num_dataset(data): print(num_train, num_val, num_test) -def init_smart_ratio(parser_args): +# initialize the layer_weight_ratio for pt_sr +def init_layer_weight_ratio(model, parser_args): if parser_args.arch.lower() == 'resnet20': + + # load smart ratio + sr = pd.read_csv("per_layer_sparsity_resnet20/smart_ratio.csv") + #import pdb; pdb.set_trace() + if parser_args.target_sparsity == 3.72: + p_arr = (sr['3.72'].array)/100 + elif parser_args.target_sparsity == 1.44: + p_arr = (sr['1.44'].array)/100 + else: + raise NotImplementedError + + # optionally, change the smart ratio + hc = pd.read_csv("per_layer_sparsity_resnet20/hc_iter.csv") if parser_args.target_sparsity == 3.72: - parser_args.init_sr = np.array([41.43518518518518, 24.305555555555557, 21.875, 19.57465277777778, 17.36111111111111, - 15.321180555555555, 13.411458333333334, 11.631944444444445, 9.971788194444445, 8.441840277777779, - 7.03125, 5.750868055555555, 4.600694444444445, 3.5807291666666665, 2.685546875, - 1.9178602430555556, 1.2776692708333333, 0.7676866319444444, 0.3824869791666667, 54.6875])/100 + p_arr[0], p_arr[-1] = hc['3_72'].array[0]/100, hc['3_72'].array[-1]/100 elif parser_args.target_sparsity == 1.44: - parser_args.init_sr = np.array([34.49074074074074, 9.114583333333334, 8.203125, 7.335069444444445, 6.510416666666667, - 5.729166666666667, 5.034722222222222, 4.361979166666667, 3.7434895833333335, 3.1575520833333335, - 2.63671875, 2.1592881944444446, 1.7252604166666667, 1.3400607638888888, 1.0064019097222223, - 0.7188585069444444, 0.4774305555555556, 0.2875434027777778, 0.1437717013888889, 25.9375])/100 + p_arr[0], p_arr[-1] = hc['1_4'].array[0]/100, hc['1_4'].array[-1]/100 else: raise NotImplementedError - #parser_args.init_sr = torch.from_numpy(parser_args.init_sr).cuda() + + # for each layer, set the initial layer_weight_ratio + conv_layers, linear_layers = get_layers(parser_args.arch, model) + i = 0 + for layer in [*conv_layers, *linear_layers]: + layer.layer_weight_ratio.data = torch.tensor([p_arr[i]]).float() + i += 1 + print(i, layer.layer_weight_ratio.item()) + else: raise NotImplementedError diff --git a/trainers/default.py b/trainers/default.py index 692a612a..7b39ff62 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -85,7 +85,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler top5.update(acc5.item(), images.size(0)) top10.update(acc10.item(), images.size(0)) - print("I am here") + #print("I am here") # compute gradient and do SGD step optimizer.zero_grad() if scaler is None: diff --git a/utils/conv_type.py b/utils/conv_type.py index 610ad1a2..f426bab8 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -136,12 +136,8 @@ def __init__(self, *args, **kwargs): # initialize the layer_score (scalar) or score vector if parser_args.algo == 'pt_sr': - #self.layer_score = nn.Parameter(torch.Tensor([parser_args.init_sr[parser_args.current_layer]])) self.layer_weight_ratio = nn.Parameter(torch.Tensor(1)) self.layer_weight_ratio.data = torch.Tensor([0.5]) - # do this in a better way. until then, let it be 0.5 - # self.layer_weight_ratio.data = parser_args.init_sr[parser_args.current_layer] - # parser_args.current_layer += 1 if parser_args.bias: self.layer_bias_ratio = nn.Parameter(torch.Tensor(1)) diff --git a/utils/net_utils.py b/utils/net_utils.py index 30d1fdea..edd68d83 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -13,6 +13,7 @@ from utils.mask_layers import MaskLinear, MaskConv from utils.conv_type import GetSubnet as GetSubnetConv +from utils.conv_type import GetRandomSubnet as GetRandomSubnetConv from utils.conv_type import SubnetConv @@ -500,9 +501,16 @@ def get_model_sparsity(model, threshold=0): if parser_args.bias: numer += b_numer denom += b_denom - # print('Overall sparsity: {}/{} ({:.2f} %)'.format((int)(numer), denom, 100*numer/denom)) + # print('Overall sparsity: {}/{} ({:.2f} %)'.format((int)(numer), denom, 100*numer/denom)) + return 100*numer/denom +def save_model_sparsity(model): + if isinstance(model, nn.parallel.DistributedDataParallel): + model = model.module + conv_layers, linear_layers = get_layers(parser_args.arch, model) + + # returns num_nonzero elements, total_num_elements so that it is easier to compute # average sparsity in the end @@ -528,6 +536,26 @@ def get_layer_sparsity(layer, threshold=0): else: b_numer, b_denom = 0, 0 + elif parser_args.algo in ['pt_sr']: + weight_mask, bias_mask = GetRandomSubnetConv.apply(layer.layer_weight_ratio, layer.layer_bias_ratio, layer.weight, layer.bias) + w_numer, w_denom = weight_mask.sum().item(), weight_mask.flatten().numel() + if parser_args.bias: + b_numer, b_denom = bias_mask.sum().item(), bias_mask.flatten().numel() + else: + b_numer, b_denom = 0, 0 + print(int(w_numer), w_denom, w_numer/w_denom) + + ''' + w_numer = layer.layer_weight_ratio.data * layer.weight.data.flatten().numel() + w_denom = layer.weight.data.flatten().numel() + if parser_args.bias: + b_numer = layer.layer_bias_ratio.data * layer.bias.data.flatten().numel() + b_denom = layer.bias.data.flatten().numel() + else: + b_numer, b_denom = 0, 0 + ''' + + elif parser_args.algo in ['global_ep', 'ep', 'global_ep_iter'] or parser_args.bottom_k_on_forward: if parser_args.algo == 'ep': weight_mask, bias_mask = GetSubnetConv.apply( From ad00b7a2efb2052d5ef938f7e751ab70344d3180 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 8 Feb 2022 21:39:22 +0000 Subject: [PATCH 10/30] SRv3 save/load, SRv4 --- SmartRatio.py | 10 +++++- cifar_exec_GD.sh | 41 ++++++++++++---------- configs/sr/resnet20/resnet20_find_srV3.yml | 2 +- main.py | 9 +++++ main_utils.py | 1 + utils/net_utils.py | 11 ++++-- 6 files changed, 51 insertions(+), 23 deletions(-) diff --git a/SmartRatio.py b/SmartRatio.py index 554e79c7..b17b018b 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -113,8 +113,8 @@ def SmartRatio(model, sr_args, parser_args): raise NotImplementedError root = 'per_layer_sparsity_resnet20/' + hc = pd.read_csv(root + 'hc_iter.csv') if parser_args.sr_version == 2: - hc = pd.read_csv(root + 'hc_iter.csv') if parser_args.smart_ratio == 0.9856: # 1.44% sparsity p_arr[0], p_arr[-1] = hc['1_4'].array[0]/100, hc['1_4'].array[-1]/100 elif parser_args.smart_ratio == 0.9628: # 3.72% sparsity @@ -122,12 +122,20 @@ def SmartRatio(model, sr_args, parser_args): else: raise NotImplementedError elif parser_args.sr_version == 3: + print("Check whether we are using desired csv file") + import pdb; pdb.set_trace() srV3 = pd.read_csv(root + 'smart_ratio_v3_lr1e-8_manual.csv') if parser_args.smart_ratio == 0.9856: # 1.44% sparsity #import pdb; pdb.set_trace() p_arr = srV3['{}.0'.format(parser_args.srV3_epoch)].tolist() else: raise NotImplementedError + elif parser_args.sr_version == 4: + if parser_args.smart_ratio == 0.9856: # 1.44% sparsity + p_arr[0], p_arr[-1] = 1, 1 + else: + raise NotImplementedError + else: raise NotImplementedError diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 1456c37d..d0e3040f 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -1,8 +1,9 @@ -# SRv1 + +# SRv1: original SR (in Jason Lee's paper) #config_file="configs/sr/resnet20/resnet20_sr.yml" #subfolder=tmp -# SRv2 +# SRv2: original SR + change 1st/last layer sparsity (follow GM's one) :< "$log_root$epoch$log_end" 2>&1 & done +BLOCK - - +# SRv4: original SR + change 1st/last layer sparsity to 100% +#:< Date: Wed, 9 Feb 2022 21:35:55 +0000 Subject: [PATCH 11/30] per layer sparsity for trained one --- per_layer_sparsity_resnet20/hc_iter.csv | 21 ++++ per_layer_sparsity_resnet20/manual_load_sr.py | 99 +++++++++++++++++++ per_layer_sparsity_resnet20/smart_ratio.csv | 21 ++++ .../smart_ratio_v3_lr1e-6.csv | 21 ++++ .../smart_ratio_v3_lr1e-8_manual.csv | 21 ++++ 5 files changed, 183 insertions(+) create mode 100644 per_layer_sparsity_resnet20/hc_iter.csv create mode 100644 per_layer_sparsity_resnet20/manual_load_sr.py create mode 100644 per_layer_sparsity_resnet20/smart_ratio.csv create mode 100644 per_layer_sparsity_resnet20/smart_ratio_v3_lr1e-6.csv create mode 100644 per_layer_sparsity_resnet20/smart_ratio_v3_lr1e-8_manual.csv diff --git a/per_layer_sparsity_resnet20/hc_iter.csv b/per_layer_sparsity_resnet20/hc_iter.csv new file mode 100644 index 00000000..bfececbc --- /dev/null +++ b/per_layer_sparsity_resnet20/hc_iter.csv @@ -0,0 +1,21 @@ +50,3_72,1_4,0_59 +57.175925925925924,41.43518518518518,34.49074074074074,25.694444444444443 +50.173611111111114,16.97048611111111,7.161458333333333,3.3854166666666665 +50.47743055555556,17.14409722222222,8.246527777777779,4.470486111111111 +51.345486111111114,17.79513888888889,8.680555555555555,4.600694444444445 +49.30555555555556,15.538194444444445,7.161458333333333,3.4722222222222223 +50.30381944444444,15.147569444444445,6.944444444444445,3.732638888888889 +49.348958333333336,14.27951388888889,6.380208333333333,2.6041666666666665 +49.76128472222222,10.56857638888889,4.1015625,1.5407986111111112 +48.697916666666664,7.790798611111111,3.2118055555555554,1.2261284722222223 +50.09765625,7.313368055555555,3.146701388888889,1.7578125 +50.065104166666664,6.803385416666667,2.712673611111111,1.1176215277777777 +50.33637152777778,7.139756944444445,2.267795138888889,1.1176215277777777 +50.49913194444444,5.37109375,1.8337673611111112,0.4231770833333333 +49.26215277777778,3.3311631944444446,0.6781684027777778,0.2170138888888889 +49.61480034722222,1.8473307291666667,0.6293402777777778,0.10850694444444445 +50.20073784722222,1.7171223958333333,0.3689236111111111,0.1193576388888889 +49.924045138888886,1.708984375,0.6618923611111112,0.07866753472222222 +49.527994791666664,0.9412977430555556,0.3255208333333333,0.0705295138888889 +50.79481336805556,1.7659505208333333,0.6998697916666666,0.08951822916666667 +64.375,54.6875,25.9375,23.90625 diff --git a/per_layer_sparsity_resnet20/manual_load_sr.py b/per_layer_sparsity_resnet20/manual_load_sr.py new file mode 100644 index 00000000..0a03d58b --- /dev/null +++ b/per_layer_sparsity_resnet20/manual_load_sr.py @@ -0,0 +1,99 @@ +import numpy as np +import pandas as pd + + +r10 = [10, +0.32175925925925924, +0.0920138888888889, +0.07682291666666667, +0.08159722222222222, +0.05555555555555555, +0.053385416666666664, +0.05078125, +0.042534722222222224, +0.0361328125, +0.03428819444444445, +0.027560763888888888, +0.021267361111111112, +0.014105902777777778, +0.01416015625, +0.010552300347222222, +0.007378472222222222, +0.005343967013888889, +0.0028483072916666665, +0.001681857638888889, +0.2640625 +] + +r30 = [30, +0.33101851851851855, +0.09722222222222222, +0.0876736111111111, +0.06770833333333333, +0.07074652777777778, +0.0642361111111111, +0.046440972222222224, +0.04296875, +0.038736979166666664, +0.027886284722222224, +0.026258680555555556, +0.023980034722222224, +0.016493055555555556, +0.01416015625, +0.010850694444444444, +0.007134331597222222, +0.005533854166666667, +0.002685546875, +0.0023057725694444445, +0.265625 +] + +r70 = [70, +0.29398148148148145, +0.09418402777777778, +0.09505208333333333, +0.0720486111111111, +0.0681423611111111, +0.055989583333333336, +0.04600694444444445, +0.04513888888888889, +0.035698784722222224, +0.03342013888888889, +0.027018229166666668, +0.0205078125, +0.015516493055555556, +0.013292100694444444, +0.010118272569444444, +0.007080078125, +0.006076388888888889, +0.002875434027777778, +0.002387152777777778, +0.2375 +] + +r160 = [160, +0.35648148148148145, +0.08203125, +0.08637152777777778, +0.0685763888888889, +0.06684027777777778, +0.059895833333333336, +0.047309027777777776, +0.044921875, +0.03634982638888889, +0.03309461805555555, +0.027126736111111112, +0.023328993055555556, +0.017469618055555556, +0.012803819444444444, +0.011013454861111112, +0.00732421875, +0.006537543402777778, +0.003146701388888889, +0.003200954861111111, +0.203125 +] + + + +np.savetxt('smart_ratio_v3_lr1e-8_manual.csv', [p for p in zip(r10, r30, r70, r160)], delimiter=',', fmt='%s') diff --git a/per_layer_sparsity_resnet20/smart_ratio.csv b/per_layer_sparsity_resnet20/smart_ratio.csv new file mode 100644 index 00000000..ad67eec1 --- /dev/null +++ b/per_layer_sparsity_resnet20/smart_ratio.csv @@ -0,0 +1,21 @@ +0.59,1.44,3.72,5.0,10.0,20.0,50.0 +3.7037037037037037,9.953703703703704,26.85185185185185,36.111111111111114,72.91666666666667,100.0,100.0 +3.4288194444444446,9.114583333333334,24.305555555555557,32.8125,66.14583333333333,100.0,100.0 +3.0815972222222223,8.203125,21.875,29.557291666666668,59.548611111111114,100.0,100.0 +2.7777777777777777,7.335069444444445,19.57465277777778,26.432291666666668,53.255208333333336,100.0,100.0 +2.4305555555555554,6.510416666666667,17.36111111111111,23.48090277777778,47.35243055555556,100.0,100.0 +2.170138888888889,5.729166666666667,15.321180555555555,20.703125,41.75347222222222,100.0,100.0 +1.8663194444444444,5.034722222222222,13.411458333333334,18.14236111111111,36.545138888888886,100.0,100.0 +1.6493055555555556,4.361979166666667,11.631944444444445,15.711805555555555,31.68402777777778,73.80642361111111,100.0 +1.4105902777777777,3.7434895833333335,9.971788194444445,13.4765625,27.159288194444443,54.513888888888886,100.0 +1.1935763888888888,3.1575520833333335,8.441840277777779,11.40407986111111,22.981770833333332,46.126302083333336,100.0 +0.9982638888888888,2.63671875,7.03125,9.505208333333334,19.151475694444443,38.444010416666664,100.0 +0.8138020833333334,2.1592881944444446,5.750868055555555,7.769097222222222,15.668402777777779,31.4453125,100.0 +0.6510416666666666,1.7252604166666667,4.600694444444445,6.217447916666667,12.532552083333334,25.162760416666668,100.0 +0.5045572916666666,1.3400607638888888,3.5807291666666665,4.839409722222222,9.749348958333334,19.569227430555557,100.0 +0.3797743055555556,1.0064019097222223,2.685546875,3.6295572916666665,7.310655381944445,14.678276909722221,77.70453559027777 +0.2712673611111111,0.7188585069444444,1.9178602430555556,2.590603298611111,5.221896701388889,10.484483506944445,26.26681857638889 +0.18174913194444445,0.4774305555555556,1.2776692708333333,1.7279730902777777,3.4803602430555554,6.987847222222222,17.51030815972222 +0.10850694444444445,0.2875434027777778,0.7676866319444444,1.0362413194444444,2.0887586805555554,4.193793402777778,10.506184895833334 +0.054253472222222224,0.1437717013888889,0.3824869791666667,0.5181206597222222,1.0443793402777777,2.096896701388889,5.251736111111111 +30.0,30.0,30.0,30.0,30.0,30.0,30.0 diff --git a/per_layer_sparsity_resnet20/smart_ratio_v3_lr1e-6.csv b/per_layer_sparsity_resnet20/smart_ratio_v3_lr1e-6.csv new file mode 100644 index 00000000..d33d1284 --- /dev/null +++ b/per_layer_sparsity_resnet20/smart_ratio_v3_lr1e-6.csv @@ -0,0 +1,21 @@ +1,10,30,70,149 +0.33796296296296297,0.3402777777777778,0.31712962962962965,0.30787037037037035,0.3587962962962963 +0.08333333333333333,0.09418402777777778,0.09244791666666667,0.0920138888888889,0.07682291666666667 +0.08289930555555555,0.0842013888888889,0.0837673611111111,0.08159722222222222,0.08333333333333333 +0.07465277777777778,0.07074652777777778,0.06597222222222222,0.0720486111111111,0.06901041666666667 +0.0642361111111111,0.06597222222222222,0.0685763888888889,0.06163194444444445,0.06727430555555555 +0.05295138888888889,0.062065972222222224,0.05381944444444445,0.06987847222222222,0.0625 +0.04904513888888889,0.04600694444444445,0.052083333333333336,0.05772569444444445,0.0390625 +0.04405381944444445,0.040364583333333336,0.042534722222222224,0.046440972222222224,0.04513888888888889 +0.03689236111111111,0.0341796875,0.038628472222222224,0.035698784722222224,0.03721788194444445 +0.031575520833333336,0.031792534722222224,0.035698784722222224,0.03515625,0.033745659722222224 +0.03309461805555555,0.023871527777777776,0.026041666666666668,0.0244140625,0.023328993055555556 +0.021918402777777776,0.022243923611111112,0.021050347222222224,0.0224609375,0.020833333333333332 +0.017252604166666668,0.016493055555555556,0.016927083333333332,0.014865451388888888,0.014431423611111112 +0.012532552083333334,0.013237847222222222,0.012369791666666666,0.012749565972222222,0.0126953125 +0.009874131944444444,0.009657118055555556,0.008680555555555556,0.009304470486111112,0.009548611111111112 +0.007269965277777778,0.007595486111111111,0.008192274305555556,0.007785373263888889,0.008924696180555556 +0.005452473958333333,0.008544921875,0.009792751736111112,0.012668185763888888,0.01318359375 +0.0032823350694444445,0.0030653211805555555,0.003634982638888889,0.003879123263888889,0.00390625 +0.001003689236111111,0.003689236111111111,0.005208333333333333,0.006618923611111111,0.006618923611111111 +0.2546875,0.2546875,0.18125,0.10625,0.0921875 diff --git a/per_layer_sparsity_resnet20/smart_ratio_v3_lr1e-8_manual.csv b/per_layer_sparsity_resnet20/smart_ratio_v3_lr1e-8_manual.csv new file mode 100644 index 00000000..2f972d30 --- /dev/null +++ b/per_layer_sparsity_resnet20/smart_ratio_v3_lr1e-8_manual.csv @@ -0,0 +1,21 @@ +10.0,30.0,70.0,160.0 +0.32175925925925924,0.33101851851851855,0.29398148148148145,0.35648148148148145 +0.0920138888888889,0.09722222222222222,0.09418402777777778,0.08203125 +0.07682291666666667,0.0876736111111111,0.09505208333333333,0.08637152777777778 +0.08159722222222222,0.06770833333333333,0.0720486111111111,0.0685763888888889 +0.05555555555555555,0.07074652777777778,0.0681423611111111,0.06684027777777778 +0.053385416666666664,0.0642361111111111,0.055989583333333336,0.059895833333333336 +0.05078125,0.046440972222222224,0.04600694444444445,0.047309027777777776 +0.042534722222222224,0.04296875,0.04513888888888889,0.044921875 +0.0361328125,0.038736979166666664,0.035698784722222224,0.03634982638888889 +0.03428819444444445,0.027886284722222224,0.03342013888888889,0.03309461805555555 +0.027560763888888888,0.026258680555555556,0.027018229166666668,0.027126736111111112 +0.021267361111111112,0.023980034722222224,0.0205078125,0.023328993055555556 +0.014105902777777778,0.016493055555555556,0.015516493055555556,0.017469618055555556 +0.01416015625,0.01416015625,0.013292100694444444,0.012803819444444444 +0.010552300347222222,0.010850694444444444,0.010118272569444444,0.011013454861111112 +0.007378472222222222,0.007134331597222222,0.007080078125,0.00732421875 +0.005343967013888889,0.005533854166666667,0.006076388888888889,0.006537543402777778 +0.0028483072916666665,0.002685546875,0.002875434027777778,0.003146701388888889 +0.001681857638888889,0.0023057725694444445,0.002387152777777778,0.003200954861111111 +0.2640625,0.265625,0.2375,0.203125 From 6152409c19d9bcc226cea0cb903c8420bade36d7 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Feb 2022 03:26:26 +0000 Subject: [PATCH 12/30] SRv3 using the trained score --- SmartRatio.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/SmartRatio.py b/SmartRatio.py index b17b018b..5374097c 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -123,12 +123,14 @@ def SmartRatio(model, sr_args, parser_args): raise NotImplementedError elif parser_args.sr_version == 3: print("Check whether we are using desired csv file") - import pdb; pdb.set_trace() - srV3 = pd.read_csv(root + 'smart_ratio_v3_lr1e-8_manual.csv') - if parser_args.smart_ratio == 0.9856: # 1.44% sparsity - #import pdb; pdb.set_trace() - p_arr = srV3['{}.0'.format(parser_args.srV3_epoch)].tolist() - else: + filename = 'smart_ratio_v3_lr1e-6.csv' # 'smart_ratio_v3_lr1e-8_manual.csv' + print("filename: ", filename) + #import pdb; pdb.set_trace() + srV3 = pd.read_csv(root + filename) + p_arr = srV3['{}'.format(parser_args.srV3_epoch)].tolist() + #p_arr = srV3['{}.0'.format(parser_args.srV3_epoch)].tolist() + + if not parser_args.smart_ratio == 0.9856: # 1.44% sparsity raise NotImplementedError elif parser_args.sr_version == 4: if parser_args.smart_ratio == 0.9856: # 1.44% sparsity From 1b20fafe7f77642c239c5574861bebb2ae5a8413 Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Fri, 11 Feb 2022 23:18:39 -0600 Subject: [PATCH 13/30] grid search for smart ratio --- .../grid_search_saved.csv | 144 ++++++++++++++++++ per_layer_sparsity_resnet20/imp.csv | 21 +++ 2 files changed, 165 insertions(+) create mode 100644 per_layer_sparsity_resnet20/grid_search_saved.csv create mode 100644 per_layer_sparsity_resnet20/imp.csv diff --git a/per_layer_sparsity_resnet20/grid_search_saved.csv b/per_layer_sparsity_resnet20/grid_search_saved.csv new file mode 100644 index 00000000..9290f736 --- /dev/null +++ b/per_layer_sparsity_resnet20/grid_search_saved.csv @@ -0,0 +1,144 @@ +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,16.875001907348636 +54.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,16.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,26.875001907348636 +44.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,26.875001907348636 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,9.723958969116211,7.769097328186035,6.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,10.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,6.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,11.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,5.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.898003339767456,1.5182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.8420138359069824,2.398003339767456,2.0182290077209477,1.2144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.7552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,0.6773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.436371511220932,0.347222238779068,0.2146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.5638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.1197265625,36.87500190734863 +34.90740585327149,10.590277671813963,9.85243034362793,8.723958969116211,7.769097328186035,7.89930534362793,7.204861164093018,6.184895515441895,3.2552082538604736,3.3420138359069824,2.398003339767456,2.0182290077209477,1.7144097089767456,1.1773003339767456,0.4638671875,0.336371511220932,0.347222238779068,0.3146701455116272,0.2197265625,36.87500190734863 diff --git a/per_layer_sparsity_resnet20/imp.csv b/per_layer_sparsity_resnet20/imp.csv new file mode 100644 index 00000000..8f9451ee --- /dev/null +++ b/per_layer_sparsity_resnet20/imp.csv @@ -0,0 +1,21 @@ +100.0,80.0,64.0,51.2,40.96,32.77,26.21,20.97,16.78,13.42,10.74,8.59,6.87,5.5,4.4,3.52,2.81,2.25,1.8,1.44,1.15,0.92,0.74,0.59,0.47,0.38,0.3,0.24,0.19,0.15 +100.0,95.60185241699219,91.43518829345703,88.4259262084961,83.56481170654297,79.39814758300781,75.46295928955078,71.52777862548828,68.98148345947266,65.97222137451172,63.88888931274414,61.80555725097656,60.18518829345703,57.40740966796875,55.32406997680664,54.39814758300781,53.00925827026367,50.0,47.22222137451172,44.907405853271484,42.12963104248047,39.35185241699219,36.11111068725586,33.333335876464844,29.629629135131836,26.38888931274414,22.685184478759766,20.601852416992188,18.75,17.592592239379883 +100.0,91.27604675292969,82.33506774902344,72.0920181274414,62.41319274902344,55.72916793823242,48.61111068725586,42.96875,38.23784637451172,34.15798568725586,29.94791603088379,26.47569465637207,23.26388931274414,21.00694465637207,19.401042938232422,17.49131965637207,15.494791984558105,13.97569465637207,12.239583969116211,10.590277671813965,8.723958969116211,7.074652671813965,5.9895830154418945,5.121527671813965,4.383680820465088,3.472222328186035,2.9079861640930176,2.387152910232544,2.0833334922790527,1.7795138359069824 +100.0,90.0173568725586,79.6875,69.18402862548828,60.41666793823242,52.77777862548828,45.7899284362793,39.80034637451172,35.2430534362793,31.16319465637207,27.64756965637207,24.52256965637207,21.83159828186035,20.182292938232422,18.489582061767578,16.57986068725586,14.713541984558105,13.45486068725586,11.545138359069824,9.85243034362793,8.767361640930176,7.11805534362793,6.1197919845581055,5.512152671813965,4.5572919845581055,3.819444417953491,3.125,2.821180582046509,2.6041665077209473,2.34375 +100.0,88.19444274902344,77.03993225097656,66.015625,55.33854293823242,47.1788215637207,41.23263931274414,36.328125,31.51041603088379,27.38715362548828,23.48090171813965,20.703125,18.40277671813965,16.49305534362793,14.75694465637207,13.802083015441895,12.41319465637207,11.02430534362793,9.722222328186035,8.723958969116211,7.161458492279053,6.163194179534912,4.9479169845581055,3.862847328186035,3.168402671813965,2.734375,2.300347089767456,1.953125,1.6059027910232544,1.2586805820465088 +100.0,88.4548568725586,75.73784637451172,65.7986068725586,54.64409637451172,46.65798568725586,40.66840362548828,34.635414123535156,29.6875,25.43402862548828,21.83159828186035,18.79340171813965,16.796875,15.277777671813965,14.23611068725586,13.28125,12.19618034362793,10.677083969116211,9.244791030883789,7.769097328186035,6.032986164093018,4.644097328186035,3.949652671813965,3.125,2.6041665077209473,2.256944417953491,1.953125,1.6493055820465088,1.388888955116272,1.0416667461395264 +100.0,89.32292175292969,78.77604675292969,68.01215362548828,58.15972137451172,50.30381774902344,44.401039123535156,39.713539123535156,34.895835876464844,30.98958396911621,26.86631965637207,23.4375,21.05034828186035,19.31423568725586,18.14236068725586,16.40625,14.019097328186035,11.675347328186035,9.765625,7.89930534362793,6.163194179534912,5.078125,4.1666669845581055,3.168402671813965,2.647569417953491,2.1267361640930176,1.8229166269302368,1.4322917461395264,1.0850694179534912,0.9548611044883728 +100.0,89.40972137451172,78.03819274902344,66.14582824707031,56.59722137451172,49.5225715637207,43.098960876464844,37.9774284362793,32.8993034362793,27.82118034362793,24.13194465637207,20.703125,18.619792938232422,17.057292938232422,15.71180534362793,14.019097328186035,11.892361640930176,10.286458969116211,8.767361640930176,7.204861164093018,6.1197919845581055,5.034722328186035,4.123263835906982,3.428819417953491,2.821180582046509,2.5173611640930176,2.256944417953491,1.8229166269302368,1.6059027910232544,1.2586805820465088 +100.0,89.53993225097656,78.36371612548828,67.66493225097656,57.96440887451172,50.10850524902344,43.5546875,37.86892318725586,32.3350715637207,27.45225715637207,23.26388931274414,19.31423568725586,16.68836784362793,15.277777671813965,13.97569465637207,12.6953125,10.894097328186035,9.396700859069824,7.421875,6.1848955154418945,4.904513835906982,4.058159828186035,3.168402671813965,2.5607638359069824,2.0182290077209473,1.5842013359069824,1.0633680820465088,0.8246527910232544,0.7161458730697632,0.5642361044883728 +100.0,86.1328125,73.29644012451172,61.42578125,50.98741149902344,42.79513931274414,35.8506965637207,30.03472137451172,24.85894203186035,20.638019561767578,17.1875,13.68272590637207,11.71875,10.796441078186035,9.689669609069824,8.561197280883789,6.868489742279053,5.544704914093018,4.19921875,3.2552082538604736,2.6584200859069824,2.202690839767456,1.7686631679534912,1.3346354961395264,0.9114583134651184,0.7486978769302368,0.575086772441864,0.4448784589767456,0.3255208134651184,0.1953125 +100.0,86.24131774902344,73.0251693725586,61.17621612548828,51.94227600097656,43.7391471862793,37.0008659362793,31.40190887451172,26.34548568725586,22.34157943725586,18.522136688232422,15.34288215637207,12.99913215637207,11.61024284362793,10.25390625,9.049478530883789,7.23741340637207,5.6315107345581055,4.296875,3.3420138359069824,2.6584200859069824,2.137586832046509,1.7035590410232544,1.3671875,1.1284722089767456,0.8572048544883728,0.6510416269302368,0.4665798544883728,0.29296875,0.1519097238779068 +100.0,85.87239074707031,72.72135925292969,60.70963668823242,50.49913024902344,42.33940887451172,35.72048568725586,29.81770896911621,24.65277671813965,20.540363311767578,16.731769561767578,13.55251693725586,11.469183921813965,9.9609375,8.452691078186035,6.857638835906982,5.2734375,3.949652671813965,3.114149332046509,2.398003339767456,1.8988715410232544,1.5299479961395264,1.247829794883728,0.9657118320465088,0.8138020634651184,0.6184896230697632,0.48828125,0.390625,0.2821180522441864,0.173611119389534 +100.0,85.68793487548828,71.9943618774414,59.30989456176758,48.860679626464844,40.89626693725586,34.23394012451172,28.56987762451172,23.70876693725586,19.76996421813965,16.40625,13.617621421813965,11.23046875,9.55946159362793,7.931857585906982,6.412760257720947,4.904513835906982,3.461371421813965,2.6584200859069824,2.0182290077209473,1.3780381679534912,1.1067707538604736,0.8029513955116272,0.596788227558136,0.4557291567325592,0.3797743022441864,0.2712673544883728,0.1627604067325592,0.086805559694767,0.0651041716337204 +100.0,83.75651550292969,69.19487762451172,55.96788024902344,46.223960876464844,38.32465362548828,31.59722137451172,25.64019012451172,21.061199188232422,17.55642318725586,14.18185806274414,11.566840171813965,9.375,7.89930534362793,6.369357585906982,4.9153642654418945,3.765191078186035,2.7994790077209473,2.094184160232544,1.7144097089767456,1.291232705116272,1.095920205116272,0.78125,0.6401909589767456,0.4557291567325592,0.336371511220932,0.2170138955116272,0.1085069477558136,0.0651041716337204,0.0434027798473835 +100.0,83.18684387207031,67.95790100097656,55.07270050048828,44.43359375,35.8506965637207,29.31857681274414,23.75759506225586,18.87478256225586,15.04991340637207,12.103949546813965,9.700521469116211,7.91015625,6.35850715637207,4.7688798904418945,3.477647542953491,2.625868082046509,1.9802517890930176,1.540798544883728,1.1773003339767456,0.927734375,0.6673177480697632,0.5099826455116272,0.3743489384651184,0.2712673544883728,0.2007378488779068,0.1410590261220932,0.086805559694767,0.0434027798473835,0.0325520858168602 +100.0,81.15234375,64.02723693847656,50.18446350097656,39.3446159362793,31.01128387451172,24.47374153137207,19.173175811767578,14.946831703186035,11.452908515930176,8.951822280883789,7.047525882720947,5.357530117034912,3.9306640625,2.728949785232544,1.8446180820465088,1.3319227695465088,0.9467231035232544,0.6646050214767456,0.4638671875,0.347222238779068,0.2197265625,0.149197056889534,0.1057942733168602,0.0759548619389534,0.0569661445915699,0.0434027798473835,0.0244140625,0.0162760429084301,0.005425347480922937 +100.0,81.12250518798828,64.43413543701172,50.3662109375,39.64301300048828,31.64876365661621,25.0244140625,19.50954818725586,15.351019859313965,11.81640625,8.916558265686035,6.830512046813965,4.888237953186035,3.2741971015930176,2.1457247734069824,1.3563368320465088,0.9711371660232544,0.721571147441864,0.5181206464767456,0.336371511220932,0.2360026091337204,0.1627604067325592,0.1030815988779068,0.0461154505610466,0.0271267369389534,0.0135633684694767,0.0135633684694767,0.00813802145421505,0.0027126737404614687,0.0 +100.0,78.88996887207031,60.63639450073242,46.48166275024414,35.782875061035156,27.6611328125,21.402996063232422,16.63682746887207,12.763129234313965,9.784613609313965,7.4462890625,5.609808921813965,3.998480796813965,2.4983725547790527,1.7605252265930176,1.1610243320465088,0.8544921875,0.6184896230697632,0.4720052182674408,0.347222238779068,0.2577039897441864,0.1329210102558136,0.0678168386220932,0.0406901016831398,0.0244140625,0.0135633684694767,0.010850694961845875,0.00813802145421505,0.0,0.0 +100.0,79.4216537475586,62.44303512573242,49.275718688964844,38.9051628112793,30.84581184387207,24.422199249267578,19.04025650024414,14.713541984558105,11.1572265625,8.365884780883789,6.212022304534912,4.5247392654418945,3.0056424140930176,1.9015841484069824,1.2125651836395264,0.8436415195465088,0.6157768964767456,0.4828559160232544,0.3146701455116272,0.2332899272441864,0.1600477397441864,0.1030815988779068,0.0678168386220932,0.0379774309694767,0.0135633684694767,0.005425347480922937,0.0027126737404614687,0.0,0.0 +100.0,65.673828125,48.29372787475586,37.62749481201172,28.59700584411621,19.76182746887207,12.91775131225586,8.561197280883789,5.856662273406982,4.161241054534912,3.1168618202209473,2.275933027267456,1.7062716484069824,1.2586805820465088,0.8544921875,0.6510416269302368,0.48828125,0.3770616352558136,0.2902560830116272,0.2197265625,0.1546224057674408,0.1057942733168602,0.0895182341337204,0.0623914934694767,0.0379774309694767,0.005425347480922937,0.0027126737404614687,0.0,0.0,0.0 +100.0,97.34375,94.375,92.8125,90.625,89.21875,85.46875,82.65625,79.21875,75.3125,72.5,68.4375,62.5,56.406246185302734,49.21875,42.1875,36.25,31.875,28.437501907348633,26.875001907348633,24.0625,22.96875,20.625,18.28125,15.9375,13.124999046325684,10.78125,9.375,7.343749523162842,6.40625 From 51c61d294f4786b25a765c8f4b5d091a07c96214 Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Fri, 11 Feb 2022 23:50:36 -0600 Subject: [PATCH 14/30] split for parallel running --- per_layer_sparsity_resnet20/.DS_Store | Bin 0 -> 6148 bytes .../grid_search_saved_0.csv | 24 ++++++++++++++++++ .../grid_search_saved_1.csv | 24 ++++++++++++++++++ .../grid_search_saved_2.csv | 24 ++++++++++++++++++ .../grid_search_saved_3.csv | 24 ++++++++++++++++++ .../grid_search_saved_4.csv | 24 ++++++++++++++++++ .../grid_search_saved_5.csv | 24 ++++++++++++++++++ 7 files changed, 144 insertions(+) create mode 100644 per_layer_sparsity_resnet20/.DS_Store create mode 100644 per_layer_sparsity_resnet20/grid_search_saved_0.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_saved_1.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_saved_2.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_saved_3.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_saved_4.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_saved_5.csv diff --git a/per_layer_sparsity_resnet20/.DS_Store b/per_layer_sparsity_resnet20/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Sat, 12 Feb 2022 06:04:16 +0000 Subject: [PATCH 15/30] smart ratio grid search --- SmartRatio.py | 5 +++ args_helper.py | 6 +++ cifar_exec_GD.sh | 110 +++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 117 insertions(+), 4 deletions(-) diff --git a/SmartRatio.py b/SmartRatio.py index 5374097c..fcfe466a 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -45,6 +45,11 @@ def SmartRatio(model, sr_args, parser_args): model = copy.deepcopy(model) # .eval() model.zero_grad() + # 0. if the sr_seq is pre-defined, use it + if parser_args.sr_seq is not None: + + else: + # 1. Compute the number of weights to be retrained conv_layers, linear_layers = get_layers(parser_args.arch, model) m_arr = [] diff --git a/args_helper.py b/args_helper.py index 65c0fc15..9a89ffa5 100644 --- a/args_helper.py +++ b/args_helper.py @@ -910,6 +910,12 @@ def parse_arguments(self, jupyter_mode=False): type=int, help="number of intermediate epochs used for loading srV3", ) + parser.add_argument( + "--sr_seq", + default=None, + type=str, + help="sequence to test for smart ratio", + ) ''' parser.add_argument( diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index d0e3040f..82c9e58f 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -1,4 +1,77 @@ +###################################################### +########## ResNet-18 ############################# +###################################################### + +# weight training +:< "$subfolder_root$log_end" 2>&1 & +BLOCK + +# smart ratio +:< "$subfolder_root$sr$log_end" 2>&1 #& +done +BLOCK + +# Gem-Miner (hypercube) +:< "$subfolder_root$log_end" 2>&1 #& +BLOCK + + + +###################################################### +########## ResNet-20 ############################# +###################################################### + + # SRv1: original SR (in Jason Lee's paper) #config_file="configs/sr/resnet20/resnet20_sr.yml" #subfolder=tmp @@ -43,7 +116,8 @@ log_root="srV3_1e-6_real_" log_end="_log" subfolder_root="srV3_1e-6_real_" -for epoch in 10 30 70 160 + +for epoch in 10 30 70 149 #160 do python main.py \ --config "$conf_file" \ @@ -51,11 +125,11 @@ do --srV3-epoch $epoch \ --subfolder "$subfolder_root$epoch" > "$log_root$epoch$log_end" 2>&1 & done -BLOCK +#BLOCK # SRv4: original SR + change 1st/last layer sparsity to 100% -#:< Date: Sat, 12 Feb 2022 00:06:46 -0600 Subject: [PATCH 16/30] indent --- SmartRatio.py | 295 +++++++++++++++++++++++++------------------------- 1 file changed, 148 insertions(+), 147 deletions(-) diff --git a/SmartRatio.py b/SmartRatio.py index fcfe466a..1fb35425 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -28,159 +28,160 @@ def SmartRatio(model, sr_args, parser_args): - ## TODO: current method assumes we are not using bias. We need to add lines for bias_scores, bias_flag, bias... - - if 'vgg' in parser_args.arch.lower(): - resnet_flag = False - elif 'resnet' in parser_args.arch.lower(): - resnet_flag = True - elif 'transformer' in parser_args.arch.lower() or 'mobile' in parser_args.arch.lower(): - resnet_flag = True # NOTE: hard code - else: - raise NotImplementedError("Smart Ratio only works for vgg and resnet") + ## TODO: current method assumes we are not using bias. We need to add lines for bias_scores, bias_flag, bias... + + if 'vgg' in parser_args.arch.lower(): + resnet_flag = False + elif 'resnet' in parser_args.arch.lower(): + resnet_flag = True + elif 'transformer' in parser_args.arch.lower() or 'mobile' in parser_args.arch.lower(): + resnet_flag = True # NOTE: hard code + else: + raise NotImplementedError("Smart Ratio only works for vgg and resnet") - keep_ratio = 1-parser_args.smart_ratio - linear_keep_ratio = sr_args.linear_keep_ratio + keep_ratio = 1-parser_args.smart_ratio + linear_keep_ratio = sr_args.linear_keep_ratio - model = copy.deepcopy(model) # .eval() - model.zero_grad() + model = copy.deepcopy(model) # .eval() + model.zero_grad() # 0. if the sr_seq is pre-defined, use it if parser_args.sr_seq is not None: - + p_arr_str = parser_args.sr_seq + import pdb; pdb.set_trace() else: - # 1. Compute the number of weights to be retrained - conv_layers, linear_layers = get_layers(parser_args.arch, model) - m_arr = [] - layer_num = 0 - for layer in [*conv_layers, *linear_layers]: - layer.scores.data = torch.ones_like(layer.scores.data) - m_arr.append(layer.weight.data.view(-1).size()[0]) - #m_dict[layer_num] = layer.weight.data.view(-1).size()[0] - layer_num += 1 - print(layer_num, layer) - - linear_layer_num = len(linear_layers) - if parser_args.arch == 'transformer': - linear_layer_num = 1 - - num_remain_weights = keep_ratio * sum(m_arr) - num_layers = layer_num - - print(m_arr) - print(sum(m_arr)) #print(sum(m_dict.values())) - print(num_layers, ' layers') - - - # 2. set p_l = (L-l+1)^2 + (L-l+1) - p_arr = [] - for l in range(1, num_layers+1): - if l > num_layers - linear_layer_num: # hacky way applicable for resnet_kaiming.py models - p_arr.append(linear_keep_ratio) - elif resnet_flag: - p_arr.append((num_layers-l+1)**2 + (num_layers-l+1)) - else: - p_arr.append( ((num_layers-l+1)**2 + (num_layers-l+1)) / (l * l) ) - - # 3. Find gamma such that p = 1 - \frac{ \sum_l m_l gamma p_l }{ \sum_l m_l } - - conv_term = np.multiply(np.array(m_arr[:-linear_layer_num]), np.array(p_arr[:-linear_layer_num])).sum() - # lin_term = m_arr[-1] * p_arr[-1] - lin_term = np.multiply(np.array(m_arr[-linear_layer_num:]), np.array(p_arr[-linear_layer_num:])).sum() - num_weights = sum(m_arr) - print("conv_term:", conv_term, "lin_term:", lin_term) - scale = (num_weights * keep_ratio - lin_term) / conv_term - p_arr[:-1] = scale * np.array(p_arr[:-1]) - - # sometimes, if the prune_ratio is too small, some layer's keep ratio may be larger than 1 - ExtraNum = 0 - for i in range(num_layers): - size = m_arr[i] - if i < num_layers - 1: - if p_arr[i] >= 1: - ExtraNum = ExtraNum + int((p_arr[i]-1) * size) - p_arr[i] = 1 - else: - RestNum = int((1-p_arr[i])*m_arr[i]) - if RestNum >= ExtraNum: - p_arr[i] = p_arr[i] + ExtraNum / m_arr[i] - ExtraNum = 0 - else: - ExtraNum = ExtraNum - RestNum - p_arr[i] = 1 - if ExtraNum == 0: - break - - # if we use modified version of smart ratio - if parser_args.sr_version >= 2: - if parser_args.arch.lower() != 'resnet20': - raise NotImplementedError - - root = 'per_layer_sparsity_resnet20/' - hc = pd.read_csv(root + 'hc_iter.csv') - if parser_args.sr_version == 2: - if parser_args.smart_ratio == 0.9856: # 1.44% sparsity - p_arr[0], p_arr[-1] = hc['1_4'].array[0]/100, hc['1_4'].array[-1]/100 - elif parser_args.smart_ratio == 0.9628: # 3.72% sparsity - p_arr[0], p_arr[-1] = hc['3_72'].array[0]/100, hc['3_72'].array[-1]/100 - else: - raise NotImplementedError - elif parser_args.sr_version == 3: - print("Check whether we are using desired csv file") - filename = 'smart_ratio_v3_lr1e-6.csv' # 'smart_ratio_v3_lr1e-8_manual.csv' - print("filename: ", filename) - #import pdb; pdb.set_trace() - srV3 = pd.read_csv(root + filename) - p_arr = srV3['{}'.format(parser_args.srV3_epoch)].tolist() - #p_arr = srV3['{}.0'.format(parser_args.srV3_epoch)].tolist() - - if not parser_args.smart_ratio == 0.9856: # 1.44% sparsity - raise NotImplementedError - elif parser_args.sr_version == 4: - if parser_args.smart_ratio == 0.9856: # 1.44% sparsity - p_arr[0], p_arr[-1] = 1, 1 - else: - raise NotImplementedError - - else: - raise NotImplementedError - - - - print("p_arr", p_arr) - #print(sum(p_arr)) - - # 4. Randomly set the mask of each layer - """ - layer_idx = 0 - for layer in [*conv_layers, *linear_layers]: # hacky way since linear layer is at the last part for resnet_kaiming.py models - p_curr = p_arr[layer_idx] - layer.flag.data = torch.bernoulli(p_curr * torch.ones_like(layer.flag.data)) - layer_idx += 1 - - print(layer_idx, torch.sum(layer.flag.data)/layer.flag.data.view(-1).size()[0]) - """ - # This part is modified to have exact number per layer (instead of Bernoulli sampling) - layer_idx = 0 - conv_layers, linear_layers = get_layers(arch=parser_args.arch, model=model) - for layer in conv_layers: - N = np.prod(layer.weight.shape) - K = int(p_arr[layer_idx] * N) - tmp_array = np.array([0] * (N-K) + [1] * K) - np.random.shuffle(tmp_array) - layer.flag.data = torch.nn.Parameter(torch.from_numpy(tmp_array).float().reshape(layer.weight.shape)) - layer.scores.data = torch.nn.Parameter(torch.ones(layer.weight.shape)) - layer_idx += 1 - for layer in linear_layers: - N = np.prod(layer.weight.shape) - K = int(p_arr[layer_idx] * N) - tmp_array = np.array([0] * (N-K) + [1] * K) - np.random.shuffle(tmp_array) - layer.flag.data = torch.nn.Parameter(torch.from_numpy(tmp_array).float().reshape(layer.weight.shape)) - layer.scores.data = torch.nn.Parameter(torch.ones(layer.weight.shape)) - layer_idx += 1 - - return model + # 1. Compute the number of weights to be retrained + conv_layers, linear_layers = get_layers(parser_args.arch, model) + m_arr = [] + layer_num = 0 + for layer in [*conv_layers, *linear_layers]: + layer.scores.data = torch.ones_like(layer.scores.data) + m_arr.append(layer.weight.data.view(-1).size()[0]) + #m_dict[layer_num] = layer.weight.data.view(-1).size()[0] + layer_num += 1 + print(layer_num, layer) + + linear_layer_num = len(linear_layers) + if parser_args.arch == 'transformer': + linear_layer_num = 1 + + num_remain_weights = keep_ratio * sum(m_arr) + num_layers = layer_num + + print(m_arr) + print(sum(m_arr)) #print(sum(m_dict.values())) + print(num_layers, ' layers') + + + # 2. set p_l = (L-l+1)^2 + (L-l+1) + p_arr = [] + for l in range(1, num_layers+1): + if l > num_layers - linear_layer_num: # hacky way applicable for resnet_kaiming.py models + p_arr.append(linear_keep_ratio) + elif resnet_flag: + p_arr.append((num_layers-l+1)**2 + (num_layers-l+1)) + else: + p_arr.append( ((num_layers-l+1)**2 + (num_layers-l+1)) / (l * l) ) + + # 3. Find gamma such that p = 1 - \frac{ \sum_l m_l gamma p_l }{ \sum_l m_l } + + conv_term = np.multiply(np.array(m_arr[:-linear_layer_num]), np.array(p_arr[:-linear_layer_num])).sum() + # lin_term = m_arr[-1] * p_arr[-1] + lin_term = np.multiply(np.array(m_arr[-linear_layer_num:]), np.array(p_arr[-linear_layer_num:])).sum() + num_weights = sum(m_arr) + print("conv_term:", conv_term, "lin_term:", lin_term) + scale = (num_weights * keep_ratio - lin_term) / conv_term + p_arr[:-1] = scale * np.array(p_arr[:-1]) + + # sometimes, if the prune_ratio is too small, some layer's keep ratio may be larger than 1 + ExtraNum = 0 + for i in range(num_layers): + size = m_arr[i] + if i < num_layers - 1: + if p_arr[i] >= 1: + ExtraNum = ExtraNum + int((p_arr[i]-1) * size) + p_arr[i] = 1 + else: + RestNum = int((1-p_arr[i])*m_arr[i]) + if RestNum >= ExtraNum: + p_arr[i] = p_arr[i] + ExtraNum / m_arr[i] + ExtraNum = 0 + else: + ExtraNum = ExtraNum - RestNum + p_arr[i] = 1 + if ExtraNum == 0: + break + + # if we use modified version of smart ratio + if parser_args.sr_version >= 2: + if parser_args.arch.lower() != 'resnet20': + raise NotImplementedError + + root = 'per_layer_sparsity_resnet20/' + hc = pd.read_csv(root + 'hc_iter.csv') + if parser_args.sr_version == 2: + if parser_args.smart_ratio == 0.9856: # 1.44% sparsity + p_arr[0], p_arr[-1] = hc['1_4'].array[0]/100, hc['1_4'].array[-1]/100 + elif parser_args.smart_ratio == 0.9628: # 3.72% sparsity + p_arr[0], p_arr[-1] = hc['3_72'].array[0]/100, hc['3_72'].array[-1]/100 + else: + raise NotImplementedError + elif parser_args.sr_version == 3: + print("Check whether we are using desired csv file") + filename = 'smart_ratio_v3_lr1e-6.csv' # 'smart_ratio_v3_lr1e-8_manual.csv' + print("filename: ", filename) + #import pdb; pdb.set_trace() + srV3 = pd.read_csv(root + filename) + p_arr = srV3['{}'.format(parser_args.srV3_epoch)].tolist() + #p_arr = srV3['{}.0'.format(parser_args.srV3_epoch)].tolist() + + if not parser_args.smart_ratio == 0.9856: # 1.44% sparsity + raise NotImplementedError + elif parser_args.sr_version == 4: + if parser_args.smart_ratio == 0.9856: # 1.44% sparsity + p_arr[0], p_arr[-1] = 1, 1 + else: + raise NotImplementedError + + else: + raise NotImplementedError + + + + print("p_arr", p_arr) + #print(sum(p_arr)) + + # 4. Randomly set the mask of each layer + """ + layer_idx = 0 + for layer in [*conv_layers, *linear_layers]: # hacky way since linear layer is at the last part for resnet_kaiming.py models + p_curr = p_arr[layer_idx] + layer.flag.data = torch.bernoulli(p_curr * torch.ones_like(layer.flag.data)) + layer_idx += 1 + + print(layer_idx, torch.sum(layer.flag.data)/layer.flag.data.view(-1).size()[0]) + """ + # This part is modified to have exact number per layer (instead of Bernoulli sampling) + layer_idx = 0 + conv_layers, linear_layers = get_layers(arch=parser_args.arch, model=model) + for layer in conv_layers: + N = np.prod(layer.weight.shape) + K = int(p_arr[layer_idx] * N) + tmp_array = np.array([0] * (N-K) + [1] * K) + np.random.shuffle(tmp_array) + layer.flag.data = torch.nn.Parameter(torch.from_numpy(tmp_array).float().reshape(layer.weight.shape)) + layer.scores.data = torch.nn.Parameter(torch.ones(layer.weight.shape)) + layer_idx += 1 + for layer in linear_layers: + N = np.prod(layer.weight.shape) + K = int(p_arr[layer_idx] * N) + tmp_array = np.array([0] * (N-K) + [1] * K) + np.random.shuffle(tmp_array) + layer.flag.data = torch.nn.Parameter(torch.from_numpy(tmp_array).float().reshape(layer.weight.shape)) + layer.scores.data = torch.nn.Parameter(torch.ones(layer.weight.shape)) + layer_idx += 1 + + return model From f434de97263b738e7e26f2ee99e4089a8f2a782d Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Feb 2022 06:49:54 +0000 Subject: [PATCH 17/30] smart ratio grid --- configs/sr/resnet20/resnet20_sr_grid.yml | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 configs/sr/resnet20/resnet20_sr_grid.yml diff --git a/configs/sr/resnet20/resnet20_sr_grid.yml b/configs/sr/resnet20/resnet20_sr_grid.yml new file mode 100644 index 00000000..e8299427 --- /dev/null +++ b/configs/sr/resnet20/resnet20_sr_grid.yml @@ -0,0 +1,47 @@ +# subfolder: sr +algo: 'hc_iter' +random_subnet: True +#sr_version: 4 +# smart_ratio: 0.98 # 2% sparsity + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_cifar10_sc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +# doesn't matter +# lr: 0.1 +# lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr # cosine_lr +fine_tune_wd: 0.0001 + +# ===== Network training config ===== # +epochs: 160 #150 +momentum: 0.9 +batch_size: 128 # 256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bias: False +bn_type: NonAffineBatchNorm +init: kaiming_normal # signed_constant +mode: fan_in +nonlinearity: relu +# prune_rate: 0.5 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +# gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True From 31a6fba494e592bfae679b4ade4b12540f9a6bcc Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Feb 2022 06:50:50 +0000 Subject: [PATCH 18/30] sr grid done --- SmartRatio.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SmartRatio.py b/SmartRatio.py index 1fb35425..97216707 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -42,13 +42,18 @@ def SmartRatio(model, sr_args, parser_args): keep_ratio = 1-parser_args.smart_ratio linear_keep_ratio = sr_args.linear_keep_ratio + #import pdb; pdb.set_trace() model = copy.deepcopy(model) # .eval() model.zero_grad() # 0. if the sr_seq is pre-defined, use it if parser_args.sr_seq is not None: p_arr_str = parser_args.sr_seq - import pdb; pdb.set_trace() + print(p_arr_str) + p_arr = [float(e)/100 for e in p_arr_str.split(",")] + #print(p_arr) + # print(p_arr_str.split(",")) + # print(p_arr_str.split("\r")) else: # 1. Compute the number of weights to be retrained From 20c2c81a332aaad98bde21386946304511e9b7c8 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 14 Feb 2022 21:11:15 +0000 Subject: [PATCH 19/30] grid search result gather --- grid_search_SR.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 grid_search_SR.py diff --git a/grid_search_SR.py b/grid_search_SR.py new file mode 100644 index 00000000..b4c19b8f --- /dev/null +++ b/grid_search_SR.py @@ -0,0 +1,36 @@ +''' +grid search for SR +''' + +import os +import glob +import pandas as pd +import pdb + +# use glob to get all the csv files in the folder + +best_idx = 0 +best_acc = 0 +best_sp = 0 +for idx in range(1, 49): + + path = 'results/SR_grid_sp_1_44_{}'.format(idx) + csv_files = glob.glob(os.path.join(path, "*/**.csv")) + #print(idx) + + # loop over the list of csv files + for f in csv_files: + + df = pd.read_csv(f) + #print('Location:', f) + #print('File Name:', f.split("\\")[-1]) + curr_acc = df['test_acc'].tolist()[-1] + curr_sp = df['model_sparsity'].tolist()[-1] + print('idx: {}, acc: {}, sparsity: {:.2f}'.format(idx, curr_acc, curr_sp)) + #pdb.set_trace() + if curr_acc > best_acc: + best_idx = idx + best_acc = curr_acc + best_sp = curr_sp + +print('best_idx: {}, best_acc: {}, best_sparsity: {:.2f}'.format(best_idx, best_acc, best_sp)) From 2ec8c708b77c13c038c510210827ef36de82efb7 Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Mon, 14 Feb 2022 22:14:39 -0600 Subject: [PATCH 20/30] grid search --- grid_search_smart_ratio.py | 100 +++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 grid_search_smart_ratio.py diff --git a/grid_search_smart_ratio.py b/grid_search_smart_ratio.py new file mode 100644 index 00000000..dd366bb7 --- /dev/null +++ b/grid_search_smart_ratio.py @@ -0,0 +1,100 @@ +#grid_search_smart_ratio.py +import pandas as pd +import pdb +import csv +import copy + + + +def sum_list(list1, list2): + zipped_lists = zip(list1, list2) + return [x + y for (x, y) in zipped_lists] + + + +sparsity_candidate = [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] + + +# 1. load imp sparsity pattern +imp_sp = pd.read_csv("per_layer_sparsity_resnet20/imp.csv") +init_sp = imp_sp["1.44"].tolist() +#print(init_sp) + +# 2. Find possible sparsity pattern and save it (restriction: p_i can be three nearest neighbor of p_i^{IMP}) + +i = 10 +arr_10 = [ + [+i, -i], + [0, 0], + [-i, +i] + ] +i = 1 +arr_1 = [ + [+i, +i, +i, 0, -i, -i, -i], + [+i, +i, 0, 0, 0, -i, -i], + [+i, 0, 0, 0, 0, 0, -i], + [0, 0, 0, 0, 0, 0, 0] + ] +i = 0.5 +arr_0_5 = [ + [+i, +i, +i, -i, -i, -i], + [+i, +i, 0, 0, -i, -i], + [+i, 0, 0, 0, 0, -i], + [0, 0, 0, 0, 0, 0] + ] +i = 0.1 +arr_0_1 = [ + [+i, +i, 0, -i, -i], + [+i, 0, 0, 0, -i], + [0, 0, 0, 0, 0] + ] + +result_list = [] +for i in range(len(arr_10)): + for j in range(len(arr_1)): + for k in range(len(arr_0_5)): + for l in range(len(arr_0_1)): + curr_sp = copy.deepcopy(init_sp) + + assert sum(arr_10[i]) == 0 + assert sum(arr_1[j]) == 0 + assert sum(arr_0_5[k]) == 0 + assert sum(arr_0_1[l]) == 0 + + #print(curr_sp) + + # update 1st & last layer + curr_sp[0] += arr_10[i][0] + curr_sp[-1] += arr_10[i][1] + #print(curr_sp) + + # update for 7 layers + curr_sp[1:8] = sum_list(curr_sp[1:8], arr_1[j]) + #print(curr_sp) + + # update for 6 layers + curr_sp[8:14] = sum_list(curr_sp[8:14], arr_0_5[k]) + #print(curr_sp) + + # update for 5 layers + curr_sp[14:19] = sum_list(curr_sp[14:19], arr_0_1[l]) + #print(curr_sp) + + + #print(len(curr_sp)) + # save it in the list + result_list.append(curr_sp) + +unit = 24 +root = "per_layer_sparsity_resnet20/" +for i in range(6): + with open(root + "grid_search_{}.csv".format(i), "w") as f: + write = csv.writer(f) + write.writerows(result_list[int(unit*i):int(unit*(i+1))]) + + +# with open("per_layer_sparsity_resnet20/grid_search.csv", "w") as f: +# write = csv.writer(f) +# write.writerows(result_list) + + From a452041f7a3b2c12cc38f959e5c1a068a2a01d0d Mon Sep 17 00:00:00 2001 From: "J.Sohn" Date: Mon, 14 Feb 2022 22:24:39 -0600 Subject: [PATCH 21/30] grid search near GM model --- grid_search_smart_ratio.py | 10 +++++--- .../grid_search_hc_saved_0.csv | 24 +++++++++++++++++++ .../grid_search_hc_saved_1.csv | 24 +++++++++++++++++++ .../grid_search_hc_saved_2.csv | 24 +++++++++++++++++++ .../grid_search_hc_saved_3.csv | 24 +++++++++++++++++++ .../grid_search_hc_saved_4.csv | 24 +++++++++++++++++++ .../grid_search_hc_saved_5.csv | 24 +++++++++++++++++++ 7 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 per_layer_sparsity_resnet20/grid_search_hc_saved_0.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_hc_saved_1.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_hc_saved_2.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_hc_saved_3.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_hc_saved_4.csv create mode 100644 per_layer_sparsity_resnet20/grid_search_hc_saved_5.csv diff --git a/grid_search_smart_ratio.py b/grid_search_smart_ratio.py index dd366bb7..7ab68ea5 100644 --- a/grid_search_smart_ratio.py +++ b/grid_search_smart_ratio.py @@ -16,9 +16,13 @@ def sum_list(list1, list2): # 1. load imp sparsity pattern -imp_sp = pd.read_csv("per_layer_sparsity_resnet20/imp.csv") -init_sp = imp_sp["1.44"].tolist() -#print(init_sp) +#imp_sp = pd.read_csv("per_layer_sparsity_resnet20/imp.csv") +#init_sp = imp_sp["1.44"].tolist() + +hc_sp = pd.read_csv("per_layer_sparsity_resnet20/hc_iter.csv") +init_sp = hc_sp["1_4"].tolist() + +print(init_sp) # 2. Find possible sparsity pattern and save it (restriction: p_i can be three nearest neighbor of p_i^{IMP}) diff --git a/per_layer_sparsity_resnet20/grid_search_hc_saved_0.csv b/per_layer_sparsity_resnet20/grid_search_hc_saved_0.csv new file mode 100644 index 00000000..ab23e629 --- /dev/null +++ b/per_layer_sparsity_resnet20/grid_search_hc_saved_0.csv @@ -0,0 +1,24 @@ +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 diff --git a/per_layer_sparsity_resnet20/grid_search_hc_saved_1.csv b/per_layer_sparsity_resnet20/grid_search_hc_saved_1.csv new file mode 100644 index 00000000..e85137fe --- /dev/null +++ b/per_layer_sparsity_resnet20/grid_search_hc_saved_1.csv @@ -0,0 +1,24 @@ +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,15.9375 +44.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,15.9375 diff --git a/per_layer_sparsity_resnet20/grid_search_hc_saved_2.csv b/per_layer_sparsity_resnet20/grid_search_hc_saved_2.csv new file mode 100644 index 00000000..8e19c222 --- /dev/null +++ b/per_layer_sparsity_resnet20/grid_search_hc_saved_2.csv @@ -0,0 +1,24 @@ +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 diff --git a/per_layer_sparsity_resnet20/grid_search_hc_saved_3.csv b/per_layer_sparsity_resnet20/grid_search_hc_saved_3.csv new file mode 100644 index 00000000..3ea132ea --- /dev/null +++ b/per_layer_sparsity_resnet20/grid_search_hc_saved_3.csv @@ -0,0 +1,24 @@ +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,25.9375 +34.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,25.9375 diff --git a/per_layer_sparsity_resnet20/grid_search_hc_saved_4.csv b/per_layer_sparsity_resnet20/grid_search_hc_saved_4.csv new file mode 100644 index 00000000..7e30eb5a --- /dev/null +++ b/per_layer_sparsity_resnet20/grid_search_hc_saved_4.csv @@ -0,0 +1,24 @@ +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,9.680555555555555,7.161458333333332,5.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,9.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,5.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 diff --git a/per_layer_sparsity_resnet20/grid_search_hc_saved_5.csv b/per_layer_sparsity_resnet20/grid_search_hc_saved_5.csv new file mode 100644 index 00000000..ef074768 --- /dev/null +++ b/per_layer_sparsity_resnet20/grid_search_hc_saved_5.csv @@ -0,0 +1,24 @@ +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,8.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,3.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,3.212673611111111,1.7677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.646701388888889,2.712673611111111,2.2677951388888893,1.3337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.711805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.1781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.46892361111111114,0.6618923611111112,0.2255208333333333,0.5998697916666667,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.7293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.5998697916666667,35.9375 +24.49074074074074,7.161458333333332,8.246527777777779,8.680555555555555,7.161458333333332,6.9444444444444455,6.380208333333332,4.1015625,3.211805555555556,3.146701388888889,2.712673611111111,2.2677951388888893,1.8337673611111112,0.6781684027777778,0.6293402777777778,0.3689236111111111,0.6618923611111112,0.3255208333333333,0.6998697916666666,35.9375 From c4367e20a0bf45a05a06eaf80eb0cc54c137d7c5 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 20 Feb 2022 19:07:35 +0000 Subject: [PATCH 22/30] grid search V5, and V6 init --- cifar_exec_GD.sh | 45 +++++++++++++++++++++++++++++++++++++++++++-- grid_search_SR.py | 18 +++++++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 82c9e58f..5dd4c440 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -143,9 +143,12 @@ BLOCK # SRv5: grid search -group=0 -n_gpu=2 +:< "$log_root$epoch$log_end" 2>&1 & +done +#BLOCK diff --git a/grid_search_SR.py b/grid_search_SR.py index b4c19b8f..8cdf078f 100644 --- a/grid_search_SR.py +++ b/grid_search_SR.py @@ -12,7 +12,7 @@ best_idx = 0 best_acc = 0 best_sp = 0 -for idx in range(1, 49): +for idx in range(1, 145): path = 'results/SR_grid_sp_1_44_{}'.format(idx) csv_files = glob.glob(os.path.join(path, "*/**.csv")) @@ -34,3 +34,19 @@ best_sp = curr_sp print('best_idx: {}, best_acc: {}, best_sparsity: {:.2f}'.format(best_idx, best_acc, best_sp)) + + +# save the optimal grid search result (sparsity pattern) +#best_idx = 113 # for DEBUG +group = int(best_idx/24) +line_idx = best_idx % 24 + +PATH = 'per_layer_sparsity_resnet20/grid_search_saved_{}.csv'.format(group) +df = pd.read_csv(PATH) +if line_idx == 0: + best_pattern = df.columns.tolist() +else: + best_pattern = df.values.tolist()[line_idx-1] + +print(best_pattern) + From ac594c57199cfbb52247255b01ffc59b5f2425b5 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 20 Feb 2022 19:09:36 +0000 Subject: [PATCH 23/30] set IMP ratio as the starting point for the grid search --- grid_search_smart_ratio.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/grid_search_smart_ratio.py b/grid_search_smart_ratio.py index 7ab68ea5..87097a69 100644 --- a/grid_search_smart_ratio.py +++ b/grid_search_smart_ratio.py @@ -16,11 +16,11 @@ def sum_list(list1, list2): # 1. load imp sparsity pattern -#imp_sp = pd.read_csv("per_layer_sparsity_resnet20/imp.csv") -#init_sp = imp_sp["1.44"].tolist() +imp_sp = pd.read_csv("per_layer_sparsity_resnet20/imp.csv") +init_sp = imp_sp["1.44"].tolist() -hc_sp = pd.read_csv("per_layer_sparsity_resnet20/hc_iter.csv") -init_sp = hc_sp["1_4"].tolist() +#hc_sp = pd.read_csv("per_layer_sparsity_resnet20/hc_iter.csv") +#init_sp = hc_sp["1_4"].tolist() print(init_sp) From 00a6084f76578f07599fb712dffacca325801c01 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 21 Feb 2022 17:15:19 +0000 Subject: [PATCH 24/30] changes --- SmartRatio.py | 12 ++++ cifar_exec_GD.sh | 36 +++++++++-- .../bias_AffineBN_True/sparsity_1_44.yml | 64 +++++++++++++++++++ .../bias_AffineBN_True/sparsity_3_72.yml | 63 ++++++++++++++++++ configs/sr/resnet20/resnet20_find_srV3.yml | 2 +- configs/sr/resnet20/resnet20_find_srV6.yml | 51 +++++++++++++++ configs/sr/resnet20/resnet20_srV3.yml | 2 +- configs/sr/resnet20/resnet20_srV6.yml | 47 ++++++++++++++ grid_search_SR.py | 2 +- main.py | 1 + main_utils.py | 47 +++++++++----- per_layer_sparsity_resnet20/grid_best.csv | 21 ++++++ .../smart_ratio_v6_lr1e-6.csv | 21 ++++++ 13 files changed, 342 insertions(+), 27 deletions(-) create mode 100644 configs/hypercube/resnet20/bias_AffineBN_True/sparsity_1_44.yml create mode 100644 configs/hypercube/resnet20/bias_AffineBN_True/sparsity_3_72.yml create mode 100644 configs/sr/resnet20/resnet20_find_srV6.yml create mode 100644 configs/sr/resnet20/resnet20_srV6.yml create mode 100644 per_layer_sparsity_resnet20/grid_best.csv create mode 100644 per_layer_sparsity_resnet20/smart_ratio_v6_lr1e-6.csv diff --git a/SmartRatio.py b/SmartRatio.py index 97216707..cc2af028 100644 --- a/SmartRatio.py +++ b/SmartRatio.py @@ -149,6 +149,18 @@ def SmartRatio(model, sr_args, parser_args): else: raise NotImplementedError + elif parser_args.sr_version == 6: + print("Check whether we are using desired csv file") + filename = 'smart_ratio_v6_lr1e-6.csv' # 'smart_ratio_v3_lr1e-8_manual.csv' + print("filename: ", filename) + #import pdb; pdb.set_trace() + srV3 = pd.read_csv(root + filename) + p_arr = srV3['{}'.format(parser_args.srV3_epoch)].tolist() + + if not parser_args.smart_ratio == 0.9856: # 1.44% sparsity + raise NotImplementedError + + else: raise NotImplementedError diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 5dd4c440..e688756e 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -1,4 +1,10 @@ + + + + + + ###################################################### ########## ResNet-18 ############################# ###################################################### @@ -71,6 +77,20 @@ BLOCK ########## ResNet-20 ############################# ###################################################### +# turn on bias, affine batch-norm +:< "$subfolder_root$log_end" 2>&1 #& +BLOCK + + + + # SRv1: original SR (in Jason Lee's paper) #config_file="configs/sr/resnet20/resnet20_sr.yml" @@ -145,8 +165,8 @@ BLOCK # SRv5: grid search :< "$log_root$epoch$log_end" 2>&1 & + --subfolder "$subfolder_root$epoch" #> "$log_root$epoch$log_end" 2>&1 & done -#BLOCK +BLOCK diff --git a/configs/hypercube/resnet20/bias_AffineBN_True/sparsity_1_44.yml b/configs/hypercube/resnet20/bias_AffineBN_True/sparsity_1_44.yml new file mode 100644 index 00000000..1005afda --- /dev/null +++ b/configs/hypercube/resnet20/bias_AffineBN_True/sparsity_1_44.yml @@ -0,0 +1,64 @@ +#subfolder: hc_sparsity_1_44_unflagT_real +trial_num: 1 +bias: True + + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: AffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.44 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet20/bias_AffineBN_True/sparsity_3_72.yml b/configs/hypercube/resnet20/bias_AffineBN_True/sparsity_3_72.yml new file mode 100644 index 00000000..0aba7aae --- /dev/null +++ b/configs/hypercube/resnet20/bias_AffineBN_True/sparsity_3_72.yml @@ -0,0 +1,63 @@ +#subfolder: target_sparsity_3_72_unflagT_real + + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 +bias: True + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: AffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 3.72 +# decide if you want to "unflag" +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00003 # 3e-5 + +# ===== Hardware setup ===== # +workers: 4 +#gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/sr/resnet20/resnet20_find_srV3.yml b/configs/sr/resnet20/resnet20_find_srV3.yml index c7a128aa..ade5d478 100644 --- a/configs/sr/resnet20/resnet20_find_srV3.yml +++ b/configs/sr/resnet20/resnet20_find_srV3.yml @@ -1,7 +1,7 @@ # subfolder: sr algo: 'pt_sr' #random_subnet: True -#sr_version: 3 +sr_version: 3 # smart_ratio: 0.98 # 2% sparsity # Architecture diff --git a/configs/sr/resnet20/resnet20_find_srV6.yml b/configs/sr/resnet20/resnet20_find_srV6.yml new file mode 100644 index 00000000..a34a42ed --- /dev/null +++ b/configs/sr/resnet20/resnet20_find_srV6.yml @@ -0,0 +1,51 @@ +# subfolder: sr +algo: 'pt_sr' +#random_subnet: True +sr_version: 6 +# smart_ratio: 0.98 # 2% sparsity + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_cifar10_sc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.000001 #0.0000001 +lr_policy: cosine_lr +#fine_tune_lr: 0.1 +#fine_tune_lr_policy: multistep_lr # cosine_lr +#fine_tune_wd: 0.0001 + +# ===== Network training config ===== # +epochs: 150 #150 +momentum: 0.9 +batch_size: 128 # 256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bias: False +bn_type: NonAffineBatchNorm +init: signed_constant +mode: fan_in +nonlinearity: relu +scale_fan: False +freeze_weights: True +target_sparsity: 1.44 + +# ===== Hardware setup ===== # +workers: 4 +# gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True + +# ==== fine-tune ==== # +skip_fine_tune: True + + diff --git a/configs/sr/resnet20/resnet20_srV3.yml b/configs/sr/resnet20/resnet20_srV3.yml index 8cf42580..f346a2c3 100644 --- a/configs/sr/resnet20/resnet20_srV3.yml +++ b/configs/sr/resnet20/resnet20_srV3.yml @@ -38,7 +38,7 @@ freeze_weights: True # ===== Hardware setup ===== # workers: 4 -gpu: 2 +gpu: 1 # ===== Checkpointing ===== # checkpoint_at_prune: True diff --git a/configs/sr/resnet20/resnet20_srV6.yml b/configs/sr/resnet20/resnet20_srV6.yml new file mode 100644 index 00000000..40deedef --- /dev/null +++ b/configs/sr/resnet20/resnet20_srV6.yml @@ -0,0 +1,47 @@ +# subfolder: sr +algo: 'hc_iter' +random_subnet: True +sr_version: 6 +# smart_ratio: 0.98 # 2% sparsity + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_cifar10_sc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +# doesn't matter +# lr: 0.1 +# lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: multistep_lr # cosine_lr +fine_tune_wd: 0.0001 + +# ===== Network training config ===== # +epochs: 160 #150 +momentum: 0.9 +batch_size: 128 # 256 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bias: False +bn_type: NonAffineBatchNorm +init: kaiming_normal # signed_constant +mode: fan_in +nonlinearity: relu +# prune_rate: 0.5 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/grid_search_SR.py b/grid_search_SR.py index 8cdf078f..39fd265e 100644 --- a/grid_search_SR.py +++ b/grid_search_SR.py @@ -37,7 +37,7 @@ # save the optimal grid search result (sparsity pattern) -#best_idx = 113 # for DEBUG +#best_idx = 22 # for DEBUG group = int(best_idx/24) line_idx = best_idx % 24 diff --git a/main.py b/main.py index 678390eb..1dccda30 100644 --- a/main.py +++ b/main.py @@ -289,6 +289,7 @@ def main_worker(gpu, ngpus_per_node): # save trained sparsity pattern for pt_sr if parser_args.algo in ['pt_sr']: sparsity_df.to_csv(result_root + 'sparsity_pattern.csv', index=False) + sparsity_df.to_csv('per_layer_sparsity_resnet20/smart_ratio_v{}_lr1e-6.csv'.format(parser_args.sr_version), index=False) # hard-coded # finetune weights cp_model = copy.deepcopy(model) diff --git a/main_utils.py b/main_utils.py index 67794147..87c37b08 100644 --- a/main_utils.py +++ b/main_utils.py @@ -1139,24 +1139,37 @@ def print_num_dataset(data): def init_layer_weight_ratio(model, parser_args): if parser_args.arch.lower() == 'resnet20': - # load smart ratio - sr = pd.read_csv("per_layer_sparsity_resnet20/smart_ratio.csv") - #import pdb; pdb.set_trace() - if parser_args.target_sparsity == 3.72: - p_arr = (sr['3.72'].array)/100 - elif parser_args.target_sparsity == 1.44: - p_arr = (sr['1.44'].array)/100 - else: - raise NotImplementedError + if parser_args.algo != 'pt_sr': + raise ValueError + + if parser_args.sr_version == 3: + # load smart ratio + sr = pd.read_csv("per_layer_sparsity_resnet20/smart_ratio.csv") + #import pdb; pdb.set_trace() + if parser_args.target_sparsity == 3.72: + p_arr = (sr['3.72'].array)/100 + elif parser_args.target_sparsity == 1.44: + p_arr = (sr['1.44'].array)/100 + else: + raise NotImplementedError - # optionally, change the smart ratio - hc = pd.read_csv("per_layer_sparsity_resnet20/hc_iter.csv") - if parser_args.target_sparsity == 3.72: - p_arr[0], p_arr[-1] = hc['3_72'].array[0]/100, hc['3_72'].array[-1]/100 - elif parser_args.target_sparsity == 1.44: - p_arr[0], p_arr[-1] = hc['1_4'].array[0]/100, hc['1_4'].array[-1]/100 - else: - raise NotImplementedError + + # optionally, change the smart ratio + hc = pd.read_csv("per_layer_sparsity_resnet20/hc_iter.csv") + if parser_args.target_sparsity == 3.72: + p_arr[0], p_arr[-1] = hc['3_72'].array[0]/100, hc['3_72'].array[-1]/100 + elif parser_args.target_sparsity == 1.44: + p_arr[0], p_arr[-1] = hc['1_4'].array[0]/100, hc['1_4'].array[-1]/100 + else: + raise NotImplementedError + + if parser_args.sr_version == 6: + # optionally, change the smart ratio + grid = pd.read_csv("per_layer_sparsity_resnet20/grid_best.csv") + if parser_args.target_sparsity == 1.44: + p_arr = (grid['1_4'].array)/100 + else: + raise NotImplementedError # for each layer, set the initial layer_weight_ratio conv_layers, linear_layers = get_layers(parser_args.arch, model) diff --git a/per_layer_sparsity_resnet20/grid_best.csv b/per_layer_sparsity_resnet20/grid_best.csv new file mode 100644 index 00000000..612fdd8f --- /dev/null +++ b/per_layer_sparsity_resnet20/grid_best.csv @@ -0,0 +1,21 @@ +1_4 +54.90740585327149 +11.590277671813965 +10.85243034362793 +8.723958969116211 +7.769097328186035 +7.89930534362793 +6.204861164093018 +5.184895515441895 +3.2552082538604736 +3.3420138359069824 +2.398003339767456 +2.018229007720948 +1.7144097089767456 +1.1773003339767456 +0.5638671875 +0.336371511220932 +0.347222238779068 +0.3146701455116272 +0.1197265625 +16.875001907348636 diff --git a/per_layer_sparsity_resnet20/smart_ratio_v6_lr1e-6.csv b/per_layer_sparsity_resnet20/smart_ratio_v6_lr1e-6.csv new file mode 100644 index 00000000..53fdaf4c --- /dev/null +++ b/per_layer_sparsity_resnet20/smart_ratio_v6_lr1e-6.csv @@ -0,0 +1,21 @@ +1,10,30,70,149 +0.5532407407407407,0.5486111111111112,0.5023148148148148,0.5509259259259259,0.6041666666666666 +0.10980902777777778,0.1171875,0.12109375,0.12022569444444445,0.1015625 +0.10633680555555555,0.11328125,0.1037326388888889,0.10503472222222222,0.10243055555555555 +0.08897569444444445,0.08289930555555555,0.07942708333333333,0.08680555555555555,0.08289930555555555 +0.07769097222222222,0.07725694444444445,0.07855902777777778,0.07074652777777778,0.0802951388888889 +0.07161458333333333,0.0881076388888889,0.0720486111111111,0.0920138888888889,0.0920138888888889 +0.058159722222222224,0.05642361111111111,0.0625,0.0703125,0.05381944444444445 +0.05186631944444445,0.04709201388888889,0.04991319444444445,0.057942708333333336,0.052734375 +0.03211805555555555,0.030056423611111112,0.03396267361111111,0.030924479166666668,0.03342013888888889 +0.03396267361111111,0.033854166666666664,0.037543402777777776,0.03700086805555555,0.035807291666666664 +0.030381944444444444,0.021918402777777776,0.023220486111111112,0.023003472222222224,0.0224609375 +0.020724826388888888,0.021050347222222224,0.020073784722222224,0.021592881944444444,0.019639756944444444 +0.017252604166666668,0.016167534722222224,0.017144097222222224,0.014973958333333334,0.014756944444444444 +0.010904947916666666,0.012044270833333334,0.01025390625,0.01171875,0.012044270833333334 +0.005859375,0.005805121527777778,0.005018446180555556,0.004991319444444444,0.005886501736111111 +0.003634982638888889,0.003689236111111111,0.0037706163194444445,0.0037977430555555555,0.004204644097222222 +0.004340277777777778,0.005886501736111111,0.007459852430555556,0.009765625,0.009955512152777778 +0.0035807291666666665,0.0033365885416666665,0.003607855902777778,0.0035807291666666665,0.0033365885416666665 +0.0010850694444444445,0.0038248697916666665,0.004720052083333333,0.006130642361111111,0.005750868055555556 +0.171875,0.153125,0.121875,0.0734375,0.053125 From a90752422b80d711e748896a2040fcbc96663115 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 28 Feb 2022 02:08:21 +0000 Subject: [PATCH 25/30] srV6 re-run --- cifar_exec_GD.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index e688756e..25c8dca9 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -213,7 +213,7 @@ BLOCK ## Step 2: train the model from the obtained smart ratio -:< "$log_root$epoch$log_end" 2>&1 & + --subfolder "$subfolder_root$epoch" > "$log_root$epoch$log_end" 2>&1 & done -BLOCK +#BLOCK From 9bf119c02e447a7fd0b09b9bbd2a70405c81623c Mon Sep 17 00:00:00 2001 From: root Date: Mon, 28 Feb 2022 02:17:58 +0000 Subject: [PATCH 26/30] comment srv6 --- cifar_exec_GD.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 25c8dca9..af5b227f 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -213,7 +213,7 @@ BLOCK ## Step 2: train the model from the obtained smart ratio -#:< "$log_root$epoch$log_end" 2>&1 & done -#BLOCK +BLOCK From 9e3b354d15fea627bfb9ed263fd15ba1bdae6232 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 28 Feb 2022 03:29:20 +0000 Subject: [PATCH 27/30] resnet18, cifar10 config file for GM --- .../sparsity_0_5_85lam6_iter_20_adam.yml | 61 +++++++++++++++++++ .../cifar10/sparsity_5_3lam6_iter_20_adam.yml | 61 +++++++++++++++++++ .../sparsity_5_85lam6_iter_20_adam.yml | 61 +++++++++++++++++++ configs/sr/resnet18/cifar10_resnet18_sr.yml | 46 ++++++++++++++ .../resnet18/cifar10_resnet18_training.yml | 39 ++++++++++++ 5 files changed, 268 insertions(+) create mode 100644 configs/hypercube/resnet18/cifar10/sparsity_0_5_85lam6_iter_20_adam.yml create mode 100644 configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml create mode 100644 configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml create mode 100644 configs/sr/resnet18/cifar10_resnet18_sr.yml create mode 100644 configs/training/resnet18/cifar10_resnet18_training.yml diff --git a/configs/hypercube/resnet18/cifar10/sparsity_0_5_85lam6_iter_20_adam.yml b/configs/hypercube/resnet18/cifar10/sparsity_0_5_85lam6_iter_20_adam.yml new file mode 100644 index 00000000..24352814 --- /dev/null +++ b/configs/hypercube/resnet18/cifar10/sparsity_0_5_85lam6_iter_20_adam.yml @@ -0,0 +1,61 @@ +#subfolder: resnet18_cifar10_hc_sparsity_5_7lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 20 + +# Architecture +arch: cResNet18 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet18_cifar10_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 0.5 +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000085 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml b/configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml new file mode 100644 index 00000000..34fee404 --- /dev/null +++ b/configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml @@ -0,0 +1,61 @@ +#subfolder: resnet18_cifar10_hc_sparsity_5_7lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 20 + +# Architecture +arch: cResNet18 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet18_cifar10_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000003 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml b/configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml new file mode 100644 index 00000000..350d2293 --- /dev/null +++ b/configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml @@ -0,0 +1,61 @@ +#subfolder: resnet18_cifar10_hc_sparsity_5_7lam6 +trial_num: 1 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 20 + +# Architecture +arch: cResNet18 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet18_cifar10_hc + +# ===== Learning Rate Policy ======== # +optimizer: adam +lr: 0.01 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5.0 +unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000085 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 3 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/sr/resnet18/cifar10_resnet18_sr.yml b/configs/sr/resnet18/cifar10_resnet18_sr.yml new file mode 100644 index 00000000..e81b11a8 --- /dev/null +++ b/configs/sr/resnet18/cifar10_resnet18_sr.yml @@ -0,0 +1,46 @@ +# subfolder: sr +algo: 'hc_iter' +random_subnet: True +# smart_ratio: 0.98 # 2% sparsity + +# Architecture +arch: cResNet18 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet18_cifar10_sc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +# doesn't matter +# lr: 0.1 +# lr_policy: cosine_lr +fine_tune_lr: 0.1 +fine_tune_lr_policy: cosine_lr #multistep_lr # cosine_lr + +# ===== Network training config ===== # +epochs: 200 +weight_decay: 0.0005 +momentum: 0.9 +batch_size: 128 # 256 +bias: False + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +init: kaiming_normal # signed_constant +mode: fan_in +nonlinearity: relu +# prune_rate: 0.5 +scale_fan: True +freeze_weights: True + +# ===== Hardware setup ===== # +workers: 4 +gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/training/resnet18/cifar10_resnet18_training.yml b/configs/training/resnet18/cifar10_resnet18_training.yml new file mode 100644 index 00000000..70f9a5e2 --- /dev/null +++ b/configs/training/resnet18/cifar10_resnet18_training.yml @@ -0,0 +1,39 @@ +#subfolder: resnet18_cifar10_weight_training +trial_num: 1 + + +# algorithm +algo: 'hc_iter' +weight_training: True + +# Architecture +arch: cResNet18 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet18_cifar10_training + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 +lr_policy: cosine_lr #multistep_lr #cosine_lr #constant_lr + +# ===== Network training config ===== # +epochs: 200 +wd: 0.0005 +momentum: 0.9 +batch_size: 128 + + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_rate: -1 +init: kaiming_normal +scale_fan: True + +# ===== Hardware setup ===== # +workers: 4 +gpu: 1 + From bd4746337ec5bffbacec7e786c5402bb2aa46b7d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 28 Feb 2022 03:31:19 +0000 Subject: [PATCH 28/30] srvN for resent20, gm/sr for resnet18 --- cifar_exec_GD.sh | 25 ++++++++++++++++++++----- grid_search_SR.py | 3 ++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 82c9e58f..8700aed6 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -30,7 +30,7 @@ done BLOCK # Gem-Miner (hypercube) -:< Date: Mon, 28 Feb 2022 14:27:55 +0000 Subject: [PATCH 29/30] merge --- grid_search_SR.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/grid_search_SR.py b/grid_search_SR.py index 3449bb94..39fd265e 100644 --- a/grid_search_SR.py +++ b/grid_search_SR.py @@ -13,10 +13,6 @@ best_acc = 0 best_sp = 0 for idx in range(1, 145): -<<<<<<< HEAD -#for idx in range(1, 49): -======= ->>>>>>> 9bf119c02e447a7fd0b09b9bbd2a70405c81623c path = 'results/SR_grid_sp_1_44_{}'.format(idx) csv_files = glob.glob(os.path.join(path, "*/**.csv")) From 41a1bac82236fa7a9a96c61da0a9aa046570fb78 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 1 Mar 2022 10:16:41 +0000 Subject: [PATCH 30/30] sanity check for cifar10, resnet18, gm --- cifar_exec_GD.sh | 32 +++++++++++++++++-- .../cifar10/sparsity_5_3lam6_iter_20_adam.yml | 4 +-- .../sparsity_5_85lam6_iter_20_adam.yml | 4 +-- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index 01dbf8bf..2f1d2579 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -63,8 +63,8 @@ BLOCK #conf_file="configs/hypercube/resnet18/cifar10/sparsity_5_1lam6_iter_20_adam_1e4.yml" #subfolder_root="resnet18_cifar10_hc_sparsity_5_iter_20_adam_1lam6_1e4" -conf_file="configs/hypercube/resnet18/cifar10/sparsity_5_1lam6_iter_20_adam_1e3.yml" -subfolder_root="resnet18_cifar10_hc_sparsity_5_iter_20_adam_1lam6_1e3" +#conf_file="configs/hypercube/resnet18/cifar10/sparsity_5_1lam6_iter_20_adam_1e3.yml" +#subfolder_root="resnet18_cifar10_hc_sparsity_5_iter_20_adam_1lam6_1e3" # 2% sparsity @@ -78,12 +78,38 @@ subfolder_root="resnet18_cifar10_hc_sparsity_5_iter_20_adam_1lam6_1e3" #conf_file="configs/hypercube/resnet18/cifar10/sparsity_0_5_85lam6_iter_20_adam.yml" #subfolder_root="resnet18_cifar10_hc_sparsity_0_5_iter_20_adam_85lam6" + +########################################################################### +####### Sanity check +########################################################################### + +# 5% sparsity +#conf_file="configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml" +#subfolder_root="sanity_check_resnet18_cifar10_hc_sparsity_5" +# 2% sparsity +#conf_file="configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml" +#subfolder_root="sanity_check_resnet18_cifar10_hc_sparsity_2" +# 0.5% sparsity +#conf_file="configs/hypercube/resnet18/cifar10/sparsity_0_5_85lam6_iter_20_adam.yml" +#subfolder_root="sanity_check_resnet18_cifar10_hc_sparsity_0_5" + +#### invert score +# 5% sparsity +#conf_file="configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam_invert.yml" +#subfolder_root="sanity_check_resnet18_cifar10_hc_sparsity_5_invert" +# 2% sparsity +#conf_file="configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam_invert.yml" +#subfolder_root="sanity_check_resnet18_cifar10_hc_sparsity_2_invert" +# 0.5% sparsity +conf_file="configs/hypercube/resnet18/cifar10/sparsity_0_5_85lam6_iter_20_adam_invert.yml" +subfolder_root="sanity_check_resnet18_cifar10_hc_sparsity_0_5_invert" + log_end="_log" python main.py \ --config "$conf_file" \ --subfolder "$subfolder_root" > "$subfolder_root$log_end" 2>&1 #& -BLOCK +#BLOCK diff --git a/configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml b/configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml index 34fee404..06a59e98 100644 --- a/configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml +++ b/configs/hypercube/resnet18/cifar10/sparsity_5_3lam6_iter_20_adam.yml @@ -52,10 +52,10 @@ lmbda: 0.000003 # ===== Hardware setup ===== # workers: 4 -gpu: 2 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # -skip_sanity_checks: True +skip_sanity_checks: False diff --git a/configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml b/configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml index 350d2293..e4c772e4 100644 --- a/configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml +++ b/configs/hypercube/resnet18/cifar10/sparsity_5_85lam6_iter_20_adam.yml @@ -52,10 +52,10 @@ lmbda: 0.0000085 # ===== Hardware setup ===== # workers: 4 -gpu: 3 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False # ==== sanity check ==== # -skip_sanity_checks: True +skip_sanity_checks: False