diff --git a/args_helper.py b/args_helper.py index f5b0a487..36ec690e 100644 --- a/args_helper.py +++ b/args_helper.py @@ -720,6 +720,18 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="chg weights before sanity check" ) + parser.add_argument( + "--finetune_last_epochs", + action="store_true", + default=False, + help="used for adding finetune loss at last 10 epochs of HC" + ) + parser.add_argument( + "--finetuned", + action="store_true", + default=False, + help="used for finetuned loss (please set it false in the command line. The code will automatically turn on/off it" + ) parser.add_argument( "--fine-tune-optimizer", type=str, diff --git a/cifar_exec_GD.sh b/cifar_exec_GD.sh index c9c67803..9196afc8 100644 --- a/cifar_exec_GD.sh +++ b/cifar_exec_GD.sh @@ -1,5 +1,6 @@ #export cuda_visible_devices=3 +# ResNet-18 #### ResNet-18 #python main.py --config configs/hypercube/resnet18/resnet18_sc_hypercube_reg.yml # 93.17% at 150 epoch @@ -12,13 +13,41 @@ -### ResNet-20 +# ResNet-20 +#python main.py --config configs/ep/resnet20/resnet20_sc_ep.yml #> log_EP_sparsity_50 2>&1 +## HC for multiple trials +:< log_hc_sparsity_3_72_t2 2>&1 +python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t3.yml > log_hc_sparsity_3_72_t3 2>&1 +python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t4.yml > log_hc_sparsity_3_72_t4 2>&1 +python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t5.yml > log_hc_sparsity_3_72_t5 2>&1 +BLOCK + +:< log_hc_sparsity_0_59_t2 2>&1 +python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t3.yml > log_hc_sparsity_0_59_t3 2>&1 +python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t4.yml > log_hc_sparsity_0_59_t4 2>&1 +python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_0_59_t5.yml > log_hc_sparsity_0_59_t5 2>&1 +BLOCK + +## ran in finetune_loss repo +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t1.yml > log_hc_sparsity_1_35_t1 2>&1 +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t2.yml > log_hc_sparsity_1_35_t2 2>&1 + +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t3.yml > log_hc_sparsity_1_35_t3 2>&1 +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t4.yml > log_hc_sparsity_1_35_t4 2>&1 +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_1_35_t5.yml > log_hc_sparsity_1_35_t5 2>&1 + + + +## EP +#python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_50.yml > log_EP_sparsity_50 2>&1 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t1.yml #> log_hc_sparsity_3_72_t1 2>&1 # HC for multiple trials #:< log_hc_sparsity_3_72_t1 2>&1 +#python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t1.yml > log_hc_sparsity_3_72_t1 2>&1 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t2.yml > log_hc_sparsity_3_72_t2 2>&1 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t3.yml > log_hc_sparsity_3_72_t3 2>&1 #python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3_72_t4.yml > log_hc_sparsity_3_72_t4 2>&1 @@ -42,6 +71,7 @@ python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3 #python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_1_44.yml > log_EP_sparsity_1_44 2>&1 #python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_0_59.yml > log_EP_sparsity_0_59 2>&1 #python main.py --config configs/ep/resnet20/resnet20_sc_ep_sparsity_0_15.yml > log_EP_sparsity_0_15 2>&1 + #python main.py --config configs/ep/resnet20/resnet20_sc_global_ep.yml @@ -59,11 +89,40 @@ python main.py --config configs/hypercube/resnet20/error_bar/resnet20_sparsity_3 #python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag.yml > log_target_sparsity_50_without_flag_lam_0 2>&1 +## adding finetune for denser models +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag_with_finetune.yml > log_target_sparsity_5_without_unflag_with_finetune_lam_3e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune.yml > log_target_sparsity_20_without_unflag_with_finetune_lam_1e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag_with_finetune.yml > log_target_sparsity_50_without_unflag_with_finetune_lam_0 2>&1 + +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_5_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_5_without_unflag_with_finetune_last_10_epochs_lam_3e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_20_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_20_without_unflag_with_finetune_last_10_epochs_lam_1e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/resnet20_quantized_iter_hc_target_sparsity_50_without_unflag_with_finetune_last_10_epochs.yml > log_target_sparsity_50_without_unflag_with_finetune_last_10_epochs_lam_0 2>&1 ## testing adding finetune loss +#python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune.yml > log_no_finetune_5e-5 2>&1 +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs_without_unflag.yml > log_finetune_1e-4_lam_1_num_5_last_10_epochs_without_unflag 2>&1 +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs.yml > log_finetune_1e-4_lam_1_num_5_last_10_epochs 2>&1 +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs_without_unflag.yml > log_finetune_5e-5_lam_1_num_5_last_10_epochs_without_unflag 2>&1 +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_last_10_epochs.yml > log_finetune_5e-5_lam_1_num_5_last_10_epochs 2>&1 + + +#python main.py --config configs/hypercube/resnet20/finetune_check/no_finetune_without_unflag.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_without_unflag.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_5_without_unflag.yml + + + #python main.py --config config1.yml --run_idx 1 # +#python main.py --config config1.yml --run_idx 0 # #python main.py --config config2.yml --run_idx 2 +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_10.yml +#python main.py --config configs/hypercube/resnet20/finetune_check/finetune_lam_0_1_num_10.yml + + + + + diff --git a/config1.yml b/config1.yml new file mode 100644 index 00000000..bd0c975f --- /dev/null +++ b/config1.yml @@ -0,0 +1,62 @@ +subfolder: quantize_0_5_finetune_loss_test +lam_finetune_loss: 0 #100 +num_step_finetune: 10 + +# Hypercube optimization +algo: 'hc_iter' +# effectively prune once in the entire process +iter_period: 100 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +prune_rate: 0.982 #0.995 +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0 #0.00005 # 0.01 #0.0001 #0.000001 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: False diff --git a/configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_without_unflag.yml b/configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_without_unflag.yml new file mode 100644 index 00000000..4663aaa7 --- /dev/null +++ b/configs/hypercube/resnet20/finetune_check/finetune_lam_1_num_5_without_unflag.yml @@ -0,0 +1,63 @@ +subfolder: target_sparsity_1_4_finetune_lam_1_num_5_without_unflag +lam_finetune_loss: 1 +num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.4 +# decide if you want to "unflag" +#unflag_before_finetune: True +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/trainers/default.py b/trainers/default.py index 9349cfa1..66d60af8 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -8,12 +8,15 @@ from utils.logging import AverageMeter, ProgressMeter from utils.net_utils import get_regularization_loss, prune, get_layers +from main_utils import get_model + from torch import optim __all__ = ["train", "validate", "modifier"] def train(train_loader, model, criterion, optimizer, epoch, args, writer): + batch_time = AverageMeter("Time", ":6.3f") data_time = AverageMeter("Data", ":6.3f") losses = AverageMeter("Loss", ":.3f") @@ -33,13 +36,11 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): num_batches = len(train_loader) end = time.time() for i, (images, target) in tqdm.tqdm( - enumerate(train_loader), ascii=True, total=len(train_loader) + enumerate(train_loader), ascii=True, total=len(train_loader) ): # measure data loading time data_time.update(time.time() - end) - #print(images.shape, target.shape) - if args.gpu is not None: images = images.cuda(args.gpu, non_blocking=True) @@ -62,85 +63,45 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer): loss = criterion(output, target) if args.lam_finetune_loss > 0: - - model2 = copy.deepcopy(model) - #delta_list = {} - conv, linear = get_layers(arch=args.arch, model=model) - conv2, linear2 = get_layers(arch=args.arch, model=model2) - layer_list = [*conv, *linear] - layer_list2 = [*conv2, *linear2] - - # pdb.set_trace() - # clone parameters from model - # model.modules(), model2.modules()): - for m_from, m_to in zip(layer_list, layer_list2): - m_to.scores.data = m_from.scores.data.clone() - m_to.flag.data = m_from.flag.data.clone() - - # turn on gradient for weight, turn off gradient for mask - for name, params in model2.named_parameters(): - print(name) - if "weight" in name: - params.requires_grad = True - elif "score" in name: - params.requires_grad = False - - meta_optimizer = optim.SGD( - model2.parameters(), lr=0.1, momentum=0.9) - # update for several steps - for i in range(args.num_step_finetune): - # forward and backward to update net_pi grad. - loss_updated_model = criterion(model2(images), target) - meta_optimizer.zero_grad() - loss_updated_model.backward() # retain_graph=True) - meta_optimizer.step() - - # go back to original setting - for name, params in model.named_parameters(): - if "weight" in name: - params.requires_grad = False - elif "score" in name: - params.requires_grad = True - - loss_updated_model = criterion(model2(images), target) - - ''' - # define updates - for i, (name, params) in enumerate(model2.named_parameters()): - if "weight" in name: - delta_list[i] = torch.autograd.grad(loss, params) - - model3 = clone(model) - for i, (name, params) in enumerate(model3.named_parameters()): - if "weight" in name: - params -= alpha * delta_list[i] - output3 = model3(images) - ''' - - finetune_loss = args.lam_finetune_loss * loss_updated_model - print('original loss: ', loss) - print('finetune loss: ', finetune_loss) - - print('For model2') - for name, params in model2.named_parameters(): - if params.requires_grad: - # pdb.set_trace() - grad = torch.autograd.grad( - finetune_loss, params, retain_graph=True)[0].data - print(name, 'autograd(): ', - (grad != torch.zeros_like(grad)).any().item()) - - print('For model') - for name, params in model.named_parameters(): - if params.requires_grad: - # pdb.set_trace() - print(name) - #grad =torch.autograd.grad(finetune_loss, params, retain_graph=True, allow_unused=True)[0].data - #print(name, 'autograd(): ', (grad != torch.zeros_like(grad)).any().item()) - - loss += finetune_loss - - pdb.set_trace() + if args.finetune_last_epochs and epoch < args.epochs - 10: + pass + else: + model2 = copy.deepcopy(model) + # turn on gradient for weight, turn off gradient for mask + for name, params in model2.named_parameters(): + # print(name) + if "weight" in name: + params.requires_grad = True + elif "score" in name: + params.requires_grad = False + + meta_optimizer = optim.SGD( + model2.parameters(), lr=0.1, momentum=0.9) + # update for several steps + for i in range(args.num_step_finetune): + loss_updated_model = criterion(model2(images), target) + meta_optimizer.zero_grad() + loss_updated_model.backward() # retain_graph=True) + meta_optimizer.step() + + conv, linear = get_layers(arch=args.arch, model=model) + conv2, linear2 = get_layers(arch=args.arch, model=model2) + layer_list = [*conv, *linear] + layer_list2 = [*conv2, *linear2] + + # load updated weight from model2 to model + for m_from, m_to in zip(layer_list2, layer_list): + m_to.weight_ft.data = m_from.weight.data + + # compute loss on the finetuned weights + args.finetuned = True + finetune_loss = args.lam_finetune_loss * \ + criterion(model(images), target) + args.finetuned = False + #print('original loss: ', loss) + #print('finetune loss: ', finetune_loss) + + loss += finetune_loss regularization_loss = torch.tensor(0) if args.regularization: @@ -208,7 +169,7 @@ def validate(val_loader, model, criterion, args, writer, epoch): with torch.no_grad(): end = time.time() for i, (images, target) in tqdm.tqdm( - enumerate(val_loader), ascii=True, total=len(val_loader) + enumerate(val_loader), ascii=True, total=len(val_loader) ): if args.gpu is not None: images = images.cuda(args.gpu, non_blocking=True) diff --git a/utils/conv_type.py b/utils/conv_type.py index d58b67fc..4b931fab 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -115,6 +115,9 @@ def __init__(self, *args, **kwargs): # dummy variable just so other things don't break self.bias_scores = nn.Parameter(torch.Tensor(1)) + # storage for finetuned weights + self.weight_ft = nn.Parameter(torch.zeros(self.weight.size())) + # prune scores below this for global EP in bottom-k self.scores_prune_threshold = -np.inf self.bias_scores_prune_threshold = -np.inf @@ -150,6 +153,7 @@ def __init__(self, *args, **kwargs): if parser_args.freeze_weights: # NOTE: turn the gradient on the weights off self.weight.requires_grad = False + self.weight_ft.requires_grad = False self.flag.requires_grad = False self.bias_flag.requires_grad = False if parser_args.bias: @@ -191,10 +195,17 @@ def forward(self, x): if parser_args.algo in ['imp']: # no STE, no subnet. Mask is handled outside - w = self.weight + if parser_args.finetuned: + print("Are we sure we are using finetuned loss for imp?") + w = self.weight_ft + else: + w = self.weight b = self.bias else: - w = self.weight * subnet + if parser_args.finetuned: + w = self.weight_ft * subnet + else: + w = self.weight * subnet if parser_args.bias: b = self.bias * bias_subnet else: