From 5605a21c1ed83ab80880b9e4a96d45bbd4054bf9 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 11:51:16 -0500 Subject: [PATCH 01/33] adding data loader for ffcv imagenet bunch of things will break but we'll have to fix them along the way. --- data/ffcv_imagenet.py | 118 +++++++++ ffcv_installation.sh | 37 +++ ffcv_org_train_imagenet.py | 511 +++++++++++++++++++++++++++++++++++++ 3 files changed, 666 insertions(+) create mode 100644 data/ffcv_imagenet.py create mode 100644 ffcv_installation.sh create mode 100644 ffcv_org_train_imagenet.py diff --git a/data/ffcv_imagenet.py b/data/ffcv_imagenet.py new file mode 100644 index 00000000..a37ace6e --- /dev/null +++ b/data/ffcv_imagenet.py @@ -0,0 +1,118 @@ +""" + script to create ffcv_imagenet train loader +""" + +import os +import torch +import torchvision +from torchvision import datasets, transforms +import random +from torch.utils.data.sampler import SubsetRandomSampler +from args_helper import parser_args + +import torch.multiprocessing +torch.multiprocessing.set_sharing_strategy("file_system") + +class ImageNet: + def __init__(self, args): + super(ImageNet, self).__init__() + + # data_root = os.path.join(parser_args.data, "imagenet") + # put ffcv path here + data_root = parser_args.data + + use_cuda = torch.cuda.is_available() + + # Data loading code + kwargs = {"num_workers": args.num_workers, "pin_memory": True} if use_cuda else {} + + # Data loading code + traindir = os.path.join(data_root, "train") + valdir = os.path.join(data_root, "val") + + normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + IMAGENET_MEAN = np.array([0.485, 0.456, 0.406]) * 255 + IMAGENET_STD = np.array([0.229, 0.224, 0.225]) * 255 + DEFAULT_CROP_RATIO = 224/256 + + + self.train_loader = self.create_train_loader() + self.val_loader = self.create_val_loader() + # madry does this but I don't think we need to + # self.model, self.scaler = self.create_model_and_scaler() + + def create_train_loader(self, train_dataset, num_workers, batch_size, + distributed, in_memory): + train_path = Path(train_dataset) + assert train_path.is_file() + + res = self.get_resolution(epoch=0) + self.decoder = RandomResizedCropRGBImageDecoder((res, res)) + image_pipeline: List[Operation] = [ + self.decoder, + RandomHorizontalFlip(), + ToTensor(), + ToDevice(ch.device(parser_args.gpu), non_blocking=True), + ToTorchImage(), + NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) + ] + + label_pipeline: List[Operation] = [ + IntDecoder(), + ToTensor(), + Squeeze(), + ToDevice(ch.device(parser_args.gpu), non_blocking=True) + ] + + order = OrderOption.RANDOM if distributed else OrderOption.QUASI_RANDOM + loader = Loader(train_dataset, + batch_size=batch_size, + num_workers=num_workers, + order=order, + os_cache=in_memory, + drop_last=True, + pipelines={ + 'image': image_pipeline, + 'label': label_pipeline + }, + distributed=distributed) + + return loader + + + def create_val_loader(self, val_dataset, num_workers, batch_size, + resolution, distributed): + val_path = Path(val_dataset) + assert val_path.is_file() + res_tuple = (resolution, resolution) + cropper = CenterCropRGBImageDecoder(res_tuple, ratio=DEFAULT_CROP_RATIO) + image_pipeline = [ + cropper, + ToTensor(), + ToDevice(ch.device(parser_args.gpu), non_blocking=True), + ToTorchImage(), + NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) + ] + + label_pipeline = [ + IntDecoder(), + ToTensor(), + Squeeze(), + ToDevice(ch.device(parser_args.gpu), + non_blocking=True) + ] + + loader = Loader(val_dataset, + batch_size=batch_size, + num_workers=num_workers, + order=OrderOption.SEQUENTIAL, + drop_last=False, + pipelines={ + 'image': image_pipeline, + 'label': label_pipeline + }, + distributed=distributed) + return loader \ No newline at end of file diff --git a/ffcv_installation.sh b/ffcv_installation.sh new file mode 100644 index 00000000..a4e0cf60 --- /dev/null +++ b/ffcv_installation.sh @@ -0,0 +1,37 @@ +# set up ffcv script + +docker run --gpus all --ipc=host --name kartik_imagenet -it -v /hdd1/ILSVRC2012:/data/imagenet -v /hdd3/ksreenivasan:/workspace nvcr.io/nvidia/pytorch:21.12-py3 + +sudo docker pull nvcr.io/nvidia/pytorch:22.01-py3 + +# to fix dependencies for ffcv +apt-get install ffmpeg libsm6 libxext6 -y + +# clone repo +git clone git@github.com:libffcv/ffcv-imagenet.git + +# install dependencies +cd ffcv-imagenet +pip install -r requirements.txt + +# set environment variables +export IMAGENET_DIR=/data/imagenet +export WRITE_DIR=/workspace/ffcv-imagenet/data +mkdir data + +# create ffcv data + +# Serialize images with: +# - 500px side length maximum +# - 50% JPEG encoded, 90% raw pixel values +# - quality=90 JPEGs +./write_imagenet.sh 500 0.50 90 + + +python train_imagenet.py --config-file rn18_configs/rn18_88_epochs.yaml \ + --data.train_dataset=/workspace/ffcv-imagenet/data/train_500_0.50_90.ffcv \ + --data.val_dataset=/workspace/ffcv-imagenet/data/val_500_0.50_90.ffcv \ + --data.num_workers=12 --data.in_memory=1 \ + --logging.folder=/workspace/ffcv-imagenet/logs + +# start exec: 5:24pm (with batch size 512) \ No newline at end of file diff --git a/ffcv_org_train_imagenet.py b/ffcv_org_train_imagenet.py new file mode 100644 index 00000000..0892d3ce --- /dev/null +++ b/ffcv_org_train_imagenet.py @@ -0,0 +1,511 @@ +import torch as ch +from torch.cuda.amp import GradScaler +from torch.cuda.amp import autocast +import torch.nn.functional as F +import torch.distributed as dist + +ch.backends.cudnn.benchmark = True +ch.autograd.profiler.emit_nvtx(False) +ch.autograd.profiler.profile(False) + +from torchvision import models +import torchmetrics +import numpy as np +from tqdm import tqdm + +import os +import time +import json +from uuid import uuid4 +from typing import List +from pathlib import Path +from argparse import ArgumentParser + +from fastargs import get_current_config +from fastargs.decorators import param +from fastargs import Param, Section +from fastargs.validation import And, OneOf + +from ffcv.pipeline.operation import Operation +from ffcv.loader import Loader, OrderOption +from ffcv.transforms import ToTensor, ToDevice, Squeeze, NormalizeImage, \ + RandomHorizontalFlip, ToTorchImage +from ffcv.fields.rgb_image import CenterCropRGBImageDecoder, \ + RandomResizedCropRGBImageDecoder +from ffcv.fields.basics import IntDecoder + +Section('model', 'model details').params( + arch=Param(And(str, OneOf(models.__dir__())), default='resnet18'), + pretrained=Param(int, 'is pretrained? (1/0)', default=0) +) + +Section('resolution', 'resolution scheduling').params( + min_res=Param(int, 'the minimum (starting) resolution', default=160), + max_res=Param(int, 'the maximum (starting) resolution', default=160), + end_ramp=Param(int, 'when to stop interpolating resolution', default=0), + start_ramp=Param(int, 'when to start interpolating resolution', default=0) +) + +Section('data', 'data related stuff').params( + train_dataset=Param(str, '.dat file to use for training', required=True), + val_dataset=Param(str, '.dat file to use for validation', required=True), + num_workers=Param(int, 'The number of workers', required=True), + in_memory=Param(int, 'does the dataset fit in memory? (1/0)', required=True) +) + +Section('lr', 'lr scheduling').params( + step_ratio=Param(float, 'learning rate step ratio', default=0.1), + step_length=Param(int, 'learning rate step length', default=30), + lr_schedule_type=Param(OneOf(['step', 'cyclic']), default='cyclic'), + lr=Param(float, 'learning rate', default=0.5), + lr_peak_epoch=Param(int, 'Epoch at which LR peaks', default=2), +) + +Section('logging', 'how to log stuff').params( + folder=Param(str, 'log location', required=True), + log_level=Param(int, '0 if only at end 1 otherwise', default=1) +) + +Section('validation', 'Validation parameters stuff').params( + batch_size=Param(int, 'The batch size for validation', default=512), + resolution=Param(int, 'final resized validation image size', default=224), + lr_tta=Param(int, 'should do lr flipping/avging at test time', default=1) +) + +Section('training', 'training hyper param stuff').params( + eval_only=Param(int, 'eval only?', default=0), + batch_size=Param(int, 'The batch size', default=512), + optimizer=Param(And(str, OneOf(['sgd'])), 'The optimizer', default='sgd'), + momentum=Param(float, 'SGD momentum', default=0.9), + weight_decay=Param(float, 'weight decay', default=4e-5), + epochs=Param(int, 'number of epochs', default=30), + label_smoothing=Param(float, 'label smoothing parameter', default=0.1), + distributed=Param(int, 'is distributed?', default=0), + use_blurpool=Param(int, 'use blurpool?', default=0) +) + +Section('dist', 'distributed training options').params( + world_size=Param(int, 'number gpus', default=1), + address=Param(str, 'address', default='localhost'), + port=Param(str, 'port', default='12355') +) + +IMAGENET_MEAN = np.array([0.485, 0.456, 0.406]) * 255 +IMAGENET_STD = np.array([0.229, 0.224, 0.225]) * 255 +DEFAULT_CROP_RATIO = 224/256 + +@param('lr.lr') +@param('lr.step_ratio') +@param('lr.step_length') +@param('training.epochs') +def get_step_lr(epoch, lr, step_ratio, step_length, epochs): + if epoch >= epochs: + return 0 + + num_steps = epoch // step_length + return step_ratio**num_steps * lr + +@param('lr.lr') +@param('training.epochs') +@param('lr.lr_peak_epoch') +def get_cyclic_lr(epoch, lr, epochs, lr_peak_epoch): + xs = [0, lr_peak_epoch, epochs] + ys = [1e-4 * lr, lr, 0] + return np.interp([epoch], xs, ys)[0] + +class BlurPoolConv2d(ch.nn.Module): + def __init__(self, conv): + super().__init__() + default_filter = ch.tensor([[[[1, 2, 1], [2, 4, 2], [1, 2, 1]]]]) / 16.0 + filt = default_filter.repeat(conv.in_channels, 1, 1, 1) + self.conv = conv + self.register_buffer('blur_filter', filt) + + def forward(self, x): + blurred = F.conv2d(x, self.blur_filter, stride=1, padding=(1, 1), + groups=self.conv.in_channels, bias=None) + return self.conv.forward(blurred) + +class ImageNetTrainer: + @param('training.distributed') + def __init__(self, gpu, distributed): + self.all_params = get_current_config() + self.gpu = gpu + + self.uid = str(uuid4()) + + if distributed: + self.setup_distributed() + + self.train_loader = self.create_train_loader() + self.val_loader = self.create_val_loader() + self.model, self.scaler = self.create_model_and_scaler() + self.create_optimizer() + self.initialize_logger() + + + @param('dist.address') + @param('dist.port') + @param('dist.world_size') + def setup_distributed(self, address, port, world_size): + os.environ['MASTER_ADDR'] = address + os.environ['MASTER_PORT'] = port + + dist.init_process_group("nccl", rank=self.gpu, world_size=world_size) + ch.cuda.set_device(self.gpu) + + def cleanup_distributed(self): + dist.destroy_process_group() + + @param('lr.lr_schedule_type') + def get_lr(self, epoch, lr_schedule_type): + lr_schedules = { + 'cyclic': get_cyclic_lr, + 'step': get_step_lr + } + + return lr_schedules[lr_schedule_type](epoch) + + # resolution tools + @param('resolution.min_res') + @param('resolution.max_res') + @param('resolution.end_ramp') + @param('resolution.start_ramp') + def get_resolution(self, epoch, min_res, max_res, end_ramp, start_ramp): + assert min_res <= max_res + + if epoch <= start_ramp: + return min_res + + if epoch >= end_ramp: + return max_res + + # otherwise, linearly interpolate to the nearest multiple of 32 + interp = np.interp([epoch], [start_ramp, end_ramp], [min_res, max_res]) + final_res = int(np.round(interp[0] / 32)) * 32 + return final_res + + @param('training.momentum') + @param('training.optimizer') + @param('training.weight_decay') + @param('training.label_smoothing') + def create_optimizer(self, momentum, optimizer, weight_decay, + label_smoothing): + assert optimizer == 'sgd' + + # Only do weight decay on non-batchnorm parameters + all_params = list(self.model.named_parameters()) + bn_params = [v for k, v in all_params if ('bn' in k)] + other_params = [v for k, v in all_params if not ('bn' in k)] + param_groups = [{ + 'params': bn_params, + 'weight_decay': 0. + }, { + 'params': other_params, + 'weight_decay': weight_decay + }] + + self.optimizer = ch.optim.SGD(param_groups, lr=1, momentum=momentum) + self.loss = ch.nn.CrossEntropyLoss(label_smoothing=label_smoothing) + + @param('data.train_dataset') + @param('data.num_workers') + @param('training.batch_size') + @param('training.distributed') + @param('data.in_memory') + def create_train_loader(self, train_dataset, num_workers, batch_size, + distributed, in_memory): + this_device = f'cuda:{self.gpu}' + train_path = Path(train_dataset) + assert train_path.is_file() + + res = self.get_resolution(epoch=0) + self.decoder = RandomResizedCropRGBImageDecoder((res, res)) + image_pipeline: List[Operation] = [ + self.decoder, + RandomHorizontalFlip(), + ToTensor(), + ToDevice(ch.device(this_device), non_blocking=True), + ToTorchImage(), + NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) + ] + + label_pipeline: List[Operation] = [ + IntDecoder(), + ToTensor(), + Squeeze(), + ToDevice(ch.device(this_device), non_blocking=True) + ] + + order = OrderOption.RANDOM if distributed else OrderOption.QUASI_RANDOM + loader = Loader(train_dataset, + batch_size=batch_size, + num_workers=num_workers, + order=order, + os_cache=in_memory, + drop_last=True, + pipelines={ + 'image': image_pipeline, + 'label': label_pipeline + }, + distributed=distributed) + + return loader + + @param('data.val_dataset') + @param('data.num_workers') + @param('validation.batch_size') + @param('validation.resolution') + @param('training.distributed') + def create_val_loader(self, val_dataset, num_workers, batch_size, + resolution, distributed): + this_device = f'cuda:{self.gpu}' + val_path = Path(val_dataset) + assert val_path.is_file() + res_tuple = (resolution, resolution) + cropper = CenterCropRGBImageDecoder(res_tuple, ratio=DEFAULT_CROP_RATIO) + image_pipeline = [ + cropper, + ToTensor(), + ToDevice(ch.device(this_device), non_blocking=True), + ToTorchImage(), + NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) + ] + + label_pipeline = [ + IntDecoder(), + ToTensor(), + Squeeze(), + ToDevice(ch.device(this_device), + non_blocking=True) + ] + + loader = Loader(val_dataset, + batch_size=batch_size, + num_workers=num_workers, + order=OrderOption.SEQUENTIAL, + drop_last=False, + pipelines={ + 'image': image_pipeline, + 'label': label_pipeline + }, + distributed=distributed) + return loader + + @param('training.epochs') + @param('logging.log_level') + def train(self, epochs, log_level): + for epoch in range(epochs): + res = self.get_resolution(epoch) + self.decoder.output_size = (res, res) + train_loss = self.train_loop(epoch) + + if log_level > 0: + extra_dict = { + 'train_loss': train_loss, + 'epoch': epoch + } + + self.eval_and_log(extra_dict) + + self.eval_and_log({'epoch':epoch}) + if self.gpu == 0: + ch.save(self.model.state_dict(), self.log_folder / 'final_weights.pt') + + def eval_and_log(self, extra_dict={}): + start_val = time.time() + stats = self.val_loop() + val_time = time.time() - start_val + if self.gpu == 0: + self.log(dict({ + 'current_lr': self.optimizer.param_groups[0]['lr'], + 'top_1': stats['top_1'], + 'top_5': stats['top_5'], + 'val_time': val_time + }, **extra_dict)) + + return stats + + @param('model.arch') + @param('model.pretrained') + @param('training.distributed') + @param('training.use_blurpool') + def create_model_and_scaler(self, arch, pretrained, distributed, use_blurpool): + scaler = GradScaler() + model = getattr(models, arch)(pretrained=pretrained) + def apply_blurpool(mod: ch.nn.Module): + for (name, child) in mod.named_children(): + if isinstance(child, ch.nn.Conv2d) and (np.max(child.stride) > 1 and child.in_channels >= 16): + setattr(mod, name, BlurPoolConv2d(child)) + else: apply_blurpool(child) + if use_blurpool: apply_blurpool(model) + + model = model.to(memory_format=ch.channels_last) + model = model.to(self.gpu) + + if distributed: + model = ch.nn.parallel.DistributedDataParallel(model, device_ids=[self.gpu]) + + return model, scaler + + @param('logging.log_level') + def train_loop(self, epoch, log_level): + model = self.model + model.train() + losses = [] + + lr_start, lr_end = self.get_lr(epoch), self.get_lr(epoch + 1) + iters = len(self.train_loader) + lrs = np.interp(np.arange(iters), [0, iters], [lr_start, lr_end]) + + iterator = tqdm(self.train_loader) + for ix, (images, target) in enumerate(iterator): + ### Training start + for param_group in self.optimizer.param_groups: + param_group['lr'] = lrs[ix] + + self.optimizer.zero_grad(set_to_none=True) + with autocast(): + output = self.model(images) + loss_train = self.loss(output, target) + + self.scaler.scale(loss_train).backward() + self.scaler.step(self.optimizer) + self.scaler.update() + ### Training end + + ### Logging start + if log_level > 0: + losses.append(loss_train.detach()) + + group_lrs = [] + for _, group in enumerate(self.optimizer.param_groups): + group_lrs.append(f'{group["lr"]:.3f}') + + names = ['ep', 'iter', 'shape', 'lrs'] + values = [epoch, ix, tuple(images.shape), group_lrs] + if log_level > 1: + names += ['loss'] + values += [f'{loss_train.item():.3f}'] + + msg = ', '.join(f'{n}={v}' for n, v in zip(names, values)) + iterator.set_description(msg) + ### Logging end + + if log_level > 0: + loss = ch.stack(losses).mean().cpu() + assert not ch.isnan(loss), 'Loss is NaN!' + return loss.item() + + @param('validation.lr_tta') + def val_loop(self, lr_tta): + model = self.model + model.eval() + + with ch.no_grad(): + with autocast(): + for images, target in tqdm(self.val_loader): + output = self.model(images) + if lr_tta: + output += self.model(ch.flip(images, dims=[3])) + + for k in ['top_1', 'top_5']: + self.val_meters[k](output, target) + + loss_val = self.loss(output, target) + self.val_meters['loss'](loss_val) + + stats = {k: m.compute().item() for k, m in self.val_meters.items()} + [meter.reset() for meter in self.val_meters.values()] + return stats + + @param('logging.folder') + def initialize_logger(self, folder): + self.val_meters = { + 'top_1': torchmetrics.Accuracy(compute_on_step=False).to(self.gpu), + 'top_5': torchmetrics.Accuracy(compute_on_step=False, top_k=5).to(self.gpu), + 'loss': MeanScalarMetric(compute_on_step=False).to(self.gpu) + } + + if self.gpu == 0: + folder = (Path(folder) / str(self.uid)).absolute() + folder.mkdir(parents=True) + + self.log_folder = folder + self.start_time = time.time() + + print(f'=> Logging in {self.log_folder}') + params = { + '.'.join(k): self.all_params[k] for k in self.all_params.entries.keys() + } + + with open(folder / 'params.json', 'w+') as handle: + json.dump(params, handle) + + def log(self, content): + print(f'=> Log: {content}') + if self.gpu != 0: return + cur_time = time.time() + with open(self.log_folder / 'log', 'a+') as fd: + fd.write(json.dumps({ + 'timestamp': cur_time, + 'relative_time': cur_time - self.start_time, + **content + }) + '\n') + fd.flush() + + @classmethod + @param('training.distributed') + @param('dist.world_size') + def launch_from_args(cls, distributed, world_size): + if distributed: + ch.multiprocessing.spawn(cls._exec_wrapper, nprocs=world_size, join=True) + else: + cls.exec(0) + + @classmethod + def _exec_wrapper(cls, *args, **kwargs): + make_config(quiet=True) + cls.exec(*args, **kwargs) + + @classmethod + @param('training.distributed') + @param('training.eval_only') + def exec(cls, gpu, distributed, eval_only): + trainer = cls(gpu=gpu) + if eval_only: + trainer.eval_and_log() + else: + trainer.train() + + if distributed: + trainer.cleanup_distributed() + +# Utils +class MeanScalarMetric(torchmetrics.Metric): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.add_state('sum', default=ch.tensor(0.), dist_reduce_fx='sum') + self.add_state('count', default=ch.tensor(0), dist_reduce_fx='sum') + + def update(self, sample: ch.Tensor): + self.sum += sample.sum() + self.count += sample.numel() + + def compute(self): + return self.sum.float() / self.count + +# Running +def make_config(quiet=False): + config = get_current_config() + parser = ArgumentParser(description='Fast imagenet training') + config.augment_argparse(parser) + config.collect_argparse_args(parser) + config.validate(mode='stderr') + if not quiet: + config.summary() + +if __name__ == "__main__": + make_config() + ImageNetTrainer.launch_from_args() \ No newline at end of file From 95678518dade4b5e1cb11b6054455aa79346b628 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 12:29:15 -0500 Subject: [PATCH 02/33] more bugfixes --- data/__init__.py | 1 + data/ffcv_imagenet.py | 29 +++++++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/data/__init__.py b/data/__init__.py index a70f282b..596b67d6 100644 --- a/data/__init__.py +++ b/data/__init__.py @@ -3,3 +3,4 @@ from data.tinyimagenet import TinyImageNet from data.mnist import MNIST from data.bigcifar import BigCIFAR10 +from data.ffcv_imagenet import FfcvImageNet diff --git a/data/ffcv_imagenet.py b/data/ffcv_imagenet.py index a37ace6e..0aa1b0eb 100644 --- a/data/ffcv_imagenet.py +++ b/data/ffcv_imagenet.py @@ -13,9 +13,9 @@ import torch.multiprocessing torch.multiprocessing.set_sharing_strategy("file_system") -class ImageNet: +class FfcvImageNet: def __init__(self, args): - super(ImageNet, self).__init__() + super(FfcvImageNet, self).__init__() # data_root = os.path.join(parser_args.data, "imagenet") # put ffcv path here @@ -24,23 +24,28 @@ def __init__(self, args): use_cuda = torch.cuda.is_available() # Data loading code - kwargs = {"num_workers": args.num_workers, "pin_memory": True} if use_cuda else {} + train_dataset = os.path.join(data_root, "train_500_0.50_90.ffcv") + val_dataset = os.path.join(data_root, "val_500_0.50_90.ffcv") # Data loading code - traindir = os.path.join(data_root, "train") - valdir = os.path.join(data_root, "val") - - normalize = transforms.Normalize( - mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] - ) + kwargs = {"num_workers": 1, "in_memory": 1, + "distributed": False, "resolution": 256} IMAGENET_MEAN = np.array([0.485, 0.456, 0.406]) * 255 IMAGENET_STD = np.array([0.229, 0.224, 0.225]) * 255 DEFAULT_CROP_RATIO = 224/256 - - self.train_loader = self.create_train_loader() - self.val_loader = self.create_val_loader() + self.train_loader = self.create_train_loader(train_dataset, + kwargs['num_workers'], + parser_args.batch_size, + kwargs['distributed'], + kwargs['in_memory']) + self.val_loader = self.create_val_loader(val_dataset, + kwargs['num_workers'], + parser_args.batch_size, + kwargs['resolution'], + kwargs['distributed'], + ) # madry does this but I don't think we need to # self.model, self.scaler = self.create_model_and_scaler() From a52a9751f91ab785e0f4241334de870ae84174e6 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 12:38:55 -0500 Subject: [PATCH 03/33] adding missing imports --- data/ffcv_imagenet.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/ffcv_imagenet.py b/data/ffcv_imagenet.py index 0aa1b0eb..5fda2d41 100644 --- a/data/ffcv_imagenet.py +++ b/data/ffcv_imagenet.py @@ -9,6 +9,8 @@ import random from torch.utils.data.sampler import SubsetRandomSampler from args_helper import parser_args +import numpy as np +from pathlib import Path import torch.multiprocessing torch.multiprocessing.set_sharing_strategy("file_system") @@ -120,4 +122,4 @@ def create_val_loader(self, val_dataset, num_workers, batch_size, 'label': label_pipeline }, distributed=distributed) - return loader \ No newline at end of file + return loader From ef76d9b2d092fad8639e06c283bb6f8bd194e571 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 12:44:09 -0500 Subject: [PATCH 04/33] adding code for get_resolution --- data/ffcv_imagenet.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/data/ffcv_imagenet.py b/data/ffcv_imagenet.py index 5fda2d41..16bb4e37 100644 --- a/data/ffcv_imagenet.py +++ b/data/ffcv_imagenet.py @@ -11,6 +11,15 @@ from args_helper import parser_args import numpy as np from pathlib import Path +from typing import List + +from ffcv.pipeline.operation import Operation +from ffcv.loader import Loader, OrderOption +from ffcv.transforms import ToTensor, ToDevice, Squeeze, NormalizeImage, \ + RandomHorizontalFlip, ToTorchImage +from ffcv.fields.rgb_image import CenterCropRGBImageDecoder, \ + RandomResizedCropRGBImageDecoder +from ffcv.fields.basics import IntDecoder import torch.multiprocessing torch.multiprocessing.set_sharing_strategy("file_system") @@ -123,3 +132,22 @@ def create_val_loader(self, val_dataset, num_workers, batch_size, }, distributed=distributed) return loader + + def get_resolution(self, epoch=0, min_res=160, max_res=192, end_ramp=76, start_ramp=65): + # this seems to be a hack to get good accuracy and is only between epochs + # 65 and 76. So, for now just return max_res. always. + + return max_res + + assert min_res <= max_res + + if epoch <= start_ramp: + return min_res + + if epoch >= end_ramp: + return max_res + + # otherwise, linearly interpolate to the nearest multiple of 32 + interp = np.interp([epoch], [start_ramp, end_ramp], [min_res, max_res]) + final_res = int(np.round(interp[0] / 32)) * 32 + return final_res From 71f3e3e90e5c7fb6793cb7630a7e9d990edcbce9 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 19:03:18 -0500 Subject: [PATCH 05/33] minor bugfix: changing scope of constants --- data/ffcv_imagenet.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data/ffcv_imagenet.py b/data/ffcv_imagenet.py index 16bb4e37..b5bd94f2 100644 --- a/data/ffcv_imagenet.py +++ b/data/ffcv_imagenet.py @@ -42,9 +42,9 @@ def __init__(self, args): kwargs = {"num_workers": 1, "in_memory": 1, "distributed": False, "resolution": 256} - IMAGENET_MEAN = np.array([0.485, 0.456, 0.406]) * 255 - IMAGENET_STD = np.array([0.229, 0.224, 0.225]) * 255 - DEFAULT_CROP_RATIO = 224/256 + self.IMAGENET_MEAN = np.array([0.485, 0.456, 0.406]) * 255 + self.IMAGENET_STD = np.array([0.229, 0.224, 0.225]) * 255 + self.DEFAULT_CROP_RATIO = 224/256 self.train_loader = self.create_train_loader(train_dataset, kwargs['num_workers'], @@ -73,7 +73,7 @@ def create_train_loader(self, train_dataset, num_workers, batch_size, ToTensor(), ToDevice(ch.device(parser_args.gpu), non_blocking=True), ToTorchImage(), - NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) + NormalizeImage(self.IMAGENET_MEAN, self.IMAGENET_STD, np.float16) ] label_pipeline: List[Operation] = [ @@ -110,7 +110,7 @@ def create_val_loader(self, val_dataset, num_workers, batch_size, ToTensor(), ToDevice(ch.device(parser_args.gpu), non_blocking=True), ToTorchImage(), - NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) + NormalizeImage(self.IMAGENET_MEAN, self.IMAGENET_STD, np.float16) ] label_pipeline = [ From 0509faa2350e5996553326b51372641d149c1edf Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 19:04:43 -0500 Subject: [PATCH 06/33] changing ch to torch --- data/ffcv_imagenet.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/data/ffcv_imagenet.py b/data/ffcv_imagenet.py index 16bb4e37..ebb668f1 100644 --- a/data/ffcv_imagenet.py +++ b/data/ffcv_imagenet.py @@ -21,6 +21,10 @@ RandomResizedCropRGBImageDecoder from ffcv.fields.basics import IntDecoder +torch.backends.cudnn.benchmark = True +torch.autograd.profiler.emit_nvtx(False) +torch.autograd.profiler.profile(False) + import torch.multiprocessing torch.multiprocessing.set_sharing_strategy("file_system") @@ -71,7 +75,7 @@ def create_train_loader(self, train_dataset, num_workers, batch_size, self.decoder, RandomHorizontalFlip(), ToTensor(), - ToDevice(ch.device(parser_args.gpu), non_blocking=True), + ToDevice(torch.device(parser_args.gpu), non_blocking=True), ToTorchImage(), NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) ] @@ -80,7 +84,7 @@ def create_train_loader(self, train_dataset, num_workers, batch_size, IntDecoder(), ToTensor(), Squeeze(), - ToDevice(ch.device(parser_args.gpu), non_blocking=True) + ToDevice(torch.device(parser_args.gpu), non_blocking=True) ] order = OrderOption.RANDOM if distributed else OrderOption.QUASI_RANDOM @@ -108,7 +112,7 @@ def create_val_loader(self, val_dataset, num_workers, batch_size, image_pipeline = [ cropper, ToTensor(), - ToDevice(ch.device(parser_args.gpu), non_blocking=True), + ToDevice(torch.device(parser_args.gpu), non_blocking=True), ToTorchImage(), NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) ] @@ -117,7 +121,7 @@ def create_val_loader(self, val_dataset, num_workers, batch_size, IntDecoder(), ToTensor(), Squeeze(), - ToDevice(ch.device(parser_args.gpu), + ToDevice(torch.device(parser_args.gpu), non_blocking=True) ] From fa1c0561b71fd69df65aa7a0b39a32ff98612d2f Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 19:51:55 -0500 Subject: [PATCH 07/33] data loader seems to be working! --- data/ffcv_imagenet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/ffcv_imagenet.py b/data/ffcv_imagenet.py index 507c4787..e61fa8e9 100644 --- a/data/ffcv_imagenet.py +++ b/data/ffcv_imagenet.py @@ -108,7 +108,7 @@ def create_val_loader(self, val_dataset, num_workers, batch_size, val_path = Path(val_dataset) assert val_path.is_file() res_tuple = (resolution, resolution) - cropper = CenterCropRGBImageDecoder(res_tuple, ratio=DEFAULT_CROP_RATIO) + cropper = CenterCropRGBImageDecoder(res_tuple, ratio=self.DEFAULT_CROP_RATIO) image_pipeline = [ cropper, ToTensor(), From 8107428a0aeaed4087d7e90a88ce82628655662e Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 22:23:30 -0500 Subject: [PATCH 08/33] changing conv_type to use half precision for ffcv --- utils/conv_type.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/utils/conv_type.py b/utils/conv_type.py index c235d14b..99109708 100644 --- a/utils/conv_type.py +++ b/utils/conv_type.py @@ -71,11 +71,11 @@ def forward(ctx, scores, bias_scores, k, scores_prune_threshold=-np.inf, bias_sc # NOTE: doing this EP style where the scores are unchanged, but mask is computed # can also try a variant where we actually round the scores if parser_args.bottom_k_on_forward: - out = torch.gt(scores, torch.ones_like(scores)*scores_prune_threshold).float() - bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float() + out = torch.gt(scores, torch.ones_like(scores)*scores_prune_threshold).float().half() + bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*bias_scores_prune_threshold).float().half() else: - out = torch.gt(scores, torch.ones_like(scores)*parser_args.quantize_threshold).float() - bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*parser_args.quantize_threshold).float() + out = torch.gt(scores, torch.ones_like(scores)*parser_args.quantize_threshold).float().half() + bias_out = torch.gt(bias_scores, torch.ones_like(bias_scores)*parser_args.quantize_threshold).float().half() else: print("INVALID PRUNING ALGO") @@ -171,11 +171,11 @@ def forward(self, x): if parser_args.hc_quantized: subnet, bias_subnet = GetSubnet.apply(self.scores, self.bias_scores, parser_args.prune_rate) - subnet = subnet * self.flag.data.float() - bias_subnet = subnet * self.bias_flag.data.float() + subnet = subnet * self.flag.data.float().half() + bias_subnet = subnet * self.bias_flag.data.float().half() else: - subnet = self.scores * self.flag.data.float() - bias_subnet = self.bias_scores * self.bias_flag.data.float() + subnet = self.scores * self.flag.data.float().half() + bias_subnet = self.bias_scores * self.bias_flag.data.float().half() elif parser_args.algo in ['imp']: # no STE, no subnet. Mask is handled outside pass @@ -195,7 +195,8 @@ def forward(self, x): b = self.bias * bias_subnet else: b = self.bias - + # TODO: wrap the ".half()" in something + # print("dtype: {}".format(w.dtype)) x = F.conv2d( x, w, b, self.stride, self.padding, self.dilation, self.groups ) From 602ad02532b2a697c483b05f428fda97549161f1 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 22:28:51 -0500 Subject: [PATCH 09/33] adding config for ffcv imagenet --- .../imagenet/resnet18_sparsity_10.yml | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml diff --git a/configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml b/configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml new file mode 100644 index 00000000..66426b54 --- /dev/null +++ b/configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml @@ -0,0 +1,65 @@ +subfolder: ffcv_imagenet_resnet18 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet18 + +# ===== Dataset ===== # +dataset: FfcvImageNet +name: resnet18_ffcv +data: /workspace/ffcv-imagenet/data/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 50 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 10 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From dae1a7118ab3516a2a538c0d9367ab6b0ca3a55e Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 22:34:06 -0500 Subject: [PATCH 10/33] adding script to exec imagenet --- imagenet_exec.sh | 31 +++++++++++++++++++++++++++++++ models/__init__.py | 4 +++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 imagenet_exec.sh diff --git a/imagenet_exec.sh b/imagenet_exec.sh new file mode 100644 index 00000000..b80ba719 --- /dev/null +++ b/imagenet_exec.sh @@ -0,0 +1,31 @@ +#### ResNet-18 + +# Running trials in parallel +# NOTE: make sure to delete/comment subfolder from the config file or else it may not work +:< "$log_root$trial$log_end" 2>&1 & + + python main.py \ + --config "$conf_file" \ + --trial-num $trial \ + --invert-sanity-check \ + --subfolder "invert_$subfolder_root$trial" > "invert_$log_root$trial$log_end" 2>&1 & +done + +BLOCK + +conf_file="configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml" +log_root="resnet18_ffcv" +log_end="_log" +python main.py \ + --config "$conf_file" > "$log_root$log_end" 2>&1 & \ No newline at end of file diff --git a/models/__init__.py b/models/__init__.py index 216a0bfb..1c769ae3 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -7,7 +7,9 @@ #### TODO: delete below ones (merge with above code) from models.resnet_cifar import cResNet18, cResNet50 -from models.resnet_tiny import TinyResNet18 +from models.resnet_tiny import TinyResNet18 + +# NOTE: ResNet18 is for imagenet - rename __all__ = [ "tinyvgg16", From 6055c68f758a57bdb75ace14255823225e5a3cb2 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 22:39:48 -0500 Subject: [PATCH 11/33] adding get_layers for resnet18 imagenet --- utils/net_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/utils/net_utils.py b/utils/net_utils.py index 46462981..3aeedd29 100644 --- a/utils/net_utils.py +++ b/utils/net_utils.py @@ -126,6 +126,18 @@ def get_layers(arch='Conv4', model=None): linear_layers.append(layer.mlp.fc1) linear_layers.append(layer.mlp.fc2) # linear_layers.append(model.decoder) + + elif arch == 'ResNet18': + conv_layers = [model.conv1] + for layer in [model.layer1, model.layer2, model.layer3, model.layer4]: + for basic_block_id in [0, 1]: + conv_layers.append(layer[basic_block_id].conv1) + conv_layers.append(layer[basic_block_id].conv2) + linear_layers = [model.fc] + + else: + print("ERROR: get_layers() not implemented for model {}".format(arch)) + return -1 return (conv_layers, linear_layers) From 40f423e08aa55eea54c909ac808181515bd00da7 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 21 Mar 2022 22:45:58 -0500 Subject: [PATCH 12/33] things work but loss is infinity my guess is that the last layer is not right. will fix. --- imagenet_exec.sh | 2 +- main_utils.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index b80ba719..e32e558b 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -28,4 +28,4 @@ conf_file="configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml" log_root="resnet18_ffcv" log_end="_log" python main.py \ - --config "$conf_file" > "$log_root$log_end" 2>&1 & \ No newline at end of file + --config "$conf_file" #> "$log_root$log_end" 2>&1 & diff --git a/main_utils.py b/main_utils.py index db0652ab..d8c19067 100644 --- a/main_utils.py +++ b/main_utils.py @@ -985,6 +985,13 @@ def get_model(parser_args): if parser_args.freeze_weights: freeze_model_weights(model) + # TODO: hacky. fix this. @sreeniva + if parser_args.arch == "ResNet18": + # make half precision + model = model.half() + for name, params in model.named_parameters(): + params.data = params.half() + return model From e5d18f0b6daf38fa9b2442eca60c7bf583b555c9 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 25 Mar 2022 23:17:51 -0500 Subject: [PATCH 13/33] fixing up scaler issues turns out, using .half() causes some issues with the scaler. --- .../resnet18/imagenet/resnet18_sparsity_10.yml | 1 + imagenet_exec.sh | 6 +++--- main_utils.py | 10 +++++----- trainers/default.py | 10 +++++++--- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml b/configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml index 66426b54..b044f695 100644 --- a/configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml +++ b/configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml @@ -27,6 +27,7 @@ epochs: 50 wd: 0.0 momentum: 0.9 batch_size: 128 +mixed_precision: True # ===== Sparsity =========== # conv_type: SubnetConv diff --git a/imagenet_exec.sh b/imagenet_exec.sh index e32e558b..4f99ae85 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -24,8 +24,8 @@ done BLOCK -conf_file="configs/hypercube/resnet18/imagenet/resnet18_sparsity_10.yml" -log_root="resnet18_ffcv" +conf_file="configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml" +log_root="resnet50_ffcv" log_end="_log" python main.py \ - --config "$conf_file" #> "$log_root$log_end" 2>&1 & + --config "$conf_file" > "$log_root$log_end" 2>&1 & diff --git a/main_utils.py b/main_utils.py index d8c19067..a73c68fb 100644 --- a/main_utils.py +++ b/main_utils.py @@ -986,11 +986,11 @@ def get_model(parser_args): freeze_model_weights(model) # TODO: hacky. fix this. @sreeniva - if parser_args.arch == "ResNet18": - # make half precision - model = model.half() - for name, params in model.named_parameters(): - params.data = params.half() + #if parser_args.arch == "ResNet18": + # # make half precision + # model = model.half() + # for name, params in model.named_parameters(): + # params.data = params.half() return model diff --git a/trainers/default.py b/trainers/default.py index 6597353f..864e45cb 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -7,13 +7,13 @@ from utils.eval_utils import accuracy from utils.logging import AverageMeter, ProgressMeter from utils.net_utils import get_regularization_loss, prune, get_layers +from torch.cuda.amp import autocast from torch import optim __all__ = ["train", "validate", "modifier"] - def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler=None): batch_time = AverageMeter("Time", ":6.3f") data_time = AverageMeter("Data", ":6.3f") @@ -67,7 +67,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler if args.lam_finetune_loss > 0: raise NotImplementedError # please check finetune_loss repo - + regularization_loss = torch.tensor(0) if args.regularization: regularization_loss =\ @@ -76,7 +76,9 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler alpha_prime=args.alpha_prime) #print('regularization_loss: ', regularization_loss) - loss += regularization_loss + + with torch.cuda.amp.autocast(enabled=args.mixed_precision): # mixed precision + loss += regularization_loss # measure accuracy and record loss acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) @@ -91,6 +93,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler loss.backward() optimizer.step() else: + with torch.cuda.amp.autocast(enabled=args.mixed_precision): # mixed precision + output = model(images) scaler.scale(loss).backward() scaler.step(optimizer) scaler.update() From 30486facca4105a4a694fe0108ea075d4481f846 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Mon, 28 Mar 2022 17:14:25 -0500 Subject: [PATCH 14/33] adding resnet50 config and amp for validate need mixed precision for running ffcv. --- .../ffcv_imagenet/resnet50_sparsity_10.yml | 66 +++++++++++++++++++ trainers/default.py | 6 +- 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml new file mode 100644 index 00000000..1f0f9d57 --- /dev/null +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml @@ -0,0 +1,66 @@ +subfolder: ffcv_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: FfcvImageNet +name: resnet18_ffcv +data: /workspace/ffcv-imagenet/data/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.4 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 44 +wd: 0.0 +momentum: 0.9 +batch_size: 100 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 10 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/trainers/default.py b/trainers/default.py index 864e45cb..e11fc03a 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -149,9 +149,9 @@ def validate(val_loader, model, criterion, args, writer, epoch): #print(images.shape, target.shape) # compute output - output = model(images) - - loss = criterion(output, target) + with torch.cuda.amp.autocast(enabled=args.mixed_precision): # mixed precision + output = model(images) + loss = criterion(output, target) # measure accuracy and record loss acc1, acc5, acc10 = accuracy(output, target, topk=(1, 5, 10)) From c44b67378fbc6aead528ea1130c2229fb6a9eb07 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 1 Apr 2022 13:47:53 -0500 Subject: [PATCH 15/33] fixing mixed_prec finetune bug looks like scaler is None for the finetuning step which causes an issue. Need to handle this a little better in the future but for now, this works. --- .../resnet50/ffcv_imagenet/resnet50_sparsity_10.yml | 2 +- trainers/default.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml index 1f0f9d57..0669cb01 100644 --- a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml @@ -23,7 +23,7 @@ fine_tune_lr: 0.4 fine_tune_lr_policy: multistep_lr # ===== Network training config ===== # -epochs: 44 +epochs: 88 wd: 0.0 momentum: 0.9 batch_size: 100 diff --git a/trainers/default.py b/trainers/default.py index e11fc03a..70ff3304 100644 --- a/trainers/default.py +++ b/trainers/default.py @@ -57,13 +57,9 @@ def train(train_loader, model, criterion, optimizer, epoch, args, writer, scaler scores.data = torch.clamp(scores.data, 0.0, 1.0) # compute output - if scaler is None: + with torch.cuda.amp.autocast(enabled=args.mixed_precision): # mixed precision output = model(images) loss = criterion(output, target) - else: - with torch.cuda.amp.autocast(enabled=True): # mixed precision - output = model(images) - loss = criterion(output, target) if args.lam_finetune_loss > 0: raise NotImplementedError # please check finetune_loss repo From c3747c788bf7302d6da7a5876af97ea20605677b Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Sun, 3 Apr 2022 20:25:59 -0500 Subject: [PATCH 16/33] sanity check code for prospr this isn't the right branch for it, but needed to keep it somewhere --- sanity_check_prospr.py | 161 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 sanity_check_prospr.py diff --git a/sanity_check_prospr.py b/sanity_check_prospr.py new file mode 100644 index 00000000..9d2e6caa --- /dev/null +++ b/sanity_check_prospr.py @@ -0,0 +1,161 @@ +# trying to test prospr using their codebase + +import importlib +import re + +from train import * +import argparse +import random +from pathlib import Path + +import utils +from cli import parse_args +import prospr.utils +import random +import numpy as np +import os +import copy + +from models.resnet20 import _weights_init + +WEIGHT_REINIT_SANITY = False +MASK_SHUFFLE_SANITY = True + +args = parse_args() +hparams = utils.Hyperparameters(**vars(args)) + +log_dir = utils.create_logdir("sreeniva_debug_prospr") + +utils.set_seed(hparams.seed, hparams.allow_nondeterminism) + +train_data, _, test_data, _ = dataloader_factory( + hparams.dataset, hparams.batch_size +) + +model = model_factory(hparams.model, hparams.dataset, hparams.no_model_patching) +model, masks = get_pruned_model(model, hparams) + +model_ckpt = torch.load("/workspace/pruning_is_enough/prospr_ckpts/sp95/trained_model.pt") +mask_ckpt = torch.load("/workspace/pruning_is_enough/prospr_ckpts/sp95/pruning_keep_mask.pt") +model.load_state_dict(model_ckpt) +masks = mask_ckpt + +filter_fn = pruning_filter_factory(10, hparams.structured_pruning) +structured = False + +pruned_model = prospr.utils.apply_masks_with_hooks(model, masks, structured, filter_fn) + +# from now onwards, only refer to pruned_model!!! +optimizer, lr_scheduler = get_optimizer(pruned_model, hparams) +pruned_model = pruned_model.cuda() + +test_loss, test_acc1, test_acc5 = evaluate(pruned_model, test_data) +print("Test Accuracy: {}%".format(test_acc1*100)) +orig_test_acc = test_acc1*100 + +weight_only = [] +for name, param in pruned_model.named_parameters(): + if "weight" in name and "bn" not in name: + weight_only.append((name, param)) + +# manually apply mask to model as sanity check +idx = 0 +for name, param in pruned_model.named_parameters(): + if "weight" in name and "bn" not in name: + param.data *= masks[idx] + idx += 1 + +# this should be the same as before +test_loss, test_acc1, test_acc5 = evaluate(pruned_model, test_data) +print("Test Accuracy: {}%".format(test_acc1*100)) + + +# Run sanity checks +def set_seed(seed): + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + # making sure GPU runs are deterministic even if they are slower + torch.backends.cudnn.deterministic = True + # this causes the code to vary across runs. I don't want that for now. + # torch.backends.cudnn.benchmark = True + print("Seeded everything: {}".format(seed)) + +set_seed(96) + +bkp_pruned_model = copy.deepcopy(pruned_model) + +# weight reinit sanity check +if WEIGHT_REINIT_SANITY: + pruned_model = pruned_model.apply(_weights_init) + print("Running Weight Reinit Sanity Check") +else: + shuffled_masks = [] + for mask in masks: + idx = torch.randperm(mask.nelement()) + shuffled_masks.append(mask.view(-1)[idx].view(mask.size())) + print("Running Mask Shuffle Sanity Check") + masks = shuffled_masks + +pruned_model = prospr.utils.apply_masks_with_hooks(pruned_model, masks, structured, filter_fn) +# manually apply mask to model as sanity check +idx = 0 +for name, param in pruned_model.named_parameters(): + if "weight" in name and "bn" not in name: + print("Pruning Layer: {}".format(name)) + print("Before: {}".format(torch.norm(param))) + param.data *= masks[idx] + print("After: {}".format(torch.norm(param))) + idx += 1 + +# compare norms to verify that reinit has really happened. +print("Conv1.weight norm: Before={} | After={}".format(torch.norm(bkp_pruned_model.conv1.weight), torch.norm(pruned_model.conv1.weight))) + +# this should be the same as before +test_loss, test_acc1, test_acc5 = evaluate(pruned_model, test_data) +print("Test Accuracy after flipping things (should be terrible): {}%".format(test_acc1*100)) + +# train for 200 epochs and see what happens +optimizer, lr_scheduler = get_optimizer(pruned_model, hparams) +for epoch in range(1, 200 + 1): + avg_train_loss, epoch_time = train_one_epoch(pruned_model, train_data, optimizer) + test_loss, test_acc1, test_acc5 = evaluate(pruned_model, test_data) + + print( + f"📸 Epoch {epoch} (finished in {epoch_time})\n", + f"\tTrain loss:\t{avg_train_loss:.4f}\n", + f"\tTest loss:\t{test_loss:.4f}\n", + f"\tTest acc:\t{test_acc1:.4f}\n", + f"\tTest top-5 acc:\t{test_acc5:.4f}", + ) + + lr_scheduler.step() + +print( + "✅ Training finished\n", + f"\tFinal test acc: {test_acc1}\n", + f"\tFinal test acc@5: {test_acc5}", +) + +# sanity check by applying the mask and testing accuracy +# manually apply mask to model as sanity check +idx = 0 +for name, param in pruned_model.named_parameters(): + if "weight" in name and "bn" not in name: + print("Pruning Layer: {}".format(name)) + print("Before: {}".format(torch.norm(param))) + param.data *= masks[idx] + print("After: {}".format(torch.norm(param))) + idx += 1 + +# this should be the same as before +test_loss, test_acc1, test_acc5 = evaluate(pruned_model, test_data) +print("Test Accuracy: {}%".format(test_acc1*100)) +final_test_acc = test_acc1*100 + +print("Sanity check complete!") +print("Accuracy before reinit: {} | Accuracy after weight reinit: {}".format(orig_test_acc, final_test_acc)) + From 9711ca3ae8517c3b6aca73b54d56a7b3bed9fc9e Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 5 Apr 2022 21:54:51 -0500 Subject: [PATCH 17/33] need to lower finetune_lr otherwise the model gets rekt --- .../hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml index 0669cb01..1d074cfb 100644 --- a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml @@ -19,7 +19,7 @@ data: /workspace/ffcv-imagenet/data/ optimizer: sgd lr: 0.4 #0.01 lr_policy: cosine_lr #constant_lr #multistep_lr -fine_tune_lr: 0.4 +fine_tune_lr: 0.001 fine_tune_lr_policy: multistep_lr # ===== Network training config ===== # From 06c15cd8168adee5a86517bbecf87269b0f7c722 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 5 Apr 2022 22:20:52 -0500 Subject: [PATCH 18/33] adding finetune_only feature --- args_helper.py | 11 ++++++----- main.py | 8 ++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/args_helper.py b/args_helper.py index b03fbb95..4867de73 100644 --- a/args_helper.py +++ b/args_helper.py @@ -282,8 +282,6 @@ def parse_arguments(self, jupyter_mode=False): default=1.0, help="portion of additional width compared with original width" ) - - parser.add_argument( "--hidden-size", type=int, @@ -899,14 +897,12 @@ def parse_arguments(self, jupyter_mode=False): default=False, help="Only run sanity checks on the files in specific directory or subdirectories" ) - parser.add_argument( "--invert-sanity-check", action="store_true", default=False, help="Enable this to run the inverted sanity check (for HC)" ) - parser.add_argument( "--sanity-folder", default=None, @@ -914,13 +910,18 @@ def parse_arguments(self, jupyter_mode=False): metavar="PATH", help="directory(s) to access for only sanity check", ) - parser.add_argument( "--sr-version", default=1, type=int, help="smart ratio version number (1, 2, ...)", ) + parser.add_argument( + "--only-finetune", + action="store_true", + default=False, + help="Enable this to skip pruning and jump to finetune. Typically paired with --resume" + ) if jupyter_mode: args = parser.parse_args("") diff --git a/main.py b/main.py index 4e6512a5..10caf431 100644 --- a/main.py +++ b/main.py @@ -135,6 +135,10 @@ def main_worker(gpu, ngpus_per_node): # Start training for epoch in range(parser_args.start_epoch, parser_args.epochs): + if parser_args.only_finetune: + print("Skipping pruning and going straight to finetune!!!") + break + if parser_args.multiprocessing_distributed: data.train_loader.sampler.set_epoch(epoch) #lr_policy(epoch, iteration=None) @@ -274,6 +278,10 @@ def main_worker(gpu, ngpus_per_node): print("Writing results into: {}".format(results_filename)) results_df.to_csv(results_filename, index=False) + if parser_args.resume: + print("Loading checkpoint before finetune") + best_acc1 = resume(parser_args, model, optimizer) + # save checkpoint before fine-tuning torch.save(model.state_dict(), result_root + 'model_before_finetune.pth') From bb0672596d39547c65e8fae348dc9fb3b7f7c059 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Tue, 5 Apr 2022 23:05:08 -0500 Subject: [PATCH 19/33] deleting original ffcv file --- ffcv_org_train_imagenet.py | 511 ------------------------------------- 1 file changed, 511 deletions(-) delete mode 100644 ffcv_org_train_imagenet.py diff --git a/ffcv_org_train_imagenet.py b/ffcv_org_train_imagenet.py deleted file mode 100644 index 0892d3ce..00000000 --- a/ffcv_org_train_imagenet.py +++ /dev/null @@ -1,511 +0,0 @@ -import torch as ch -from torch.cuda.amp import GradScaler -from torch.cuda.amp import autocast -import torch.nn.functional as F -import torch.distributed as dist - -ch.backends.cudnn.benchmark = True -ch.autograd.profiler.emit_nvtx(False) -ch.autograd.profiler.profile(False) - -from torchvision import models -import torchmetrics -import numpy as np -from tqdm import tqdm - -import os -import time -import json -from uuid import uuid4 -from typing import List -from pathlib import Path -from argparse import ArgumentParser - -from fastargs import get_current_config -from fastargs.decorators import param -from fastargs import Param, Section -from fastargs.validation import And, OneOf - -from ffcv.pipeline.operation import Operation -from ffcv.loader import Loader, OrderOption -from ffcv.transforms import ToTensor, ToDevice, Squeeze, NormalizeImage, \ - RandomHorizontalFlip, ToTorchImage -from ffcv.fields.rgb_image import CenterCropRGBImageDecoder, \ - RandomResizedCropRGBImageDecoder -from ffcv.fields.basics import IntDecoder - -Section('model', 'model details').params( - arch=Param(And(str, OneOf(models.__dir__())), default='resnet18'), - pretrained=Param(int, 'is pretrained? (1/0)', default=0) -) - -Section('resolution', 'resolution scheduling').params( - min_res=Param(int, 'the minimum (starting) resolution', default=160), - max_res=Param(int, 'the maximum (starting) resolution', default=160), - end_ramp=Param(int, 'when to stop interpolating resolution', default=0), - start_ramp=Param(int, 'when to start interpolating resolution', default=0) -) - -Section('data', 'data related stuff').params( - train_dataset=Param(str, '.dat file to use for training', required=True), - val_dataset=Param(str, '.dat file to use for validation', required=True), - num_workers=Param(int, 'The number of workers', required=True), - in_memory=Param(int, 'does the dataset fit in memory? (1/0)', required=True) -) - -Section('lr', 'lr scheduling').params( - step_ratio=Param(float, 'learning rate step ratio', default=0.1), - step_length=Param(int, 'learning rate step length', default=30), - lr_schedule_type=Param(OneOf(['step', 'cyclic']), default='cyclic'), - lr=Param(float, 'learning rate', default=0.5), - lr_peak_epoch=Param(int, 'Epoch at which LR peaks', default=2), -) - -Section('logging', 'how to log stuff').params( - folder=Param(str, 'log location', required=True), - log_level=Param(int, '0 if only at end 1 otherwise', default=1) -) - -Section('validation', 'Validation parameters stuff').params( - batch_size=Param(int, 'The batch size for validation', default=512), - resolution=Param(int, 'final resized validation image size', default=224), - lr_tta=Param(int, 'should do lr flipping/avging at test time', default=1) -) - -Section('training', 'training hyper param stuff').params( - eval_only=Param(int, 'eval only?', default=0), - batch_size=Param(int, 'The batch size', default=512), - optimizer=Param(And(str, OneOf(['sgd'])), 'The optimizer', default='sgd'), - momentum=Param(float, 'SGD momentum', default=0.9), - weight_decay=Param(float, 'weight decay', default=4e-5), - epochs=Param(int, 'number of epochs', default=30), - label_smoothing=Param(float, 'label smoothing parameter', default=0.1), - distributed=Param(int, 'is distributed?', default=0), - use_blurpool=Param(int, 'use blurpool?', default=0) -) - -Section('dist', 'distributed training options').params( - world_size=Param(int, 'number gpus', default=1), - address=Param(str, 'address', default='localhost'), - port=Param(str, 'port', default='12355') -) - -IMAGENET_MEAN = np.array([0.485, 0.456, 0.406]) * 255 -IMAGENET_STD = np.array([0.229, 0.224, 0.225]) * 255 -DEFAULT_CROP_RATIO = 224/256 - -@param('lr.lr') -@param('lr.step_ratio') -@param('lr.step_length') -@param('training.epochs') -def get_step_lr(epoch, lr, step_ratio, step_length, epochs): - if epoch >= epochs: - return 0 - - num_steps = epoch // step_length - return step_ratio**num_steps * lr - -@param('lr.lr') -@param('training.epochs') -@param('lr.lr_peak_epoch') -def get_cyclic_lr(epoch, lr, epochs, lr_peak_epoch): - xs = [0, lr_peak_epoch, epochs] - ys = [1e-4 * lr, lr, 0] - return np.interp([epoch], xs, ys)[0] - -class BlurPoolConv2d(ch.nn.Module): - def __init__(self, conv): - super().__init__() - default_filter = ch.tensor([[[[1, 2, 1], [2, 4, 2], [1, 2, 1]]]]) / 16.0 - filt = default_filter.repeat(conv.in_channels, 1, 1, 1) - self.conv = conv - self.register_buffer('blur_filter', filt) - - def forward(self, x): - blurred = F.conv2d(x, self.blur_filter, stride=1, padding=(1, 1), - groups=self.conv.in_channels, bias=None) - return self.conv.forward(blurred) - -class ImageNetTrainer: - @param('training.distributed') - def __init__(self, gpu, distributed): - self.all_params = get_current_config() - self.gpu = gpu - - self.uid = str(uuid4()) - - if distributed: - self.setup_distributed() - - self.train_loader = self.create_train_loader() - self.val_loader = self.create_val_loader() - self.model, self.scaler = self.create_model_and_scaler() - self.create_optimizer() - self.initialize_logger() - - - @param('dist.address') - @param('dist.port') - @param('dist.world_size') - def setup_distributed(self, address, port, world_size): - os.environ['MASTER_ADDR'] = address - os.environ['MASTER_PORT'] = port - - dist.init_process_group("nccl", rank=self.gpu, world_size=world_size) - ch.cuda.set_device(self.gpu) - - def cleanup_distributed(self): - dist.destroy_process_group() - - @param('lr.lr_schedule_type') - def get_lr(self, epoch, lr_schedule_type): - lr_schedules = { - 'cyclic': get_cyclic_lr, - 'step': get_step_lr - } - - return lr_schedules[lr_schedule_type](epoch) - - # resolution tools - @param('resolution.min_res') - @param('resolution.max_res') - @param('resolution.end_ramp') - @param('resolution.start_ramp') - def get_resolution(self, epoch, min_res, max_res, end_ramp, start_ramp): - assert min_res <= max_res - - if epoch <= start_ramp: - return min_res - - if epoch >= end_ramp: - return max_res - - # otherwise, linearly interpolate to the nearest multiple of 32 - interp = np.interp([epoch], [start_ramp, end_ramp], [min_res, max_res]) - final_res = int(np.round(interp[0] / 32)) * 32 - return final_res - - @param('training.momentum') - @param('training.optimizer') - @param('training.weight_decay') - @param('training.label_smoothing') - def create_optimizer(self, momentum, optimizer, weight_decay, - label_smoothing): - assert optimizer == 'sgd' - - # Only do weight decay on non-batchnorm parameters - all_params = list(self.model.named_parameters()) - bn_params = [v for k, v in all_params if ('bn' in k)] - other_params = [v for k, v in all_params if not ('bn' in k)] - param_groups = [{ - 'params': bn_params, - 'weight_decay': 0. - }, { - 'params': other_params, - 'weight_decay': weight_decay - }] - - self.optimizer = ch.optim.SGD(param_groups, lr=1, momentum=momentum) - self.loss = ch.nn.CrossEntropyLoss(label_smoothing=label_smoothing) - - @param('data.train_dataset') - @param('data.num_workers') - @param('training.batch_size') - @param('training.distributed') - @param('data.in_memory') - def create_train_loader(self, train_dataset, num_workers, batch_size, - distributed, in_memory): - this_device = f'cuda:{self.gpu}' - train_path = Path(train_dataset) - assert train_path.is_file() - - res = self.get_resolution(epoch=0) - self.decoder = RandomResizedCropRGBImageDecoder((res, res)) - image_pipeline: List[Operation] = [ - self.decoder, - RandomHorizontalFlip(), - ToTensor(), - ToDevice(ch.device(this_device), non_blocking=True), - ToTorchImage(), - NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) - ] - - label_pipeline: List[Operation] = [ - IntDecoder(), - ToTensor(), - Squeeze(), - ToDevice(ch.device(this_device), non_blocking=True) - ] - - order = OrderOption.RANDOM if distributed else OrderOption.QUASI_RANDOM - loader = Loader(train_dataset, - batch_size=batch_size, - num_workers=num_workers, - order=order, - os_cache=in_memory, - drop_last=True, - pipelines={ - 'image': image_pipeline, - 'label': label_pipeline - }, - distributed=distributed) - - return loader - - @param('data.val_dataset') - @param('data.num_workers') - @param('validation.batch_size') - @param('validation.resolution') - @param('training.distributed') - def create_val_loader(self, val_dataset, num_workers, batch_size, - resolution, distributed): - this_device = f'cuda:{self.gpu}' - val_path = Path(val_dataset) - assert val_path.is_file() - res_tuple = (resolution, resolution) - cropper = CenterCropRGBImageDecoder(res_tuple, ratio=DEFAULT_CROP_RATIO) - image_pipeline = [ - cropper, - ToTensor(), - ToDevice(ch.device(this_device), non_blocking=True), - ToTorchImage(), - NormalizeImage(IMAGENET_MEAN, IMAGENET_STD, np.float16) - ] - - label_pipeline = [ - IntDecoder(), - ToTensor(), - Squeeze(), - ToDevice(ch.device(this_device), - non_blocking=True) - ] - - loader = Loader(val_dataset, - batch_size=batch_size, - num_workers=num_workers, - order=OrderOption.SEQUENTIAL, - drop_last=False, - pipelines={ - 'image': image_pipeline, - 'label': label_pipeline - }, - distributed=distributed) - return loader - - @param('training.epochs') - @param('logging.log_level') - def train(self, epochs, log_level): - for epoch in range(epochs): - res = self.get_resolution(epoch) - self.decoder.output_size = (res, res) - train_loss = self.train_loop(epoch) - - if log_level > 0: - extra_dict = { - 'train_loss': train_loss, - 'epoch': epoch - } - - self.eval_and_log(extra_dict) - - self.eval_and_log({'epoch':epoch}) - if self.gpu == 0: - ch.save(self.model.state_dict(), self.log_folder / 'final_weights.pt') - - def eval_and_log(self, extra_dict={}): - start_val = time.time() - stats = self.val_loop() - val_time = time.time() - start_val - if self.gpu == 0: - self.log(dict({ - 'current_lr': self.optimizer.param_groups[0]['lr'], - 'top_1': stats['top_1'], - 'top_5': stats['top_5'], - 'val_time': val_time - }, **extra_dict)) - - return stats - - @param('model.arch') - @param('model.pretrained') - @param('training.distributed') - @param('training.use_blurpool') - def create_model_and_scaler(self, arch, pretrained, distributed, use_blurpool): - scaler = GradScaler() - model = getattr(models, arch)(pretrained=pretrained) - def apply_blurpool(mod: ch.nn.Module): - for (name, child) in mod.named_children(): - if isinstance(child, ch.nn.Conv2d) and (np.max(child.stride) > 1 and child.in_channels >= 16): - setattr(mod, name, BlurPoolConv2d(child)) - else: apply_blurpool(child) - if use_blurpool: apply_blurpool(model) - - model = model.to(memory_format=ch.channels_last) - model = model.to(self.gpu) - - if distributed: - model = ch.nn.parallel.DistributedDataParallel(model, device_ids=[self.gpu]) - - return model, scaler - - @param('logging.log_level') - def train_loop(self, epoch, log_level): - model = self.model - model.train() - losses = [] - - lr_start, lr_end = self.get_lr(epoch), self.get_lr(epoch + 1) - iters = len(self.train_loader) - lrs = np.interp(np.arange(iters), [0, iters], [lr_start, lr_end]) - - iterator = tqdm(self.train_loader) - for ix, (images, target) in enumerate(iterator): - ### Training start - for param_group in self.optimizer.param_groups: - param_group['lr'] = lrs[ix] - - self.optimizer.zero_grad(set_to_none=True) - with autocast(): - output = self.model(images) - loss_train = self.loss(output, target) - - self.scaler.scale(loss_train).backward() - self.scaler.step(self.optimizer) - self.scaler.update() - ### Training end - - ### Logging start - if log_level > 0: - losses.append(loss_train.detach()) - - group_lrs = [] - for _, group in enumerate(self.optimizer.param_groups): - group_lrs.append(f'{group["lr"]:.3f}') - - names = ['ep', 'iter', 'shape', 'lrs'] - values = [epoch, ix, tuple(images.shape), group_lrs] - if log_level > 1: - names += ['loss'] - values += [f'{loss_train.item():.3f}'] - - msg = ', '.join(f'{n}={v}' for n, v in zip(names, values)) - iterator.set_description(msg) - ### Logging end - - if log_level > 0: - loss = ch.stack(losses).mean().cpu() - assert not ch.isnan(loss), 'Loss is NaN!' - return loss.item() - - @param('validation.lr_tta') - def val_loop(self, lr_tta): - model = self.model - model.eval() - - with ch.no_grad(): - with autocast(): - for images, target in tqdm(self.val_loader): - output = self.model(images) - if lr_tta: - output += self.model(ch.flip(images, dims=[3])) - - for k in ['top_1', 'top_5']: - self.val_meters[k](output, target) - - loss_val = self.loss(output, target) - self.val_meters['loss'](loss_val) - - stats = {k: m.compute().item() for k, m in self.val_meters.items()} - [meter.reset() for meter in self.val_meters.values()] - return stats - - @param('logging.folder') - def initialize_logger(self, folder): - self.val_meters = { - 'top_1': torchmetrics.Accuracy(compute_on_step=False).to(self.gpu), - 'top_5': torchmetrics.Accuracy(compute_on_step=False, top_k=5).to(self.gpu), - 'loss': MeanScalarMetric(compute_on_step=False).to(self.gpu) - } - - if self.gpu == 0: - folder = (Path(folder) / str(self.uid)).absolute() - folder.mkdir(parents=True) - - self.log_folder = folder - self.start_time = time.time() - - print(f'=> Logging in {self.log_folder}') - params = { - '.'.join(k): self.all_params[k] for k in self.all_params.entries.keys() - } - - with open(folder / 'params.json', 'w+') as handle: - json.dump(params, handle) - - def log(self, content): - print(f'=> Log: {content}') - if self.gpu != 0: return - cur_time = time.time() - with open(self.log_folder / 'log', 'a+') as fd: - fd.write(json.dumps({ - 'timestamp': cur_time, - 'relative_time': cur_time - self.start_time, - **content - }) + '\n') - fd.flush() - - @classmethod - @param('training.distributed') - @param('dist.world_size') - def launch_from_args(cls, distributed, world_size): - if distributed: - ch.multiprocessing.spawn(cls._exec_wrapper, nprocs=world_size, join=True) - else: - cls.exec(0) - - @classmethod - def _exec_wrapper(cls, *args, **kwargs): - make_config(quiet=True) - cls.exec(*args, **kwargs) - - @classmethod - @param('training.distributed') - @param('training.eval_only') - def exec(cls, gpu, distributed, eval_only): - trainer = cls(gpu=gpu) - if eval_only: - trainer.eval_and_log() - else: - trainer.train() - - if distributed: - trainer.cleanup_distributed() - -# Utils -class MeanScalarMetric(torchmetrics.Metric): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.add_state('sum', default=ch.tensor(0.), dist_reduce_fx='sum') - self.add_state('count', default=ch.tensor(0), dist_reduce_fx='sum') - - def update(self, sample: ch.Tensor): - self.sum += sample.sum() - self.count += sample.numel() - - def compute(self): - return self.sum.float() / self.count - -# Running -def make_config(quiet=False): - config = get_current_config() - parser = ArgumentParser(description='Fast imagenet training') - config.augment_argparse(parser) - config.collect_argparse_args(parser) - config.validate(mode='stderr') - if not quiet: - config.summary() - -if __name__ == "__main__": - make_config() - ImageNetTrainer.launch_from_args() \ No newline at end of file From cb7dbf8525fbd55941694a3e9ca45c0831eb2448 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 6 Apr 2022 22:56:54 -0500 Subject: [PATCH 20/33] updating ffcv installation instructions --- ffcv_installation.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ffcv_installation.sh b/ffcv_installation.sh index a4e0cf60..9a6af4b2 100644 --- a/ffcv_installation.sh +++ b/ffcv_installation.sh @@ -4,12 +4,16 @@ docker run --gpus all --ipc=host --name kartik_imagenet -it -v /hdd1/ILSVRC2012: sudo docker pull nvcr.io/nvidia/pytorch:22.01-py3 -# to fix dependencies for ffcv -apt-get install ffmpeg libsm6 libxext6 -y - # clone repo git clone git@github.com:libffcv/ffcv-imagenet.git +conda create -y -n ffcv python=3.9 cupy pkg-config compilers libjpeg-turbo opencv pytorch torchvision cudatoolkit=11.3 numba -c pytorch -c conda-forge +conda activate ffcv +pip install ffcv + +# to fix dependencies for ffcv +apt-get install ffmpeg libsm6 libxext6 -y + # install dependencies cd ffcv-imagenet pip install -r requirements.txt From 8ddafce82b506d25992e62a9626095f0cceb2d9a Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 7 Apr 2022 11:58:19 -0500 Subject: [PATCH 21/33] adding regular imagenet training config --- .../imagenet/resnet50_sparsity_10.yml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml new file mode 100644 index 00000000..b32adbde --- /dev/null +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml @@ -0,0 +1,66 @@ +subfolder: regular_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: ImageNet +name: resnet50_imagenet +data: /home/ubuntu/data/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.001 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 10 +wd: 0.0 +momentum: 0.9 +batch_size: 1024 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From ba9c35e1e7d0ed04675d65e06c984a55cbf10e8b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 7 Apr 2022 17:42:07 +0000 Subject: [PATCH 22/33] adding imagenet only config --- .../{resnet50_sparsity_10.yml => resnet50_sparsity_5.yml} | 2 +- data/__init__.py | 2 +- imagenet_exec.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename configs/hypercube/resnet50/imagenet/{resnet50_sparsity_10.yml => resnet50_sparsity_5.yml} (98%) diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml similarity index 98% rename from configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml rename to configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml index b32adbde..da1530a8 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml @@ -26,7 +26,7 @@ fine_tune_lr_policy: multistep_lr epochs: 10 wd: 0.0 momentum: 0.9 -batch_size: 1024 +batch_size: 128 mixed_precision: True # ===== Sparsity =========== # diff --git a/data/__init__.py b/data/__init__.py index 596b67d6..82b1d473 100644 --- a/data/__init__.py +++ b/data/__init__.py @@ -3,4 +3,4 @@ from data.tinyimagenet import TinyImageNet from data.mnist import MNIST from data.bigcifar import BigCIFAR10 -from data.ffcv_imagenet import FfcvImageNet +# from data.ffcv_imagenet import FfcvImageNet diff --git a/imagenet_exec.sh b/imagenet_exec.sh index 4f99ae85..e2b72ecd 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -24,8 +24,8 @@ done BLOCK -conf_file="configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml" -log_root="resnet50_ffcv" +conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" +log_root="resnet50_imagenet" log_end="_log" python main.py \ --config "$conf_file" > "$log_root$log_end" 2>&1 & From 29a545de5c140b4176921514392c0f7a371cd4a8 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 7 Apr 2022 13:08:10 -0500 Subject: [PATCH 23/33] modifying config to 5% target sparsity --- .../{resnet50_sparsity_10.yml => resnet50_sparsity_5.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename configs/hypercube/resnet50/imagenet/{resnet50_sparsity_10.yml => resnet50_sparsity_5.yml} (100%) diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml similarity index 100% rename from configs/hypercube/resnet50/imagenet/resnet50_sparsity_10.yml rename to configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml From c51024ad5b45e2175e392d15cdc9fbb6a6214708 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 7 Apr 2022 13:45:53 -0500 Subject: [PATCH 24/33] commenting ffcv for full imagenet --- configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml | 6 +++--- data/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml index b32adbde..3576abcf 100644 --- a/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml +++ b/configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml @@ -13,7 +13,7 @@ arch: ResNet50 # ===== Dataset ===== # dataset: ImageNet name: resnet50_imagenet -data: /home/ubuntu/data/ +data: /data/imagenet/ # ===== Learning Rate Policy ======== # optimizer: sgd @@ -26,7 +26,7 @@ fine_tune_lr_policy: multistep_lr epochs: 10 wd: 0.0 momentum: 0.9 -batch_size: 1024 +batch_size: 64 mixed_precision: True # ===== Sparsity =========== # @@ -57,7 +57,7 @@ lmbda: 0.0000001 # 1e-4 #0.00005 # 5e-5 # ===== Hardware setup ===== # workers: 4 -gpu: 0 +gpu: 1 # ===== Checkpointing ===== # checkpoint_at_prune: False diff --git a/data/__init__.py b/data/__init__.py index 596b67d6..82b1d473 100644 --- a/data/__init__.py +++ b/data/__init__.py @@ -3,4 +3,4 @@ from data.tinyimagenet import TinyImageNet from data.mnist import MNIST from data.bigcifar import BigCIFAR10 -from data.ffcv_imagenet import FfcvImageNet +# from data.ffcv_imagenet import FfcvImageNet From edc87b8693438983c855f3e28047efa0d7d03401 Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Thu, 7 Apr 2022 13:49:35 -0500 Subject: [PATCH 25/33] changing benchmark for faster imagenet run --- utils/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/utils.py b/utils/utils.py index 1db4f656..2ed92dba 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -25,9 +25,9 @@ def set_seed(seed): np.random.seed(seed) os.environ['PYTHONHASHSEED'] = str(seed) # making sure GPU runs are deterministic even if they are slower - torch.backends.cudnn.deterministic = True + torch.backends.cudnn.deterministic = False # this causes the code to vary across runs. I don't want that for now. - # torch.backends.cudnn.benchmark = True + torch.backends.cudnn.benchmark = True print("Seeded everything: {}".format(seed)) From 70ad77a02d5e6b20d6fbd948fa2892af44664b00 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 7 Apr 2022 13:56:48 -0500 Subject: [PATCH 26/33] merging exec --- ...esnet50_sparsity_10.yml => resnet50_sparsity_5.yml} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename configs/hypercube/resnet50/ffcv_imagenet/{resnet50_sparsity_10.yml => resnet50_sparsity_5.yml} (92%) diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5.yml similarity index 92% rename from configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml rename to configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5.yml index 1d074cfb..b89f53cf 100644 --- a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10.yml +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5.yml @@ -26,7 +26,7 @@ fine_tune_lr_policy: multistep_lr epochs: 88 wd: 0.0 momentum: 0.9 -batch_size: 100 +batch_size: 256 mixed_precision: True # ===== Sparsity =========== # @@ -35,7 +35,7 @@ bn_type: NonAffineBatchNorm freeze_weights: True prune_type: BottomK # enter target sparsity here -target_sparsity: 10 +target_sparsity: 5 # decide if you want to "unflag" unflag_before_finetune: False init: signed_constant @@ -53,11 +53,11 @@ quantize_threshold: 0.5 # ===== Regularization ===== # regularization: L2 -lmbda: 0.000001 # 1e-4 #0.00005 # 5e-5 +lmbda: 0.0000001 # 1e-4 #0.00005 # 5e-5 # ===== Hardware setup ===== # -workers: 4 -gpu: 1 +workers: 8 +gpu: 0 # ===== Checkpointing ===== # checkpoint_at_prune: False From fdc66866e5ec84711dcd909fb31a1453a083becc Mon Sep 17 00:00:00 2001 From: root Date: Thu, 7 Apr 2022 13:57:58 -0500 Subject: [PATCH 27/33] merging conflicts in exec script --- imagenet_exec.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index e2b72ecd..23782ad2 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -24,8 +24,8 @@ done BLOCK -conf_file="configs/hypercube/resnet50/imagenet/resnet50_sparsity_5.yml" -log_root="resnet50_imagenet" +conf_file="configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5.yml" +log_root="resnet50_ffcv" log_end="_log" python main.py \ --config "$conf_file" > "$log_root$log_end" 2>&1 & From bb859692e42f10777ab8b1a14c9cb52112a32344 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 7 Apr 2022 14:01:02 -0500 Subject: [PATCH 28/33] bringing back ffcv --- data/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/__init__.py b/data/__init__.py index 82b1d473..596b67d6 100644 --- a/data/__init__.py +++ b/data/__init__.py @@ -3,4 +3,4 @@ from data.tinyimagenet import TinyImageNet from data.mnist import MNIST from data.bigcifar import BigCIFAR10 -# from data.ffcv_imagenet import FfcvImageNet +from data.ffcv_imagenet import FfcvImageNet From 2adb1d8ae2bac9d8b840aa862b235fec0cac108f Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Fri, 8 Apr 2022 00:22:34 -0500 Subject: [PATCH 29/33] adding config for imagenet sparsity 20% --- .../ffcv_imagenet/resnet50_sparsity_20.yml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml new file mode 100644 index 00000000..6426b350 --- /dev/null +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml @@ -0,0 +1,66 @@ +subfolder: ffcv_imagenet_resnet50_sp20 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: FfcvImageNet +name: resnet50_ffcv_imagenet +data: /workspace/ffcv-imagenet/data/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.001 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 64 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 20 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.00000001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From c64c23dfe5ccc8eb942a41bff3ff0fb80abfdd83 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 8 Apr 2022 11:19:39 -0500 Subject: [PATCH 30/33] adding actual_val_loader so code doesn't break --- .../ffcv_imagenet/resnet50_sparsity_20.yml | 9 ++++--- data/ffcv_imagenet.py | 26 ++++++++++++++++--- imagenet_exec.sh | 4 +-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml index 6426b350..564b8a45 100644 --- a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml @@ -19,14 +19,14 @@ data: /workspace/ffcv-imagenet/data/ optimizer: sgd lr: 0.1 #0.01 lr_policy: cosine_lr #constant_lr #multistep_lr -fine_tune_lr: 0.001 +fine_tune_lr: 0.01 fine_tune_lr_policy: multistep_lr # ===== Network training config ===== # epochs: 88 wd: 0.0 momentum: 0.9 -batch_size: 64 +batch_size: 256 mixed_precision: True # ===== Sparsity =========== # @@ -41,6 +41,7 @@ unflag_before_finetune: False init: signed_constant score_init: unif #skew #half #bimodal #skew # bern scale_fan: False #True +use_full_data: True # ===== Rounding ===== # round: naive @@ -53,7 +54,7 @@ quantize_threshold: 0.5 # ===== Regularization ===== # regularization: L2 -lmbda: 0.00000001 # 1e-4 #0.00005 # 5e-5 +lmbda: 0.000001 # 1e-4 #0.00005 # 5e-5 # ===== Hardware setup ===== # workers: 4 @@ -63,4 +64,4 @@ gpu: 1 checkpoint_at_prune: False # ==== sanity check ==== # -skip_sanity_checks: True +skip_sanity_checks: False diff --git a/data/ffcv_imagenet.py b/data/ffcv_imagenet.py index e61fa8e9..665e8a74 100644 --- a/data/ffcv_imagenet.py +++ b/data/ffcv_imagenet.py @@ -28,6 +28,8 @@ import torch.multiprocessing torch.multiprocessing.set_sharing_strategy("file_system") +from torch.utils.data import random_split + class FfcvImageNet: def __init__(self, args): super(FfcvImageNet, self).__init__() @@ -39,22 +41,40 @@ def __init__(self, args): use_cuda = torch.cuda.is_available() # Data loading code - train_dataset = os.path.join(data_root, "train_500_0.50_90.ffcv") + dataset = os.path.join(data_root, "train_500_0.50_90.ffcv") val_dataset = os.path.join(data_root, "val_500_0.50_90.ffcv") # Data loading code - kwargs = {"num_workers": 1, "in_memory": 1, + kwargs = {"num_workers": 6, "in_memory": 1, "distributed": False, "resolution": 256} self.IMAGENET_MEAN = np.array([0.485, 0.456, 0.406]) * 255 self.IMAGENET_STD = np.array([0.229, 0.224, 0.225]) * 255 self.DEFAULT_CROP_RATIO = 224/256 + if parser_args.use_full_data: + train_dataset = dataset + # use_full_data => we are not tuning hyperparameters + actual_val_dataset = val_dataset + else: + # this will break right now. But where is the test set even? + val_size = 5000 + train_size = len(dataset) - val_size + train_dataset, actual_val_dataset = random_split(dataset, [train_size, val_size]) + + self.train_loader = self.create_train_loader(train_dataset, kwargs['num_workers'], parser_args.batch_size, kwargs['distributed'], - kwargs['in_memory']) + kwargs['in_memory'] + ) + self.actual_val_loader = self.create_train_loader(actual_val_dataset, + kwargs['num_workers'], + parser_args.batch_size, + kwargs['distributed'], + kwargs['in_memory'] + ) self.val_loader = self.create_val_loader(val_dataset, kwargs['num_workers'], parser_args.batch_size, diff --git a/imagenet_exec.sh b/imagenet_exec.sh index 23782ad2..be413bf3 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -24,8 +24,8 @@ done BLOCK -conf_file="configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5.yml" -log_root="resnet50_ffcv" +conf_file="configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml" +log_root="resnet50_ffcv_sp20" log_end="_log" python main.py \ --config "$conf_file" > "$log_root$log_end" 2>&1 & From 3efec34cf1787575f4cb7413d73abacee5d6c56b Mon Sep 17 00:00:00 2001 From: ksreenivasan Date: Wed, 13 Apr 2022 18:50:32 -0500 Subject: [PATCH 31/33] pushing configs for ffcv_imagenet_serial --- .../resnet20/sreeniva_resnet20_1_4_best.yml | 64 +++++++++++++++++ .../resnet50_sparsity_10_finetune_only.yml | 68 +++++++++++++++++++ utils/utils.py | 4 +- 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 configs/hypercube/resnet20/sreeniva_resnet20_1_4_best.yml create mode 100644 configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10_finetune_only.yml diff --git a/configs/hypercube/resnet20/sreeniva_resnet20_1_4_best.yml b/configs/hypercube/resnet20/sreeniva_resnet20_1_4_best.yml new file mode 100644 index 00000000..5385628f --- /dev/null +++ b/configs/hypercube/resnet20/sreeniva_resnet20_1_4_best.yml @@ -0,0 +1,64 @@ +# subfolder: sreeniva_hc_1_44_best +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: resnet20 + +# ===== Dataset ===== # +dataset: CIFAR10 +name: resnet20_quantized_iter_hc + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 150 +wd: 0.0 +momentum: 0.9 +batch_size: 128 + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 1.44 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 4 +gpu: 2 + +# ===== Checkpointing ===== # +checkpoint_at_prune: True + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10_finetune_only.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10_finetune_only.yml new file mode 100644 index 00000000..b3effa24 --- /dev/null +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_10_finetune_only.yml @@ -0,0 +1,68 @@ +subfolder: ffcv_imagenet_resnet50_finetune_only +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: FfcvImageNet +name: resnet18_ffcv +data: /workspace/ffcv-imagenet/data/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.4 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.01 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 128 +mixed_precision: True +only_finetune: True +resume: results/resnet50_ffcv_bad_finetune/model_before_finetune.pth + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 10 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.000001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 6 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/utils/utils.py b/utils/utils.py index 1db4f656..2ed92dba 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -25,9 +25,9 @@ def set_seed(seed): np.random.seed(seed) os.environ['PYTHONHASHSEED'] = str(seed) # making sure GPU runs are deterministic even if they are slower - torch.backends.cudnn.deterministic = True + torch.backends.cudnn.deterministic = False # this causes the code to vary across runs. I don't want that for now. - # torch.backends.cudnn.benchmark = True + torch.backends.cudnn.benchmark = True print("Seeded everything: {}".format(seed)) From dd2b4513bb64078d5a9b0b02425afe7e0b1c8b78 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 13 Apr 2022 18:53:39 -0500 Subject: [PATCH 32/33] pushing ffcv configs --- .../ffcv_imagenet/resnet50_sparsity_50.yml | 66 +++++++++++++++++++ .../ffcv_imagenet/resnet50_sparsity_5_v2.yml | 66 +++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_50.yml create mode 100644 configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5_v2.yml diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_50.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_50.yml new file mode 100644 index 00000000..97eeb818 --- /dev/null +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_50.yml @@ -0,0 +1,66 @@ +subfolder: ffcv_imagenet_resnet50_sp50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: FfcvImageNet +name: resnet18_ffcv +data: /workspace/ffcv-imagenet/data/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.001 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 200 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 50 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 8 +gpu: 1 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True diff --git a/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5_v2.yml b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5_v2.yml new file mode 100644 index 00000000..f5c62b1b --- /dev/null +++ b/configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_5_v2.yml @@ -0,0 +1,66 @@ +subfolder: ffcv_imagenet_resnet50 +# trial_num: 1 +#lam_finetune_loss: 1 +#num_step_finetune: 5 + +# Hypercube optimization +algo: 'hc_iter' +iter_period: 5 + +# Architecture +arch: ResNet50 + +# ===== Dataset ===== # +dataset: FfcvImageNet +name: resnet18_ffcv +data: /workspace/ffcv-imagenet/data/ + +# ===== Learning Rate Policy ======== # +optimizer: sgd +lr: 0.1 #0.01 +lr_policy: cosine_lr #constant_lr #multistep_lr +fine_tune_lr: 0.001 +fine_tune_lr_policy: multistep_lr + +# ===== Network training config ===== # +epochs: 88 +wd: 0.0 +momentum: 0.9 +batch_size: 256 +mixed_precision: True + +# ===== Sparsity =========== # +conv_type: SubnetConv +bn_type: NonAffineBatchNorm +freeze_weights: True +prune_type: BottomK +# enter target sparsity here +target_sparsity: 5 +# decide if you want to "unflag" +unflag_before_finetune: False +init: signed_constant +score_init: unif #skew #half #bimodal #skew # bern +scale_fan: False #True + +# ===== Rounding ===== # +round: naive +noise: True +noise_ratio: 0 + +# ===== Quantization ===== # +hc_quantized: True +quantize_threshold: 0.5 + +# ===== Regularization ===== # +regularization: L2 +lmbda: 0.0000001 # 1e-4 #0.00005 # 5e-5 + +# ===== Hardware setup ===== # +workers: 8 +gpu: 0 + +# ===== Checkpointing ===== # +checkpoint_at_prune: False + +# ==== sanity check ==== # +skip_sanity_checks: True From 68fbb9a0803217f74a053b35d4d8d21209549500 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 13 Apr 2022 18:54:32 -0500 Subject: [PATCH 33/33] handling conflicts --- imagenet_exec.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imagenet_exec.sh b/imagenet_exec.sh index be413bf3..f8c08079 100644 --- a/imagenet_exec.sh +++ b/imagenet_exec.sh @@ -24,8 +24,8 @@ done BLOCK -conf_file="configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_20.yml" -log_root="resnet50_ffcv_sp20" +conf_file="configs/hypercube/resnet50/ffcv_imagenet/resnet50_sparsity_50.yml" +log_root="resnet50_ffcv_sp50" log_end="_log" python main.py \ --config "$conf_file" > "$log_root$log_end" 2>&1 &