From daffa38bcca6e6d14dc5ec7acc50c7db1fb389ca Mon Sep 17 00:00:00 2001 From: Kangbeen Ko Date: Thu, 9 Feb 2023 23:11:02 +0900 Subject: [PATCH 01/11] docs(.gitignore): add ignorance about .DS_Store --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4478f9c0..0a302a3c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ __pycache__ /results /figs .vscode -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints +.DS_Store From 300b76e4945e751c80d46c5a5bfd83de1ffb332a Mon Sep 17 00:00:00 2001 From: Kangbeen Ko Date: Thu, 9 Feb 2023 23:11:37 +0900 Subject: [PATCH 02/11] feat(API/dataloader_s4a.py): add fake class_map information infront of ann --- API/dataloader_s4a.py | 7 +++++++ main.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/API/dataloader_s4a.py b/API/dataloader_s4a.py index 1cca1012..1202f079 100644 --- a/API/dataloader_s4a.py +++ b/API/dataloader_s4a.py @@ -205,6 +205,13 @@ def prepare_train_img(self, idx: int) -> dict: ann = np.load(annpath) img = img[self.start_month:self.end_month] + + ##################TODO################################### + # add class map info infront of the input + class_map = np.ones(ann.shape) + ann = np.stack([class_map, ann],0) + # print('######NEW_ANN_SHAPE#######', ann.shape) + ######################################################### if self.transforms: img, ann = self.transforms(img, ann) diff --git a/main.py b/main.py index 9a433e06..4cfcb207 100644 --- a/main.py +++ b/main.py @@ -112,6 +112,12 @@ def train(self): num_updates = 0 # constants for other methods: eta = 1.0 # PredRNN + ##### for size test ##### + train_features, train_labels = next(iter(self.train_loader)) # iter: iterable to iteraator / next: iterator에서 한 칸씩 전진 + print(f'Features batch shape: {train_features.size()}') # tensor.size() == tensor.shape + print(f'Labels batch shape: {train_labels.size()}') + # quit() + ######################### for epoch in range(self.config['epoch']): loss_mean = 0.0 From fcefc16073c0d2a7677991a1b865ad7be2e4e219 Mon Sep 17 00:00:00 2001 From: Kangbeen Ko Date: Mon, 13 Feb 2023 00:27:02 +0900 Subject: [PATCH 03/11] docs: todo --- API/dataloader_s4a.py | 79 ++++++++++++++++++---------------- models/simvp_model.py | 99 +++++++++++++++++++++++++------------------ 2 files changed, 101 insertions(+), 77 deletions(-) diff --git a/API/dataloader_s4a.py b/API/dataloader_s4a.py index 1feb2fcd..ab9620bf 100644 --- a/API/dataloader_s4a.py +++ b/API/dataloader_s4a.py @@ -35,11 +35,12 @@ NORMALIZATION_DIV = 10000 + def min_max_normalize(image, percentile=2): image = image.astype('float32') - percent_min = np.percentile(image, percentile, axis=(0,1)) - percent_max = np.percentile(image, 100-percentile, axis=(0,1)) + percent_min = np.percentile(image, percentile, axis=(0, 1)) + percent_max = np.percentile(image, 100-percentile, axis=(0, 1)) mask = np.mean(image, axis=2) != 0 if image.shape[1] * image.shape[0] - np.sum(mask) > 0: @@ -48,9 +49,9 @@ def min_max_normalize(image, percentile=2): percent_min = np.nanpercentile(mdata, percentile, axis=(0, 1)) norm = (image-percent_min) / (percent_max - percent_min) - norm[norm<0] = 0 - norm[norm>1] = 1 - norm = norm * mask[:,:,np.newaxis] + norm[norm < 0] = 0 + norm[norm > 1] = 1 + norm = norm * mask[:, :, np.newaxis] # norm = (norm * 255).astype('uint8') * mask[:,:,np.newaxis] return norm @@ -117,12 +118,12 @@ def __init__( root_dir: str = None, img_dir: str = None, ann_dir: str = None, - band_mode: str ='nrgb', + band_mode: str = 'nrgb', bands: list = None, linear_encoder: dict = None, start_month: int = 0, end_month: int = 12, - output_size: tuple = (64,64), + output_size: tuple = (64, 64), binary_labels: bool = False, return_parcels: bool = False, mode: str = 'test', @@ -154,8 +155,9 @@ def __init__( self.bands = ['B02', 'B03', 'B04', 'B08'] elif band_mode == 'rdeg': - self.bands = ['B02', 'B03', 'B04', 'B08', 'B05', 'B06', 'B07', 'B8A', 'B11', 'B12'] - + self.bands = ['B02', 'B03', 'B04', 'B08', + 'B05', 'B06', 'B07', 'B8A', 'B11', 'B12'] + else: raise RuntimeError @@ -204,10 +206,12 @@ def __init__( self.std = 1 print('Rootdir: {}'.format(self.root_dir)) - print('Scenario: {}, MODE: {}, length of datasets: {}'.format(self.scenario, self.mode, len(self.img_infos))) - print('Acquired Data Month: From {} to {}'.format(self.start_month + 1, self.end_month + 1)) - print(f'Data shape: [T, C, H, W] ({self.end_month - self.start_month}, {len(self.bands)}, {output_size[0]}, {output_size[0]})') - + print('Scenario: {}, MODE: {}, length of datasets: {}'.format( + self.scenario, self.mode, len(self.img_infos))) + print('Acquired Data Month: From {} to {}'.format( + self.start_month + 1, self.end_month + 1)) + print( + f'Data shape: [T, C, H, W] ({self.end_month - self.start_month}, {len(self.bands)}, {output_size[0]}, {output_size[0]})') def prepare_train_img(self, idx: int) -> dict: if self.band_mode == 'nrgb': @@ -217,7 +221,8 @@ def prepare_train_img(self, idx: int) -> dict: readpath = os.path.join(self.img_dir, self.img_infos[idx]) img = np.load(readpath) - rdegpath = os.path.join(os.path.dirname(self.img_dir), 'rdeg', self.img_infos[idx]) + rdegpath = os.path.join(os.path.dirname( + self.img_dir), 'rdeg', self.img_infos[idx]) rdeg = np.load(rdegpath) img = np.stack([img, rdeg], axis=1) else: @@ -227,11 +232,11 @@ def prepare_train_img(self, idx: int) -> dict: ann = np.load(annpath) img = img[self.start_month:self.end_month] - + ##################TODO################################### # add class map info infront of the input class_map = np.ones(ann.shape) - ann = np.stack([class_map, ann],0) + ann = np.stack([class_map, ann], 0) # print('######NEW_ANN_SHAPE#######', ann.shape) ######################################################### @@ -244,32 +249,32 @@ def _normalize(self, img): if self.min_max_normalize: T, C, H, W = img.shape img = img.reshape(T*C, H, W) - img = min_max_normalize(img.transpose(1,2,0), percentile=0.5) - img = img.transpose(2,0,1) + img = min_max_normalize(img.transpose(1, 2, 0), percentile=0.5) + img = img.transpose(2, 0, 1) img = img.reshape(T, C, H, W) else: - img = np.divide(img, NORMALIZATION_DIV) # / 10000 + img = np.divide(img, NORMALIZATION_DIV) # / 10000 return img - def __getitem__(self, idx: int) -> dict: img, ann = self.prepare_train_img(idx) # Normalize data to range [0-1] img = self._normalize(img) - # out = {} - # if self.return_parcels: - # parcels = ann != 0 - # out['parcels'] = parcels - - # if self.binary_labels: - # # Map 0: background class, 1: parcel - # ann[ann != 0] = 1 - # else: - # # Map labels to 0-len(unique(crop_id)) see config - # # labels = np.vectorize(self.linear_encoder.get)(labels) - # _ = np.zeros_like(ann) + ########TODO######## + out = {} + if self.return_parcels: + parcels = ann != 0 + out['parcels'] = parcels + + if self.binary_labels: + # Map 0: background class, 1: parcel + ann[ann != 0] = 1 + else: + # Map labels to 0-len(unique(crop_id)) see config + # labels = np.vectorize(self.linear_encoder.get)(labels) + _ = np.zeros_like(ann) # for crop_id, linear_id in self.linear_encoder.items(): # _[ann == crop_id] = linear_id # ann = _ @@ -296,9 +301,10 @@ def __len__(self): def load_data(batch_size, val_batch_size, num_workers, data_root): - train_set = NpyPADDataset(root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, mode='train') - test_set = NpyPADDataset(root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, mode='val') - + train_set = NpyPADDataset( + root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, mode='train') + test_set = NpyPADDataset( + root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, mode='val') dataloader_train = torch.utils.data.DataLoader( train_set, batch_size=batch_size, shuffle=True, pin_memory=True, drop_last=True, num_workers=num_workers) @@ -312,5 +318,6 @@ def load_data(batch_size, val_batch_size, num_workers, data_root): if __name__ == '__main__': rootdir = '/home/jovyan/shared_volume/data/newdata' - dataset = NpyPADDataset(root_dir=rootdir, band_mode='nrgb', start_month=1, end_month=13, mode='val') + dataset = NpyPADDataset( + root_dir=rootdir, band_mode='nrgb', start_month=1, end_month=13, mode='val') print(dataset[0]) diff --git a/models/simvp_model.py b/models/simvp_model.py index 7d547120..57acdd61 100644 --- a/models/simvp_model.py +++ b/models/simvp_model.py @@ -6,8 +6,10 @@ def sampling_generator(N, reverse=False): samplings = [False, True] * (N // 2) - if reverse: return list(reversed(samplings[:N])) - else: return samplings[:N] + if reverse: + return list(reversed(samplings[:N])) + else: + return samplings[:N] class BasicConv2d(nn.Module): @@ -49,7 +51,7 @@ def __init__(self, C_in, C_out, kernel_size=3, downsampling=False, upsampling=Fa padding = (kernel_size - stride + 1) // 2 self.conv = BasicConv2d(C_in, C_out, kernel_size=kernel_size, stride=stride, upsampling=upsampling, - padding=padding, act_norm=act_norm) + padding=padding, act_norm=act_norm) def forward(self, x): y = self.conv(x) @@ -57,15 +59,16 @@ def forward(self, x): class GroupConv2d(nn.Module): - def __init__(self, in_channels, out_channels, kernel_size, stride, padding,groups,act_norm=False): + def __init__(self, in_channels, out_channels, kernel_size, stride, padding, groups, act_norm=False): super(GroupConv2d, self).__init__() - self.act_norm=act_norm - if in_channels%groups != 0: - groups=1 - self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding,groups=groups) - self.norm = nn.GroupNorm(groups,out_channels) + self.act_norm = act_norm + if in_channels % groups != 0: + groups = 1 + self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, + stride=stride, padding=padding, groups=groups) + self.norm = nn.GroupNorm(groups, out_channels) self.activate = nn.LeakyReLU(0.2, inplace=True) - + def forward(self, x): y = self.conv(x) if self.act_norm: @@ -74,14 +77,15 @@ def forward(self, x): class gInception_ST(nn.Module): - def __init__(self, C_in, C_hid, C_out, incep_ker = [3,5,7,11], groups = 8): + def __init__(self, C_in, C_hid, C_out, incep_ker=[3, 5, 7, 11], groups=8): super(gInception_ST, self).__init__() self.conv1 = nn.Conv2d(C_in, C_hid, kernel_size=1, stride=1, padding=0) layers = [] for ker in incep_ker: - layers.append(GroupConv2d(C_hid, C_out, kernel_size=ker, stride=1, padding=ker//2, groups=groups, act_norm=True)) - + layers.append(GroupConv2d(C_hid, C_out, kernel_size=ker, + stride=1, padding=ker//2, groups=groups, act_norm=True)) + self.layers = nn.Sequential(*layers) def forward(self, x): @@ -114,7 +118,8 @@ def __init__(self, C_hid, C_out, N_S, spatio_kernel): samplings = sampling_generator(N_S, reverse=True) super(Decoder, self).__init__() self.dec = nn.Sequential( - *[ConvSC(C_hid, C_hid, spatio_kernel, upsampling=s) for s in samplings[:-1]], + *[ConvSC(C_hid, C_hid, spatio_kernel, upsampling=s) + for s in samplings[:-1]], ConvSC(C_hid, C_hid, spatio_kernel, upsampling=samplings[-1]) ) self.readout = nn.Conv2d(C_hid, C_out, 1) @@ -133,11 +138,13 @@ def __init__(self, in_channels, out_channels, mlp_ratio=8., drop=0.0, drop_path= self.in_channels = in_channels self.out_channels = out_channels - self.block = GASubBlock(in_channels, kernel_size=21, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path, act_layer=nn.GELU) + self.block = GASubBlock(in_channels, kernel_size=21, mlp_ratio=mlp_ratio, + drop=drop, drop_path=drop_path, act_layer=nn.GELU) if in_channels != out_channels: - self.reduction = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) - + self.reduction = nn.Conv2d( + in_channels, out_channels, kernel_size=1, stride=1, padding=0) + def forward(self, x): z = self.block(x) return z if self.in_channels == self.out_channels else self.reduction(z) @@ -148,10 +155,13 @@ def __init__(self, channel_in, channel_hid, N2, mlp_ratio=4., drop=0.0, drop_pat super(Mid_GANet, self).__init__() self.N2 = N2 - enc_layers = [GABlock(channel_in, channel_hid, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path)] + enc_layers = [GABlock( + channel_in, channel_hid, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path)] for i in range(1, N2-1): - enc_layers.append(GABlock(channel_hid, channel_hid, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path)) - enc_layers.append(GABlock(channel_hid, channel_in, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path)) + enc_layers.append(GABlock(channel_hid, channel_hid, + mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path)) + enc_layers.append(GABlock(channel_hid, channel_in, + mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path)) self.enc = nn.Sequential(*enc_layers) def forward(self, x): @@ -167,28 +177,32 @@ def forward(self, x): class Mid_IncepNet(nn.Module): - def __init__(self, channel_in, channel_hid, N2, incep_ker=[3,5,7,11], groups=8, **kwargs): + def __init__(self, channel_in, channel_hid, N2, incep_ker=[3, 5, 7, 11], groups=8, **kwargs): super(Mid_IncepNet, self).__init__() self.N2 = N2 - enc_layers = [gInception_ST(channel_in, channel_hid//2, channel_hid, incep_ker= incep_ker, groups=groups)] - for i in range(1,N2-1): - enc_layers.append(gInception_ST(channel_hid, channel_hid//2, channel_hid, incep_ker= incep_ker, groups=groups)) - enc_layers.append(gInception_ST(channel_hid, channel_hid//2, channel_hid, incep_ker= incep_ker, groups=groups)) - - - dec_layers = [gInception_ST(channel_hid, channel_hid//2, channel_hid, incep_ker= incep_ker, groups=groups)] - for i in range(1,N2-1): - dec_layers.append(gInception_ST(2*channel_hid, channel_hid//2, channel_hid, incep_ker= incep_ker, groups=groups)) - dec_layers.append(gInception_ST(2*channel_hid, channel_hid//2, channel_in, incep_ker= incep_ker, groups=groups)) + enc_layers = [gInception_ST( + channel_in, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)] + for i in range(1, N2-1): + enc_layers.append(gInception_ST( + channel_hid, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)) + enc_layers.append(gInception_ST( + channel_hid, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)) + dec_layers = [gInception_ST( + channel_hid, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)] + for i in range(1, N2-1): + dec_layers.append(gInception_ST( + 2*channel_hid, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)) + dec_layers.append(gInception_ST( + 2*channel_hid, channel_hid//2, channel_in, incep_ker=incep_ker, groups=groups)) self.enc = nn.Sequential(*enc_layers) self.dec = nn.Sequential(*dec_layers) def forward(self, x): - B,T,C,H,W = x.shape - x = x.reshape(B,T*C,H,W) + B, T, C, H, W = x.shape + x = x.reshape(B, T*C, H, W) # encoder skips = [] @@ -200,20 +214,21 @@ def forward(self, x): # decoder z = self.dec[0](z) - for i in range(1,self.N2): - z = self.dec[i](torch.cat([z, skips[-i]], dim=1) ) + for i in range(1, self.N2): + z = self.dec[i](torch.cat([z, skips[-i]], dim=1)) - y = z.reshape(B,T,C,H,W) + y = z.reshape(B, T, C, H, W) return y class SimVP_Model(nn.Module): def __init__(self, in_shape, hid_S=16, hid_T=256, N_S=4, N_T=4, model_type='', - mlp_ratio=8., drop=0.0, drop_path=0.0, spatio_kernel_enc=3, spatio_kernel_dec=3, pre_seq_length=10, aft_seq_length=10, **kwargs): + mlp_ratio=8., drop=0.0, drop_path=0.0, spatio_kernel_enc=3, spatio_kernel_dec=3, pre_seq_length=10, aft_seq_length=10, **kwargs): super(SimVP_Model, self).__init__() T, C, H, W = in_shape self.aft_seq_length = aft_seq_length - self.conv = nn.Conv2d(T, aft_seq_length, kernel_size=1, stride=1, padding=0) + self.conv = nn.Conv2d( + T, aft_seq_length, kernel_size=1, stride=1, padding=0) self.enc = Encoder(C, hid_S, N_S, spatio_kernel_enc) self.dec = Decoder(hid_S, C, N_S, spatio_kernel_dec) @@ -221,8 +236,10 @@ def __init__(self, in_shape, hid_S=16, hid_T=256, N_S=4, N_T=4, model_type='', if model_type == 'IncepU': self.hid = Mid_IncepNet(T*hid_S, hid_T, N_T) else: - self.hid = Mid_GANet(T*hid_S, hid_T, N_T, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path) - + self.hid = Mid_GANet( + T*hid_S, hid_T, N_T, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path) + + ########TODO######## def forward(self, x_raw): B, T, C, H, W = x_raw.shape x = x_raw.view(B*T, C, H, W) @@ -244,5 +261,5 @@ def forward(self, x_raw): Y = self.conv(Y) Y = Y.reshape(B, C, self.aft_seq_length, H, W) Y = Y.transpose(2, 1) - + return Y From d332c09410ce7d6b2e9d97cb0510752bc2193066 Mon Sep 17 00:00:00 2001 From: KevinTheRainmaker Date: Wed, 15 Feb 2023 12:23:33 +0000 Subject: [PATCH 04/11] feat: add annotation into batch_x --- API/dataloader_s4a.py | 25 +++++++++++-------------- methods/simvp.py | 19 ++++++++++++++++--- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/API/dataloader_s4a.py b/API/dataloader_s4a.py index ab9620bf..3945e81d 100644 --- a/API/dataloader_s4a.py +++ b/API/dataloader_s4a.py @@ -233,13 +233,6 @@ def prepare_train_img(self, idx: int) -> dict: img = img[self.start_month:self.end_month] - ##################TODO################################### - # add class map info infront of the input - class_map = np.ones(ann.shape) - ann = np.stack([class_map, ann], 0) - # print('######NEW_ANN_SHAPE#######', ann.shape) - ######################################################### - if self.transforms: img, ann = self.transforms(img, ann) @@ -275,9 +268,9 @@ def __getitem__(self, idx: int) -> dict: # Map labels to 0-len(unique(crop_id)) see config # labels = np.vectorize(self.linear_encoder.get)(labels) _ = np.zeros_like(ann) - # for crop_id, linear_id in self.linear_encoder.items(): - # _[ann == crop_id] = linear_id - # ann = _ + for crop_id, linear_id in self.linear_encoder.items(): + _[ann == crop_id] = linear_id + ann = _ # # # Map all classes NOT in linear encoder's values to 0 # ann[~np.isin(ann, list(self.linear_encoder.values()))] = 0 @@ -302,9 +295,9 @@ def __len__(self): def load_data(batch_size, val_batch_size, num_workers, data_root): train_set = NpyPADDataset( - root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, mode='train') + root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, get_ann=True, mode='train') test_set = NpyPADDataset( - root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, mode='val') + root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, get_ann=True, mode='val') dataloader_train = torch.utils.data.DataLoader( train_set, batch_size=batch_size, shuffle=True, pin_memory=True, drop_last=True, num_workers=num_workers) @@ -319,5 +312,9 @@ def load_data(batch_size, val_batch_size, num_workers, data_root): if __name__ == '__main__': rootdir = '/home/jovyan/shared_volume/data/newdata' dataset = NpyPADDataset( - root_dir=rootdir, band_mode='nrgb', start_month=1, end_month=13, mode='val') - print(dataset[0]) + root_dir=rootdir, band_mode='nrgb', start_month=1, end_month=13, get_ann=True, mode='val') + + dataloader = torch.utils.data.DataLoader( + dataset, batch_size=batch_size, shuffle=True, pin_memory=True, drop_last=True, num_workers=num_workers) + + # print(dataset[0][2]) diff --git a/methods/simvp.py b/methods/simvp.py index 2ac8f596..a85d3b73 100644 --- a/methods/simvp.py +++ b/methods/simvp.py @@ -14,6 +14,7 @@ def __init__(self, args, device, steps_per_epoch): self.model = self._build_model(self.config) self.model_optim, self.scheduler = self._init_optimizer(steps_per_epoch) self.criterion = nn.MSELoss() + self.conv_c4 = nn.Conv2d(in_channels=1, out_channels=4, kernel_size=1) def _build_model(self, config): return SimVP_Model(**config).to(self.device) @@ -46,8 +47,12 @@ def train_one_epoch(self, train_loader, epoch, num_updates, loss_mean, **kwargs) self.model.train() train_pbar = tqdm(train_loader) - for batch_x, batch_y in train_pbar: + for batch_x, batch_y, ann in train_pbar: self.model_optim.zero_grad() + ann = ann.unsqueeze(1) + ann = self.conv_c4(ann.float()) + ann = ann.unsqueeze(1) + batch_x = torch.cat([batch_x, ann], dim=1) batch_x, batch_y = batch_x.to(self.device), batch_y.to(self.device) pred_y = self._predict(batch_x) @@ -68,7 +73,11 @@ def vali_one_epoch(self, vali_loader, **kwargs): self.model.eval() preds_lst, trues_lst, total_loss = [], [], [] vali_pbar = tqdm(vali_loader) - for i, (batch_x, batch_y) in enumerate(vali_pbar): + for i, (batch_x, batch_y, ann) in enumerate(vali_pbar): + ann = ann.unsqueeze(1) + ann = self.conv_c4(ann.float()) + ann = ann.unsqueeze(1) + batch_x = torch.cat([batch_x, ann], dim=1) batch_x, batch_y = batch_x.to(self.device), batch_y.to(self.device) pred_y = self._predict(batch_x) loss = self.criterion(pred_y, batch_y) @@ -91,7 +100,11 @@ def test_one_epoch(self, test_loader, **kwargs): self.model.eval() inputs_lst, trues_lst, preds_lst = [], [], [] test_pbar = tqdm(test_loader) - for batch_x, batch_y in test_pbar: + for batch_x, batch_y, ann in test_pbar: + ann = ann.unsqueeze(1) + ann = self.conv_c4(ann.float()) + ann = ann.unsqueeze(1) + batch_x = torch.cat([batch_x, ann], dim=1) pred_y = self._predict(batch_x.to(self.device)) list(map(lambda data, lst: lst.append(data.detach().cpu().numpy()), [ From ae345b0bc06a95e19270ce0ecefa43121b53f60f Mon Sep 17 00:00:00 2001 From: KevinTheRainmaker Date: Wed, 15 Feb 2023 12:27:25 +0000 Subject: [PATCH 05/11] build: change config settings --- configs/SimVP.py | 2 +- constants/dataset_constant.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/SimVP.py b/configs/SimVP.py index 4278551c..5d4429be 100644 --- a/configs/SimVP.py +++ b/configs/SimVP.py @@ -1,6 +1,6 @@ method = 'SimVP' model_type = 'gSTA' hid_S = 64 -hid_T = 512 +hid_T = 576 N_T = 8 N_S = 4 \ No newline at end of file diff --git a/constants/dataset_constant.py b/constants/dataset_constant.py index 237994de..21a7c2d3 100644 --- a/constants/dataset_constant.py +++ b/constants/dataset_constant.py @@ -6,8 +6,8 @@ 'total_length': 20 }, 's4a': { - 'in_shape': [8, 4, 64, 64], - 'pre_seq_length': 8, + 'in_shape': [9, 4, 64, 64], + 'pre_seq_length': 9, 'aft_seq_length': 4, 'total_length': 12 }, From 6c4f83bdf0a5c2d3410c206d221725073718a2ba Mon Sep 17 00:00:00 2001 From: KevinTheRainmaker Date: Wed, 15 Feb 2023 12:28:27 +0000 Subject: [PATCH 06/11] test: turn off size test --- main.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 4cfcb207..2a8f8504 100644 --- a/main.py +++ b/main.py @@ -113,9 +113,17 @@ def train(self): # constants for other methods: eta = 1.0 # PredRNN ##### for size test ##### - train_features, train_labels = next(iter(self.train_loader)) # iter: iterable to iteraator / next: iterator에서 한 칸씩 전진 - print(f'Features batch shape: {train_features.size()}') # tensor.size() == tensor.shape - print(f'Labels batch shape: {train_labels.size()}') + # train_inputs, train_outputs, train_labels, = next(iter(self.train_loader)) + # print(f'Inputs batch shape: {train_inputs.size()}') + # print(f'Outputs batch shape: {train_outputs.size()}') + # print(f'Labels batch shape: {train_labels.size()}') + # _label = np.array(train_labels) + # label = torch.from_numpy(_label) + # label = label.unsqueeze(1) + # conv = torch.nn.Conv2d(in_channels=1, out_channels=4, kernel_size=1) + # label = conv(label.float()) + # print(f'NEW Labels batch shape: {label.size()}') + # quit() ######################### for epoch in range(self.config['epoch']): From 17bf04c5739559a9861fd5dee2c7e967247caa62 Mon Sep 17 00:00:00 2001 From: KevinTheRainmaker Date: Thu, 16 Feb 2023 02:26:31 +0000 Subject: [PATCH 07/11] fix: seq length error --- configs/SimVP.py | 4 ++-- constants/dataset_constant.py | 4 ++-- main.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/configs/SimVP.py b/configs/SimVP.py index 5d4429be..3879c4e9 100644 --- a/configs/SimVP.py +++ b/configs/SimVP.py @@ -1,6 +1,6 @@ method = 'SimVP' model_type = 'gSTA' hid_S = 64 -hid_T = 576 +hid_T = 512 N_T = 8 -N_S = 4 \ No newline at end of file +N_S = 4 diff --git a/constants/dataset_constant.py b/constants/dataset_constant.py index 21a7c2d3..e7e08248 100644 --- a/constants/dataset_constant.py +++ b/constants/dataset_constant.py @@ -7,7 +7,7 @@ }, 's4a': { 'in_shape': [9, 4, 64, 64], - 'pre_seq_length': 9, + 'pre_seq_length': 8, 'aft_seq_length': 4, 'total_length': 12 }, @@ -47,4 +47,4 @@ 'aft_seq_length': 12, 'total_length': 24 } -} \ No newline at end of file +} diff --git a/main.py b/main.py index 2a8f8504..66fc3872 100644 --- a/main.py +++ b/main.py @@ -51,7 +51,8 @@ def __init__(self, args): _tmp_input = torch.ones(1, self.args.total_length, Hp, Wp, Cp).to(self.device) _tmp_flag = torch.ones(1, self.args.total_length - 2, Hp, Wp, Cp).to(self.device) flops = FlopCountAnalysis(self.method.model, (_tmp_input, _tmp_flag)) - print_log(flop_count_table(flops)) + '''RuntimeError 발생''' + # print_log(flop_count_table(flops)) def _acquire_device(self): if self.args.use_gpu: From e2bb4138adcea64e49f4f252a9a12c0682d2594b Mon Sep 17 00:00:00 2001 From: KevinTheRainmaker Date: Thu, 16 Feb 2023 02:27:46 +0000 Subject: [PATCH 08/11] docs: ignore exp_results that logging results after revision --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a302a3c..28514343 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__ /results +exp_results/ /figs .vscode .ipynb_checkpoints From 800841113878256dd8c9896f88c547b2f546d1a3 Mon Sep 17 00:00:00 2001 From: KevinTheRainmaker Date: Thu, 16 Feb 2023 02:43:24 +0000 Subject: [PATCH 09/11] docs: add comments --- API/dataloader_s4a.py | 2 +- constants/dataset_constant.py | 2 ++ methods/simvp.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/API/dataloader_s4a.py b/API/dataloader_s4a.py index 3945e81d..b75ff013 100644 --- a/API/dataloader_s4a.py +++ b/API/dataloader_s4a.py @@ -293,7 +293,7 @@ def __len__(self): def load_data(batch_size, val_batch_size, num_workers, data_root): - + # get_ann 옵션 추가 train_set = NpyPADDataset( root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, get_ann=True, mode='train') test_set = NpyPADDataset( diff --git a/constants/dataset_constant.py b/constants/dataset_constant.py index e7e08248..7160699a 100644 --- a/constants/dataset_constant.py +++ b/constants/dataset_constant.py @@ -6,6 +6,8 @@ 'total_length': 20 }, 's4a': { + # in_shape 수정 + # [8, 4, 64, 64] -> [9, 4, 64, 64] 'in_shape': [9, 4, 64, 64], 'pre_seq_length': 8, 'aft_seq_length': 4, diff --git a/methods/simvp.py b/methods/simvp.py index a85d3b73..c172f7c2 100644 --- a/methods/simvp.py +++ b/methods/simvp.py @@ -47,12 +47,24 @@ def train_one_epoch(self, train_loader, epoch, num_updates, loss_mean, **kwargs) self.model.train() train_pbar = tqdm(train_loader) + # ann 추출 후 batch_x에 추가 for batch_x, batch_y, ann in train_pbar: self.model_optim.zero_grad() + + # [16, 64, 64] -> [16, 1, 64, 64] + # B H W -> B C H W ann = ann.unsqueeze(1) + + # [16, 1, 64, 64] -> [16, 4, 64, 64] ann = self.conv_c4(ann.float()) + + # [16, 4, 64, 64] -> [16, 1, 4, 64, 64] + # B C H W -> B T C H W ann = ann.unsqueeze(1) + + # [16, 8, 4, 64, 64] -> [16, 9, 4, 64, 64] batch_x = torch.cat([batch_x, ann], dim=1) + batch_x, batch_y = batch_x.to(self.device), batch_y.to(self.device) pred_y = self._predict(batch_x) From 0a64c0690e74f7e98982e7ffd52392b4eaa8a3dc Mon Sep 17 00:00:00 2001 From: KevinTheRainmaker Date: Thu, 16 Feb 2023 08:29:59 +0000 Subject: [PATCH 10/11] docs: confirm x_raw and y's shape --- models/simvp_model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/simvp_model.py b/models/simvp_model.py index 57acdd61..bc4d2b1f 100644 --- a/models/simvp_model.py +++ b/models/simvp_model.py @@ -239,9 +239,9 @@ def __init__(self, in_shape, hid_S=16, hid_T=256, N_S=4, N_T=4, model_type='', self.hid = Mid_GANet( T*hid_S, hid_T, N_T, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path) - ########TODO######## def forward(self, x_raw): B, T, C, H, W = x_raw.shape + print('x_raw shape: ', x_raw.shape) x = x_raw.view(B*T, C, H, W) embed, skip = self.enc(x) @@ -254,6 +254,7 @@ def forward(self, x_raw): Y = self.dec(hid, skip) Y = Y.reshape(B, T, C, H, W) + if T != self.aft_seq_length: Y = Y.transpose(2, 1) @@ -261,5 +262,5 @@ def forward(self, x_raw): Y = self.conv(Y) Y = Y.reshape(B, C, self.aft_seq_length, H, W) Y = Y.transpose(2, 1) - + print('Y shape: ', Y.shape) return Y From b8c8aeb8ee31d701905278f24d3da07432685590 Mon Sep 17 00:00:00 2001 From: KevinTheRainmaker Date: Sat, 11 Mar 2023 04:02:33 +0000 Subject: [PATCH 11/11] backup --- API/dataloader_s4a.py | 10 +++++--- configs/SimVP.py | 4 ++-- constants/dataset_constant.py | 3 ++- main.py | 2 +- methods/simvp.py | 45 +++++++++++++++++++++-------------- models/simvp_model.py | 2 -- 6 files changed, 39 insertions(+), 27 deletions(-) diff --git a/API/dataloader_s4a.py b/API/dataloader_s4a.py index b75ff013..372d4870 100644 --- a/API/dataloader_s4a.py +++ b/API/dataloader_s4a.py @@ -282,7 +282,11 @@ def __getitem__(self, idx: int) -> dict: input = torch.from_numpy(input).contiguous().float() if self.get_ann: - ann = ann.astype('int16') + ann = ann.astype('float32') + _ = np.zeros_like(ann) + for crop_id, linear_id in LINEAR_ENCODER.items(): + _[ann == crop_id] = linear_id + ann = _ / len(SELECTED_CLASSES) ann = torch.from_numpy(ann).contiguous().long() return input, output, ann else: @@ -295,9 +299,9 @@ def __len__(self): def load_data(batch_size, val_batch_size, num_workers, data_root): # get_ann 옵션 추가 train_set = NpyPADDataset( - root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, get_ann=True, mode='train') + root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, get_ann=False, mode='train') test_set = NpyPADDataset( - root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, get_ann=True, mode='val') + root_dir=data_root, band_mode='nrgb', start_month=1, end_month=13, get_ann=False, mode='val') dataloader_train = torch.utils.data.DataLoader( train_set, batch_size=batch_size, shuffle=True, pin_memory=True, drop_last=True, num_workers=num_workers) diff --git a/configs/SimVP.py b/configs/SimVP.py index 3879c4e9..1a12dc28 100644 --- a/configs/SimVP.py +++ b/configs/SimVP.py @@ -1,6 +1,6 @@ method = 'SimVP' model_type = 'gSTA' -hid_S = 64 -hid_T = 512 +hid_S = 16 +hid_T = 256 N_T = 8 N_S = 4 diff --git a/constants/dataset_constant.py b/constants/dataset_constant.py index 7160699a..17888e8d 100644 --- a/constants/dataset_constant.py +++ b/constants/dataset_constant.py @@ -8,7 +8,8 @@ 's4a': { # in_shape 수정 # [8, 4, 64, 64] -> [9, 4, 64, 64] - 'in_shape': [9, 4, 64, 64], + # 'in_shape': [9, 4, 64, 64], + 'in_shape': [8, 4, 64, 64], 'pre_seq_length': 8, 'aft_seq_length': 4, 'total_length': 12 diff --git a/main.py b/main.py index 66fc3872..5791b24d 100644 --- a/main.py +++ b/main.py @@ -52,7 +52,7 @@ def __init__(self, args): _tmp_flag = torch.ones(1, self.args.total_length - 2, Hp, Wp, Cp).to(self.device) flops = FlopCountAnalysis(self.method.model, (_tmp_input, _tmp_flag)) '''RuntimeError 발생''' - # print_log(flop_count_table(flops)) + print_log(flop_count_table(flops)) def _acquire_device(self): if self.args.use_gpu: diff --git a/methods/simvp.py b/methods/simvp.py index c172f7c2..2d3eb1a2 100644 --- a/methods/simvp.py +++ b/methods/simvp.py @@ -14,7 +14,7 @@ def __init__(self, args, device, steps_per_epoch): self.model = self._build_model(self.config) self.model_optim, self.scheduler = self._init_optimizer(steps_per_epoch) self.criterion = nn.MSELoss() - self.conv_c4 = nn.Conv2d(in_channels=1, out_channels=4, kernel_size=1) + # self.conv_c4 = nn.Conv2d(in_channels=1, out_channels=4, kernel_size=1) def _build_model(self, config): return SimVP_Model(**config).to(self.device) @@ -48,22 +48,25 @@ def train_one_epoch(self, train_loader, epoch, num_updates, loss_mean, **kwargs) train_pbar = tqdm(train_loader) # ann 추출 후 batch_x에 추가 - for batch_x, batch_y, ann in train_pbar: + # for batch_x, batch_y, ann in train_pbar: + for batch_x, batch_y in train_pbar: self.model_optim.zero_grad() - # [16, 64, 64] -> [16, 1, 64, 64] # B H W -> B C H W - ann = ann.unsqueeze(1) + # ann = ann.unsqueeze(1) # [16, 1, 64, 64] -> [16, 4, 64, 64] - ann = self.conv_c4(ann.float()) - + # ann = self.conv_c4(ann.float()) + + # repeat으로 차원 추가 + # ann = ann.repeat(1,4,1,1) + # [16, 4, 64, 64] -> [16, 1, 4, 64, 64] # B C H W -> B T C H W - ann = ann.unsqueeze(1) + # ann = ann.unsqueeze(1) # [16, 8, 4, 64, 64] -> [16, 9, 4, 64, 64] - batch_x = torch.cat([batch_x, ann], dim=1) + # batch_x = torch.cat([batch_x, ann], dim=1) batch_x, batch_y = batch_x.to(self.device), batch_y.to(self.device) pred_y = self._predict(batch_x) @@ -85,11 +88,14 @@ def vali_one_epoch(self, vali_loader, **kwargs): self.model.eval() preds_lst, trues_lst, total_loss = [], [], [] vali_pbar = tqdm(vali_loader) - for i, (batch_x, batch_y, ann) in enumerate(vali_pbar): - ann = ann.unsqueeze(1) - ann = self.conv_c4(ann.float()) - ann = ann.unsqueeze(1) - batch_x = torch.cat([batch_x, ann], dim=1) + # for i, (batch_x, batch_y, ann) in enumerate(vali_pbar): + for i, (batch_x, batch_y) in enumerate(vali_pbar): + # ann = ann.unsqueeze(1) + # ann = self.conv_c4(ann.float()) + # ann = ann.repeat(1,4,1,1) + + # ann = ann.unsqueeze(1) + # batch_x = torch.cat([batch_x, ann], dim=1) batch_x, batch_y = batch_x.to(self.device), batch_y.to(self.device) pred_y = self._predict(batch_x) loss = self.criterion(pred_y, batch_y) @@ -112,11 +118,14 @@ def test_one_epoch(self, test_loader, **kwargs): self.model.eval() inputs_lst, trues_lst, preds_lst = [], [], [] test_pbar = tqdm(test_loader) - for batch_x, batch_y, ann in test_pbar: - ann = ann.unsqueeze(1) - ann = self.conv_c4(ann.float()) - ann = ann.unsqueeze(1) - batch_x = torch.cat([batch_x, ann], dim=1) + # for batch_x, batch_y, ann in test_pbar: + for batch_x, batch_y in test_pbar: + # ann = ann.unsqueeze(1) + # ann = self.conv_c4(ann.float()) + # ann = ann.repeat(1,4,1,1) + + # ann = ann.unsqueeze(1) + # batch_x = torch.cat([batch_x, ann], dim=1) pred_y = self._predict(batch_x.to(self.device)) list(map(lambda data, lst: lst.append(data.detach().cpu().numpy()), [ diff --git a/models/simvp_model.py b/models/simvp_model.py index bc4d2b1f..f6e043d7 100644 --- a/models/simvp_model.py +++ b/models/simvp_model.py @@ -241,7 +241,6 @@ def __init__(self, in_shape, hid_S=16, hid_T=256, N_S=4, N_T=4, model_type='', def forward(self, x_raw): B, T, C, H, W = x_raw.shape - print('x_raw shape: ', x_raw.shape) x = x_raw.view(B*T, C, H, W) embed, skip = self.enc(x) @@ -262,5 +261,4 @@ def forward(self, x_raw): Y = self.conv(Y) Y = Y.reshape(B, C, self.aft_seq_length, H, W) Y = Y.transpose(2, 1) - print('Y shape: ', Y.shape) return Y