From 47c7166ac07dc75a663d9f704dbf5e0904155c49 Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Sun, 18 Oct 2020 14:17:17 -0600 Subject: [PATCH 1/5] transf_rotations = False to have deterministic results --- config.py | 8 ++++---- runTest.py | 30 ++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/config.py b/config.py index 4e866a2..bdfe64d 100644 --- a/config.py +++ b/config.py @@ -11,10 +11,10 @@ # data settings dataset_path = "zerobox_dataset" class_name = "zerobox_class" -modelname = "zerobox_test" +modelname = "zerobox-2009-5" -# img_size = (448, 448) -img_size = (480, 270) +img_size = (448, 448) +# img_size = (480, 270) img_dims = [3] + list(img_size) add_img_noise = 0.01 @@ -36,7 +36,7 @@ # dataloader parameters n_transforms = 1 # number of transformations per sample in training -n_transforms_test = 64 # number of transformations per sample in testing +n_transforms_test = 1 # number of transformations per sample in testing batch_size = 1 # actual batch size is this value multiplied by n_transforms(_test) batch_size_test = 1 # batch_size * n_transforms // n_transforms_test diff --git a/runTest.py b/runTest.py index 6ad53ed..e2ef7ba 100644 --- a/runTest.py +++ b/runTest.py @@ -16,8 +16,9 @@ from utils import * from localization import export_gradient_maps from torch.autograd import Variable +from sklearn.metrics import roc_curve -def test(model, test_loader): +def test(model, test_loader, target_threshold): print("Running test") optimizer = torch.optim.Adam(model.nf.parameters(), lr=c.lr_init, betas=(0.8, 0.8), eps=1e-04, weight_decay=1e-5) # score_obs = Score_Observer('AUROC') @@ -56,10 +57,22 @@ def test(model, test_loader): print(f"test_labels={test_labels}, is_anomaly={is_anomaly},anomaly_score={anomaly_score}") # score_obs.update(roc_auc_score(is_anomaly, anomaly_score), epoch, # print_score=c.verbose or epoch == c.meta_epochs - 1) + # Code to calculate the accurarcy from Lihang's code + is_anomaly_detected = np.array([0 if l < target_threshold else 1 for l in anomaly_score]) - if c.grad_map_viz: - print("saving gradient maps...") - export_gradient_maps(model, test_loader, optimizer, -1) + # calculate test accuracy + error_count = 0 + for i in range(len(is_anomaly)): + if is_anomaly[i] != is_anomaly_detected[i]: + error_count += 1 + + test_accuracy = 1 - float(error_count) / len(is_anomaly) + + print(f"n_transforms_test = {c.n_transforms_test}, target_threshold={target_threshold}, test_accuracy={test_accuracy}") + + # if c.grad_map_viz: + # print("saving gradient maps...") + # export_gradient_maps(model, test_loader, optimizer, -1) def load_testloader(data_dir_test): def target_transform(target): @@ -100,14 +113,15 @@ def target_transform(target): # train_set, test_set = load_datasets(c.dataset_path, c.class_name) # _, test_loader = make_dataloaders(train_set, test_set) -test_loader = load_testloader("group15B.avi/") -# model = torch.load("../zerobox-v2/zerobox_differnet_model.pt", map_location=torch.device('cpu')) -model = torch.load("models/zerobox_test.pt", map_location=torch.device('cpu')) +test_loader = load_testloader("zerobox_dataset/zerobox-2009-4-rotated/2-good") +model = torch.load("../zerobox-v2/zerobox-2009-5.pt", map_location=torch.device('cpu')) +# model = torch.load("models/zerobox_test.pt", map_location=torch.device('cpu')) +target_threshold= 1.2587523460388184 print("starting to run tests after loaded model and test dataset") time_start = time.time() # model = load_model(c.modelname) -test(model, test_loader) +test(model, test_loader, target_threshold) time_end = time.time() time_c = time_end - time_start # 运行所花时间 print("time cost: {:f} s".format(time_c)) From eb4028eaa1c4bd9c0367c2f70b96197c8467b1a8 Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Sun, 18 Oct 2020 19:40:13 -0600 Subject: [PATCH 2/5] running on 2009-5 training data --- runTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runTest.py b/runTest.py index e2ef7ba..ec815f2 100644 --- a/runTest.py +++ b/runTest.py @@ -113,7 +113,7 @@ def target_transform(target): # train_set, test_set = load_datasets(c.dataset_path, c.class_name) # _, test_loader = make_dataloaders(train_set, test_set) -test_loader = load_testloader("zerobox_dataset/zerobox-2009-4-rotated/2-good") +test_loader = load_testloader("zerobox_dataset/zerobox-2009-5/train") model = torch.load("../zerobox-v2/zerobox-2009-5.pt", map_location=torch.device('cpu')) # model = torch.load("models/zerobox_test.pt", map_location=torch.device('cpu')) target_threshold= 1.2587523460388184 From cc5989abcc63ea1765711b3dd3e9f1da077375aa Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Mon, 19 Oct 2020 19:42:25 -0600 Subject: [PATCH 3/5] Switch to use built-in dataloader since my own does not work --- config.py | 7 +++---- runTest.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index bdfe64d..6b4159c 100644 --- a/config.py +++ b/config.py @@ -10,16 +10,15 @@ # data settings dataset_path = "zerobox_dataset" -class_name = "zerobox_class" +class_name = "zerobox-2009-5" modelname = "zerobox-2009-5" img_size = (448, 448) -# img_size = (480, 270) img_dims = [3] + list(img_size) add_img_noise = 0.01 # transformation settings -transf_rotations = False +transf_rotations = True transf_brightness = 0.0 transf_contrast = 0.0 transf_saturation = 0.0 @@ -36,7 +35,7 @@ # dataloader parameters n_transforms = 1 # number of transformations per sample in training -n_transforms_test = 1 # number of transformations per sample in testing +n_transforms_test = 16 # number of transformations per sample in testing batch_size = 1 # actual batch size is this value multiplied by n_transforms(_test) batch_size_test = 1 # batch_size * n_transforms // n_transforms_test diff --git a/runTest.py b/runTest.py index ec815f2..8b8ad24 100644 --- a/runTest.py +++ b/runTest.py @@ -110,10 +110,10 @@ def target_transform(target): return testloader ########################## Main #################### -# train_set, test_set = load_datasets(c.dataset_path, c.class_name) -# _, test_loader = make_dataloaders(train_set, test_set) +train_set, test_set = load_datasets(c.dataset_path, c.class_name) +_, test_loader = make_dataloaders(train_set, test_set) -test_loader = load_testloader("zerobox_dataset/zerobox-2009-5/train") +# test_loader = load_testloader("zerobox_dataset/zerobox-2009-5/train") model = torch.load("../zerobox-v2/zerobox-2009-5.pt", map_location=torch.device('cpu')) # model = torch.load("models/zerobox_test.pt", map_location=torch.device('cpu')) target_threshold= 1.2587523460388184 From c24aeb43c5a8c12e5f15613e6c2eb06bcf3c9bab Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Sat, 24 Oct 2020 17:52:05 -0600 Subject: [PATCH 4/5] restore random rotation --- runTest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runTest.py b/runTest.py index 8b8ad24..f70de26 100644 --- a/runTest.py +++ b/runTest.py @@ -93,8 +93,8 @@ def target_transform(target): class_idx += 1 augmentative_transforms = [] - # if c.transf_rotations: - # augmentative_transforms += [transforms.RandomRotation(180)] + if c.transf_rotations: + augmentative_transforms += [transforms.RandomRotation(180)] if c.transf_brightness > 0.0 or c.transf_contrast > 0.0 or c.transf_saturation > 0.0: augmentative_transforms += [transforms.ColorJitter(brightness=c.transf_brightness, contrast=c.transf_contrast, saturation=c.transf_saturation)] @@ -110,10 +110,10 @@ def target_transform(target): return testloader ########################## Main #################### -train_set, test_set = load_datasets(c.dataset_path, c.class_name) -_, test_loader = make_dataloaders(train_set, test_set) +# train_set, test_set = load_datasets(c.dataset_path, c.class_name) +# _, test_loader = make_dataloaders(train_set, test_set) -# test_loader = load_testloader("zerobox_dataset/zerobox-2009-5/train") +test_loader = load_testloader("zerobox_dataset/zerobox-2009-5/train") model = torch.load("../zerobox-v2/zerobox-2009-5.pt", map_location=torch.device('cpu')) # model = torch.load("models/zerobox_test.pt", map_location=torch.device('cpu')) target_threshold= 1.2587523460388184 From 2d1eb01981b73e3bd7b5d5d5c1f5f9e22bbaf9c6 Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Thu, 28 Jan 2021 20:16:24 -0700 Subject: [PATCH 5/5] adjust test to run zian's model: zerobox#221 --- runTest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runTest.py b/runTest.py index f70de26..956519d 100644 --- a/runTest.py +++ b/runTest.py @@ -113,10 +113,10 @@ def target_transform(target): # train_set, test_set = load_datasets(c.dataset_path, c.class_name) # _, test_loader = make_dataloaders(train_set, test_set) -test_loader = load_testloader("zerobox_dataset/zerobox-2009-5/train") -model = torch.load("../zerobox-v2/zerobox-2009-5.pt", map_location=torch.device('cpu')) +test_loader = load_testloader("dataset/group15B.avi.Products/test") +model = torch.load("models/zerobox-2010-1-black-yolo_0_0.10_0.05_0.10_0.05_0.9980.pth", map_location=torch.device('cpu')) # model = torch.load("models/zerobox_test.pt", map_location=torch.device('cpu')) -target_threshold= 1.2587523460388184 +target_threshold= 3.5129451751708984 print("starting to run tests after loaded model and test dataset") time_start = time.time()