From 4193a47f3139b99479a5066f825ae267c0dbb550 Mon Sep 17 00:00:00 2001 From: Misha Wagner Date: Fri, 28 Oct 2022 15:39:35 +0100 Subject: [PATCH 1/6] Remove unused latent masking code. We can always add this back later if needed. --- experiments.py | 2 -- main.py | 10 ---------- 2 files changed, 12 deletions(-) diff --git a/experiments.py b/experiments.py index 72c3512..15100b0 100644 --- a/experiments.py +++ b/experiments.py @@ -40,8 +40,6 @@ class Experiment: quadrant_threshold: int = 0 sparsity: int = 1 # repr_loss scaled up by sparsity, applied every 1/sparsity latent_noise_std: float = 0 - latent_masking: bool = False - latent_masking_incentive: float = 0.1 baseline = Experiment( diff --git a/main.py b/main.py index 7ba8320..e02b51a 100644 --- a/main.py +++ b/main.py @@ -438,8 +438,6 @@ def _train(experiment: Experiment) -> TrainResult: target_latent_fn: Callable repr_loss_mask_fn: Callable - latent_use_mask_fn = torch.nn.Sigmoid() - if experiment.loss_quadrants == "all": repr_loss_mask_fn = lambda x: torch.ones(x.shape[0]) repr_loss_scale = 1.0 @@ -527,14 +525,6 @@ def _train(experiment: Experiment) -> TrainResult: vector_input = vector latent_repr = encoder(vector_input) - if experiment.latent_masking: - latent_repr = latent_repr[: experiment.preferred_rep_size] - repr_mask = latent_use_mask_fn(latent_repr) - # TODO: Unused variable? - repr_use_loss = torch.mean(repr_mask) - else: - repr_use_loss = torch.Tensor(0) - noise = torch.normal( mean=0, std=experiment.latent_noise_std, size=latent_repr.shape ) From af7105c77c53f6e06b099e6cde198368ed52f965 Mon Sep 17 00:00:00 2001 From: Misha Wagner Date: Fri, 28 Oct 2022 15:39:51 +0100 Subject: [PATCH 2/6] Remove unused if. --- main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/main.py b/main.py index e02b51a..7dadbf7 100644 --- a/main.py +++ b/main.py @@ -490,9 +490,6 @@ def _train(experiment: Experiment) -> TrainResult: step_results = [] encoder_to_decoder_idx = list(range(len(models))) for step in range(experiment.num_batches): - # TODO: Delete this if? - if experiment.dropout_prob is not None and step == 9000: - pass if experiment.shuffle_decoders: random.shuffle(encoder_to_decoder_idx) From b653d9a2a8ae996f8a6da62fc11a405ead0ceaef Mon Sep 17 00:00:00 2001 From: Misha Wagner Date: Fri, 28 Oct 2022 15:41:02 +0100 Subject: [PATCH 3/6] Wrap diagonal representation fn. This means we don't "over-encode" data into the preferred representation, which will make comparisons fairer. --- main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.py b/main.py index 7dadbf7..208e697 100644 --- a/main.py +++ b/main.py @@ -677,8 +677,7 @@ def _make_diagonal_repr_fn(rep_size: int) -> Callable: def diagonal_repr_target(_input: torch.Tensor) -> torch.Tensor: assert rep_size + 1 <= _input.shape[1] dir_1 = _input[:, :rep_size] - # TODO: Wrap this so that we don't use N+1 variables? - dir_2 = _input[:, 1 : rep_size + 1] + dir_2 = torch.concat([_input[:, 1 : rep_size], _input[:, :1]], dim=1) repr_target = (dir_1 + dir_2) / np.sqrt(2) return repr_target From 2d92588ad6258200f29052c248385848a4d5d015 Mon Sep 17 00:00:00 2001 From: Misha Wagner Date: Fri, 28 Oct 2022 15:43:10 +0100 Subject: [PATCH 4/6] Remove redundant TODO. --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 208e697..b23425d 100644 --- a/main.py +++ b/main.py @@ -542,7 +542,6 @@ def _train(experiment: Experiment) -> TrainResult: target_latent_fn(latent_repr), ) # Scaling here to compensate for quadrant sparsity - # TODO: Should we roll this into `experiment.representation_loss`? representation_loss *= repr_loss_scale loss = reconstruction_loss if experiment.representation_loss is not None: From e0c3254e8e0bf1b4744942bb50dcc487c7e4ef73 Mon Sep 17 00:00:00 2001 From: Misha Wagner Date: Fri, 28 Oct 2022 15:46:47 +0100 Subject: [PATCH 5/6] Remove unused print statement. --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index b23425d..85233fe 100644 --- a/main.py +++ b/main.py @@ -453,7 +453,6 @@ def _train(experiment: Experiment) -> TrainResult: raise ValueError( f"Loss quadrant must be 'all', 'bin_sum' or 'bin_val', got {experiment.loss_quadrants}." ) - print(f"repr_loss_scale = {repr_loss_scale}") if experiment.use_class: reconstruction_loss_fn = torch.nn.CrossEntropyLoss() From 83c66dc521a544e5428647b0f743597d2906942d Mon Sep 17 00:00:00 2001 From: Misha Wagner Date: Sat, 29 Oct 2022 17:10:55 +0100 Subject: [PATCH 6/6] Formatting fixes. --- main.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 85233fe..680d2e2 100644 --- a/main.py +++ b/main.py @@ -346,12 +346,16 @@ def _hyperparameter_search(*experiments_iterable: Experiment) -> None: df = [] for train_result in train_results: last_step = max(step_result.step for step_result in train_result.step_results) - reconstruction_loss_p2 = np.mean([ - step_result.reconstruction_loss_p2 - for step_result in train_result.step_results - if step_result.step >= last_step * 0.9 - ]) - df.append(dict(tag=train_result.tag, reconstruction_loss_p2=reconstruction_loss_p2)) + reconstruction_loss_p2 = np.mean( + [ + step_result.reconstruction_loss_p2 + for step_result in train_result.step_results + if step_result.step >= last_step * 0.9 + ] + ) + df.append( + dict(tag=train_result.tag, reconstruction_loss_p2=reconstruction_loss_p2) + ) df = pd.DataFrame(df) fig, ax = plt.subplots() sns.barplot(data=df, x="tag", y="reconstruction_loss_p2", ax=ax) @@ -675,7 +679,7 @@ def _make_diagonal_repr_fn(rep_size: int) -> Callable: def diagonal_repr_target(_input: torch.Tensor) -> torch.Tensor: assert rep_size + 1 <= _input.shape[1] dir_1 = _input[:, :rep_size] - dir_2 = torch.concat([_input[:, 1 : rep_size], _input[:, :1]], dim=1) + dir_2 = torch.concat([_input[:, 1:rep_size], _input[:, :1]], dim=1) repr_target = (dir_1 + dir_2) / np.sqrt(2) return repr_target