From 4a4a07cbf5cd31f1c0fc320ba303f14a56025e6a Mon Sep 17 00:00:00 2001 From: Zhu Lin Date: Wed, 14 Jan 2026 09:08:24 +0800 Subject: [PATCH 1/6] formatting and simplification --- analysis_CLIP.ipynb | 36 +++-- analysis_Classifier.ipynb | 24 +-- analysis_Dataset.ipynb | 105 ++++++++----- analysis_Similarity.ipynb | 22 ++- imageGen.py | 185 ++++++++++++---------- imageGen_trace.py | 8 +- tools/data.py | 188 ++++++++++------------- tools/synth.py | 315 ++++++++++++++++++-------------------- tools/transforms.py | 120 ++++++--------- tools/utils.py | 3 +- trainClassifier.py | 160 ++++++++++--------- validateClassifier.ipynb | 8 +- 12 files changed, 605 insertions(+), 569 deletions(-) diff --git a/analysis_CLIP.ipynb b/analysis_CLIP.ipynb index a059ae9..8a39329 100644 --- a/analysis_CLIP.ipynb +++ b/analysis_CLIP.ipynb @@ -110,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -128,7 +128,7 @@ "genInput_dir = f\"{dataset_type}/train2\"\n", "val_path = f\"{dataset_type}/val2\"\n", "synth_path = f\"{dataset_type}/synthetic2\"\n", - "metadata_path = f\"{dataset_type}/metadata/analysis\" # create the metadata folder if it does not exist" + "metadata_path = f\"{dataset_type}/metadata/analysis\" # create the metadata folder if it does not exist" ] }, { @@ -240,7 +240,9 @@ "metadata": {}, "outputs": [], "source": [ - "clip_df.to_csv(f\"{metadata_path}/clip_analyze_{synth_path.replace('/','_')}.csv\", index=False)" + "clip_df.to_csv(\n", + " f\"{metadata_path}/clip_analyze_{synth_path.replace('/','_')}.csv\", index=False\n", + ")" ] }, { @@ -321,7 +323,7 @@ ], "source": [ "# get base model class_to_idx\n", - "val2_path = f\"{dataset_type}/val\" # path to the full validation dataset that the classifier was tested on\n", + "val2_path = f\"{dataset_type}/val\" # path to the full validation dataset that the classifier was tested on\n", "data.cacheValData(val2_path, \"imagenet2_val\", save_path=cache_path)\n", "val2_dataset = data.loadData(\"imagenet2_val\", cache_path=cache_path)\n", "base_class_to_idx = val2_dataset.class_to_idx\n", @@ -967,11 +969,15 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "analysis.analyze_clf_stats(df, class_pairs=class_pairs, save_csv=f\"{metadata_path}/stats_clf_{synth_path.replace('/','_')}.csv\")" + "analysis.analyze_clf_stats(\n", + " df,\n", + " class_pairs=class_pairs,\n", + " save_csv=f\"{metadata_path}/stats_clf_{synth_path.replace('/','_')}.csv\",\n", + ")" ] }, { @@ -1489,7 +1495,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1506,10 +1512,14 @@ " genInput_dir = f\"{combinations[ds]['dataset_type']}/train\"\n", " val_path = f\"{combinations[ds]['dataset_type']}/val\"\n", " synth_path = f\"{combinations[ds]['dataset_type']}/synthetic2{combinations[ds]['c']}\"\n", - " metadata_path = f\"{combinations[ds]['dataset_type']}/metadata/analysis{combinations[ds]['c']}\"\n", + " metadata_path = (\n", + " f\"{combinations[ds]['dataset_type']}/metadata/analysis{combinations[ds]['c']}\"\n", + " )\n", " prompt_format = combinations[ds][\"prompt_format\"]\n", "\n", - " df = pd.read_csv(f\"{metadata_path}/clip_clf_{synth_path.replace('/','_')}.csv\").copy()\n", + " df = pd.read_csv(\n", + " f\"{metadata_path}/clip_clf_{synth_path.replace('/','_')}.csv\"\n", + " ).copy()\n", " classpairs = sorted(list(df[\"actual_class\"].unique()))\n", " classnames = dict()\n", " prompts = dict()\n", @@ -1553,7 +1563,7 @@ " # main[\"Class\"].append(classnames[row[\"actual_class\"]])\n", " # main[\"Prompt\"].append(prompts[row[\"actual_class\"]])\n", " # main[\"CLIP Score to Prompt 1\"].append(row[\"sim_0\"])\n", - " # main[\"CLIP Score to Prompt 2\"].append(row[\"sim_1\"]) " + " # main[\"CLIP Score to Prompt 2\"].append(row[\"sim_1\"])" ] }, { @@ -1624,7 +1634,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1650,7 +1660,9 @@ ], "source": [ "g = sns.FacetGrid(df_main, col=\"Type\", row=\"Dataset\", hue=\"Class\", sharex=False)\n", - "ax = g.map_dataframe(sns.kdeplot, x=\"CLIP Score to Prompt 1\", y=\"CLIP Score to Prompt 2\")\n", + "ax = g.map_dataframe(\n", + " sns.kdeplot, x=\"CLIP Score to Prompt 1\", y=\"CLIP Score to Prompt 2\"\n", + ")\n", "ax.set_titles(\"CLIP Score for {row_name}: {col_name}\")\n", "ax.figure.legend(\n", " loc=\"upper center\",\n", diff --git a/analysis_Classifier.ipynb b/analysis_Classifier.ipynb index 8633494..86affa4 100644 --- a/analysis_Classifier.ipynb +++ b/analysis_Classifier.ipynb @@ -40,12 +40,12 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "wandb_results_dir = 'results/wandb'\n", - "output_dir = 'results'" + "wandb_results_dir = \"results/wandb\"\n", + "output_dir = \"results\"" ] }, { @@ -69,16 +69,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "dfs = dict()\n", "for t in type:\n", " fs = [f for f in files if t in f]\n", - " df = pd.concat([pd.read_csv(f'{wandb_results_dir}/' + f) for f in fs], axis=1)\n", + " df = pd.concat([pd.read_csv(f\"{wandb_results_dir}/\" + f) for f in fs], axis=1)\n", " # drop columns if contain __MIN, __MAX, _step\n", - " df = df.drop([c for c in df.columns if '__MIN' in c], axis=1)\n", + " df = df.drop([c for c in df.columns if \"__MIN\" in c], axis=1)\n", " df = df.drop([c for c in df.columns if \"__MAX\" in c], axis=1)\n", " df = df.drop([c for c in df.columns if \"_step\" in c], axis=1)\n", " # rename columns by replacing model names\n", @@ -103,7 +103,7 @@ " \"epoch\": [],\n", " }\n", " for p in params:\n", - " pivot_dict[p.replace('epoch/','')] = []\n", + " pivot_dict[p.replace(\"epoch/\", \"\")] = []\n", " for i, row in df.iterrows():\n", " ep = row[\"epoch/epoch\"]\n", " for m in models:\n", @@ -111,7 +111,7 @@ " pivot_dict[\"model\"].append(m)\n", " pivot_dict[\"epoch\"].append(ep)\n", " for p in params:\n", - " param = p.replace('epoch/','')\n", + " param = p.replace(\"epoch/\", \"\")\n", " pivot_dict[param].append(row[m + \" - \" + p])\n", " pivot_df = pd.DataFrame(pivot_dict)\n", " dfs[t] = pivot_df" @@ -299,7 +299,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -376,14 +376,14 @@ } ], "source": [ - "df_plot['data'] = df_plot['model'].str.split('_').str[1]\n", + "df_plot[\"data\"] = df_plot[\"model\"].str.split(\"_\").str[1]\n", "df_plot[\"model\"] = df_plot[\"model\"].str.split(\"_\").str[0]\n", "df_plot.head(2)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -407,7 +407,7 @@ "g.add_legend()\n", "g.set_titles(\"{row_name}: {col_name}\")\n", "g.set_axis_labels(\"Epoch\", \"Accuracy@1\")\n", - "g.savefig(f'{output_dir}/merged_results.svg')" + "g.savefig(f\"{output_dir}/merged_results.svg\")" ] }, { diff --git a/analysis_Dataset.ipynb b/analysis_Dataset.ipynb index 45ac681..932b374 100644 --- a/analysis_Dataset.ipynb +++ b/analysis_Dataset.ipynb @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -138,12 +138,7 @@ "ds = (\"imagenette\", \"imagewoof\", \"stanford-dogs\")\n", "types = (\"train\", \"val\", \"synthetic\", \"synthetic-cleaned\")\n", "\n", - "ds_classes = {\n", - " \"ds\": [],\n", - " \"type\": [],\n", - " \"class\": [],\n", - " \"n_images\": []\n", - "}\n", + "ds_classes = {\"ds\": [], \"type\": [], \"class\": [], \"n_images\": []}\n", "\n", "# count number of images in each dataset\n", "for d in ds:\n", @@ -228,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -305,10 +300,14 @@ "synthetic_cleaned = synthetic_cleaned.reset_index()\n", "\n", "synthetic = synthetic.rename(columns={\"n_images\": \"n_images_synthetic\"})\n", - "synthetic_cleaned = synthetic_cleaned.rename(columns={\"n_images\": \"n_images_synthetic_cleaned\"})\n", + "synthetic_cleaned = synthetic_cleaned.rename(\n", + " columns={\"n_images\": \"n_images_synthetic_cleaned\"}\n", + ")\n", "change = pd.merge(synthetic, synthetic_cleaned, on=\"ds\")\n", "change[\"abs_change\"] = change.n_images_synthetic_cleaned - change.n_images_synthetic\n", - "change[\"rel_change\"] = (change.n_images_synthetic_cleaned - change.n_images_synthetic) / change.n_images_synthetic\n", + "change[\"rel_change\"] = (\n", + " change.n_images_synthetic_cleaned - change.n_images_synthetic\n", + ") / change.n_images_synthetic\n", "change.dropna()\n", "change.to_csv(\"results/synthetic_changes.csv\", index=False)\n", "change" @@ -323,7 +322,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -617,11 +616,19 @@ "change_by_class = df[df.type == \"synthetic-cleaned\"]\n", "change_by_class = change_by_class.groupby([\"ds\", \"class\"]).n_images\n", "change_by_class = change_by_class.sum().reset_index()\n", - "change_by_class = change_by_class.rename(columns={\"n_images\": \"n_images_synthetic_cleaned\"})\n", - "change_by_class = pd.merge(change_by_class, df[df.type == \"synthetic\"], on=[\"ds\", \"class\"])\n", + "change_by_class = change_by_class.rename(\n", + " columns={\"n_images\": \"n_images_synthetic_cleaned\"}\n", + ")\n", + "change_by_class = pd.merge(\n", + " change_by_class, df[df.type == \"synthetic\"], on=[\"ds\", \"class\"]\n", + ")\n", "change_by_class = change_by_class.rename(columns={\"n_images\": \"n_images_synthetic\"})\n", - "change_by_class[\"abs_change\"] = change_by_class.n_images_synthetic_cleaned - change_by_class.n_images_synthetic\n", - "change_by_class[\"rel_change\"] = (change_by_class.n_images_synthetic_cleaned - change_by_class.n_images_synthetic) / change_by_class.n_images_synthetic\n", + "change_by_class[\"abs_change\"] = (\n", + " change_by_class.n_images_synthetic_cleaned - change_by_class.n_images_synthetic\n", + ")\n", + "change_by_class[\"rel_change\"] = (\n", + " change_by_class.n_images_synthetic_cleaned - change_by_class.n_images_synthetic\n", + ") / change_by_class.n_images_synthetic\n", "change_by_class.dropna()\n", "change_by_class.to_csv(\"results/synthetic_changes_by_class.csv\", index=False)\n", "change_by_class" @@ -629,7 +636,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -715,12 +722,7 @@ ], "source": [ "# get list of files in each dataset, type and class\n", - "ds_files = {\n", - " \"ds\": [],\n", - " \"type\": [],\n", - " \"class\": [],\n", - " \"file\": []\n", - "}\n", + "ds_files = {\"ds\": [], \"type\": [], \"class\": [], \"file\": []}\n", "\n", "for d in ds:\n", " for t in types:\n", @@ -747,7 +749,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -845,9 +847,13 @@ } ], "source": [ - "df_synthetics = df_files[(df_files.type == \"synthetic\")|(df_files.type == \"synthetic-cleaned\")].copy()\n", + "df_synthetics = df_files[\n", + " (df_files.type == \"synthetic\") | (df_files.type == \"synthetic-cleaned\")\n", + "].copy()\n", "# file name is in seed-id_interpolation_step\n", - "df_synthetics[\"interpolation_step\"] = df_synthetics['file'].apply(lambda x: x.split(\"_\")[-1].split(\".\")[0])\n", + "df_synthetics[\"interpolation_step\"] = df_synthetics[\"file\"].apply(\n", + " lambda x: x.split(\"_\")[-1].split(\".\")[0]\n", + ")\n", "df_synthetics[\"interpolation_step\"] = df_synthetics[\"interpolation_step\"].astype(int)\n", "df_synthetics[\"interpolation_step\"] = df_synthetics[\"interpolation_step\"]\n", "df_synthetics.head()" @@ -855,7 +861,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -968,22 +974,40 @@ "# get breakdown of changes by interpolation step\n", "df_synth = df_synthetics[df_synthetics.type == \"synthetic\"].copy()\n", "df_synth_clean = df_synthetics[df_synthetics.type == \"synthetic-cleaned\"].copy()\n", - "df_synth = df_synth.groupby([\"ds\", \"class\", \"interpolation_step\"]).file.count().reset_index()\n", - "df_synth_clean = df_synth_clean.groupby([\"ds\", \"class\", \"interpolation_step\"]).file.count().reset_index()\n", + "df_synth = (\n", + " df_synth.groupby([\"ds\", \"class\", \"interpolation_step\"]).file.count().reset_index()\n", + ")\n", + "df_synth_clean = (\n", + " df_synth_clean.groupby([\"ds\", \"class\", \"interpolation_step\"])\n", + " .file.count()\n", + " .reset_index()\n", + ")\n", "df_synth = df_synth.rename(columns={\"file\": \"n_images_synthetic\"})\n", "df_synth_clean = df_synth_clean.rename(columns={\"file\": \"n_images_synthetic_cleaned\"})\n", - "interpolation_change = pd.merge(df_synth, df_synth_clean, on=[\"ds\", \"class\", \"interpolation_step\"])\n", + "interpolation_change = pd.merge(\n", + " df_synth, df_synth_clean, on=[\"ds\", \"class\", \"interpolation_step\"]\n", + ")\n", "interpolation_change = analysis.symmetrize_steps(interpolation_change)\n", - "interpolation_change['abs_change'] = interpolation_change.n_images_synthetic_cleaned - interpolation_change.n_images_synthetic\n", - "interpolation_change['rel_change'] = (interpolation_change.n_images_synthetic_cleaned - interpolation_change.n_images_synthetic) / interpolation_change.n_images_synthetic\n", - "interpolation_change[\"interpolation_step\"] = interpolation_change[\"interpolation_step\"] + 1\n", - "interpolation_change.to_csv(\"results/synthetic_changes_by_interpolation_step.csv\", index=False)\n", + "interpolation_change[\"abs_change\"] = (\n", + " interpolation_change.n_images_synthetic_cleaned\n", + " - interpolation_change.n_images_synthetic\n", + ")\n", + "interpolation_change[\"rel_change\"] = (\n", + " interpolation_change.n_images_synthetic_cleaned\n", + " - interpolation_change.n_images_synthetic\n", + ") / interpolation_change.n_images_synthetic\n", + "interpolation_change[\"interpolation_step\"] = (\n", + " interpolation_change[\"interpolation_step\"] + 1\n", + ")\n", + "interpolation_change.to_csv(\n", + " \"results/synthetic_changes_by_interpolation_step.csv\", index=False\n", + ")\n", "interpolation_change.head()" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1088,10 +1112,17 @@ ], "source": [ "# group interpolation_change by ds\n", - "interpolation_change_by_ds = interpolation_change.groupby([\"ds\", \"interpolation_step\"]).sum().reset_index()\n", - "interpolation_change_by_ds['rel_change'] = interpolation_change_by_ds['abs_change'] / interpolation_change_by_ds['n_images_synthetic']\n", + "interpolation_change_by_ds = (\n", + " interpolation_change.groupby([\"ds\", \"interpolation_step\"]).sum().reset_index()\n", + ")\n", + "interpolation_change_by_ds[\"rel_change\"] = (\n", + " interpolation_change_by_ds[\"abs_change\"]\n", + " / interpolation_change_by_ds[\"n_images_synthetic\"]\n", + ")\n", "interpolation_change_by_ds.drop(columns=[\"class\"], inplace=True)\n", - "interpolation_change_by_ds.to_csv(\"results/synthetic_changes_by_interpolation_step_by_ds.csv\", index=False)\n", + "interpolation_change_by_ds.to_csv(\n", + " \"results/synthetic_changes_by_interpolation_step_by_ds.csv\", index=False\n", + ")\n", "interpolation_change_by_ds.head()" ] } diff --git a/analysis_Similarity.ipynb b/analysis_Similarity.ipynb index 0ce59b2..dfdef85 100644 --- a/analysis_Similarity.ipynb +++ b/analysis_Similarity.ipynb @@ -47,12 +47,12 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metadata_path = \"data/imagewoof/metadata\"\n", - "dataset_name = \"Imagewoof\" # Title on the plots\n", + "dataset_name = \"Imagewoof\" # Title on the plots\n", "\n", "clip_model_path = \"openai/clip-vit-large-patch14\"\n", "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"" @@ -615,7 +615,13 @@ "metadata": {}, "outputs": [], "source": [ - "combined = {\"Interpolation Steps\": [], \"Similarity\": [], \"Dataset\": [], \"Measure\": [], \"Class\": []}" + "combined = {\n", + " \"Interpolation Steps\": [],\n", + " \"Similarity\": [],\n", + " \"Dataset\": [],\n", + " \"Measure\": [],\n", + " \"Class\": [],\n", + "}" ] }, { @@ -774,7 +780,15 @@ " plt.Line2D([0], [0], color=sns.color_palette()[0], lw=2),\n", " plt.Rectangle((0, 0), 1, 1, fc=\"#e1e9f3\", ec=\"#e1e9f3\", lw=0),\n", " plt.Line2D([0], [0], marker=\"o\", color=sns.color_palette()[0], lw=0),\n", - " plt.Line2D([0], [0], marker=\"|\", color=sns.color_palette()[0], lw=0, markersize=10, markeredgewidth=2),\n", + " plt.Line2D(\n", + " [0],\n", + " [0],\n", + " marker=\"|\",\n", + " color=sns.color_palette()[0],\n", + " lw=0,\n", + " markersize=10,\n", + " markeredgewidth=2,\n", + " ),\n", " ],\n", " labels=[\"Regression Line\", \"95% Confidence\", \"Similarity Mean\", \"Similarity Range\"],\n", " ncol=4,\n", diff --git a/imageGen.py b/imageGen.py index 8326bb9..595cf66 100644 --- a/imageGen.py +++ b/imageGen.py @@ -1,10 +1,11 @@ """ -Script to generate synthetic images for the specified dataset. +Generate synthetic images for dataset augmentation using Stable Diffusion. -The script generates synthetic images for each class pair of the confusing/top-misclassified classes in the dataset. -Each run of the script generates approximately 1:1 ratio of train-to-synthetic images. +This script generates synthetic images for confusing class pairs identified from +validation metrics. It uses image-to-image diffusion with prompt interpolation +to create training data that helps improve classifier performance. -While the script works in both Windows and Linux, it is recommended to run the script in Linux for better performance as torch.compile() is not supported in Windows. +Note: Linux is recommended for optimal performance (torch.compile support). """ from collections import Counter @@ -14,148 +15,162 @@ from tools import classes, data, synth +# Enable performance optimizations torch.backends.cudnn.benchmark = True torch.backends.cuda.matmul.allow_tf32 = True -### Load Data ### +# Configuration +DATASET_TYPE = "data/imagewoof" # Path to dataset -dataset_type = "data/imagewoof" # path to dataset +# Setup paths +cache_path = f"./{DATASET_TYPE}" +genInput_dir = f"{DATASET_TYPE}/train" +synth_path = f"{DATASET_TYPE}/synthetic" +metadata_path = f"{DATASET_TYPE}/metadata" +val_classifier_json = f"{DATASET_TYPE}/val.json" -cache_path = f"./{dataset_type}" -genInput_dir = f"{dataset_type}/train" -synth_path = f"{dataset_type}/synthetic" -metadata_path = f"{dataset_type}/metadata" -val_classifier_json = f"{dataset_type}/val.json" -# get the top 5 misclassified classes from dev from base classifier +# Load class pairs from validation metrics class_list = synth.get_class_list(val_classifier_json) -# combine all misclassified class pairs class_pairs_combo = synth.generateClassPairs(val_classifier_json) + +# Cache and load input dataset data.cacheGenData( genInput_dir, "imagenet_inputImg", save_path=cache_path, resize=(512, 512) ) genInput_dataset = data.loadData("imagenet_inputImg", cache_path=cache_path) - img_subsets = data.getSubsets(genInput_dataset, genInput_dir) -print(f"Running for {dataset_type}.") - -### Parameters for Image Generation ### - -# prompt adapted from Learning Transferable Visual Models From Natural Language Supervision -# prompt_format = "a photo of a " -prompt_format = "a photo of a , a type of dog" -# add ", a type of dog" for imagewoof/stanford dogs - -negative_prompt = "blurry image, disfigured, deformed, distorted, cartoon, drawings" # adapted from 10.1007/978-3-031-44237-7_14 - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model_id_or_path = "runwayml/stable-diffusion-v1-5" -# model_id_or_path = "./models/stable-diffusion-v1-5" # if available locally - -height, width = 512, 512 # set the optimal image size for the model - -# The guidance scale is set to its normal range (7 - 10). -guidance_scale = 8 - -# The number of inference steps was chosen empirically to generate an acceptable picture within an acceptable time. -num_inference_steps = 25 - -# Again, these values were chosen empirically. -num_interpolation_steps = 16 -sample_mid_interpolation = 16 -remove_n_middle = 0 +print(f"Running for {DATASET_TYPE}.") -### Interpolation Checks ### - -assert num_interpolation_steps % 2 == 0 -assert sample_mid_interpolation % 2 == 0 -assert remove_n_middle % 2 == 0 -assert num_interpolation_steps >= sample_mid_interpolation -assert num_interpolation_steps >= 2 and sample_mid_interpolation >= 2 -assert sample_mid_interpolation - remove_n_middle >= 2 - -# Setup pipeline - -pipe = synth.pipe_img(model_id_or_path, device=device) - -# To reproduce: 4796730343513556238 for woof, 1127962904372660145 for stanford dogs, 18316237598377439927 for imagenette +# Image Generation Parameters +# Prompt format adapted from "Learning Transferable Visual Models From Natural Language Supervision" +PROMPT_FORMAT = ( + "a photo of a , a type of dog" # Add ", a type of dog" for dog datasets +) +NEGATIVE_PROMPT = "blurry image, disfigured, deformed, distorted, cartoon, drawings" + +# Model configuration +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") +MODEL_ID = "runwayml/stable-diffusion-v1-5" # Or use "./models/stable-diffusion-v1-5" for local + +# Generation parameters +IMAGE_HEIGHT = 512 +IMAGE_WIDTH = 512 +GUIDANCE_SCALE = 8 # Guidance scale in normal range (7-10) +NUM_INFERENCE_STEPS = 25 # Empirically chosen for quality/speed balance + +# Interpolation parameters +NUM_INTERPOLATION_STEPS = 16 +SAMPLE_MID_INTERPOLATION = 16 +REMOVE_N_MIDDLE = 0 + +# Validate interpolation parameters +assert NUM_INTERPOLATION_STEPS % 2 == 0, "Interpolation steps must be even" +assert SAMPLE_MID_INTERPOLATION % 2 == 0, "Sample mid interpolation must be even" +assert REMOVE_N_MIDDLE % 2 == 0, "Remove n middle must be even" +assert ( + NUM_INTERPOLATION_STEPS >= SAMPLE_MID_INTERPOLATION +), "Interpolation steps must be >= sample mid" +assert ( + NUM_INTERPOLATION_STEPS >= 2 and SAMPLE_MID_INTERPOLATION >= 2 +), "Minimum 2 steps required" +assert SAMPLE_MID_INTERPOLATION - REMOVE_N_MIDDLE >= 2, "Must keep at least 2 samples" + +# Initialize pipeline +pipe = synth.pipe_img(MODEL_ID, device=DEVICE) + +# Set random seed for reproducibility +# Seeds: 4796730343513556238 (woof), 1127962904372660145 (stanford dogs), 18316237598377439927 (imagenette) seed = torch.Generator().seed() print(f"Seed: {seed}") -class_iterables = dict() -for c in class_list: - total_pair_count = Counter(c == x or c == y for x, y in class_pairs_combo)[True] - class_iterables[c] = synth.getPairIndices( - len(img_subsets[c]), total_pair_count, seed=seed +# Prepare class iterables for pair generation +class_iterables = {} +for class_id in class_list: + total_pair_count = Counter( + class_id == x or class_id == y for x, y in class_pairs_combo + )[True] + class_iterables[class_id] = synth.getPairIndices( + len(img_subsets[class_id]), total_pair_count, seed=seed ) -# Generate Images for each class pair +# Generate images for each class pair for combo_iter, class_pairs in enumerate(tqdm(class_pairs_combo)): + # Get class names from ImageNet mapping class_name_pairs = ( classes.IMAGENET2012_CLASSES[class_pairs[0]], classes.IMAGENET2012_CLASSES[class_pairs[1]], ) + + # Setup output directories synth.outputDirectory(class_pairs, synth_path, metadata_path) + + # Create prompts for the class pair prompts, negative_prompts = synth.createPrompts( class_name_pairs, - prompt_structure=prompt_format, - negative_prompt=negative_prompt, + prompt_structure=PROMPT_FORMAT, + negative_prompt=NEGATIVE_PROMPT, ) print(f"Generating images for {prompts[0]} and {prompts[1]}.") + + # Interpolate positive prompts interpolated_prompt_embeds, prompt_metadata = synth.interpolatePrompts( prompts, pipe, - num_interpolation_steps, - sample_mid_interpolation, - remove_n_middle=remove_n_middle, - device=device, + NUM_INTERPOLATION_STEPS, + SAMPLE_MID_INTERPOLATION, + remove_n_middle=REMOVE_N_MIDDLE, + device=DEVICE, ) + + # Interpolate negative prompts if provided if negative_prompts is not None: interpolated_negative_prompts_embeds, negative_prompt_metadata = ( synth.interpolatePrompts( negative_prompts, pipe, - num_interpolation_steps, - sample_mid_interpolation, - remove_n_middle=remove_n_middle, - device=device, + NUM_INTERPOLATION_STEPS, + SAMPLE_MID_INTERPOLATION, + remove_n_middle=REMOVE_N_MIDDLE, + device=DEVICE, ) ) else: - interpolated_negative_prompts_embeds, negative_prompt_metadata = [None] * len( - interpolated_prompt_embeds - ), None + interpolated_negative_prompts_embeds = [None] * len(interpolated_prompt_embeds) + negative_prompt_metadata = None + # Generate synthetic images ssim_scores = synth.generateImagesFromDataset( img_subsets, class_iterables, pipe, interpolated_prompt_embeds, interpolated_negative_prompts_embeds, - num_inference_steps, - guidance_scale, - height=height, - width=width, + NUM_INFERENCE_STEPS, + GUIDANCE_SCALE, + height=IMAGE_HEIGHT, + width=IMAGE_WIDTH, seed=seed, save_path=synth_path, class_pairs=class_pairs, save_image=True, image_type="jpg", interpolate_range="nearest", - device=device, + device=DEVICE, return_images=False, ) + # Save metadata for this class pair metadata = synth.getMetadata( class_pairs, synth_path, seed, - guidance_scale, - num_inference_steps, - num_interpolation_steps, - sample_mid_interpolation, - height, - width, + GUIDANCE_SCALE, + NUM_INFERENCE_STEPS, + NUM_INTERPOLATION_STEPS, + SAMPLE_MID_INTERPOLATION, + IMAGE_HEIGHT, + IMAGE_WIDTH, prompts, negative_prompts, pipe, diff --git a/imageGen_trace.py b/imageGen_trace.py index db8ba28..2676504 100644 --- a/imageGen_trace.py +++ b/imageGen_trace.py @@ -1,6 +1,9 @@ """ -For debugging and analysis purposes, this script generates the trace for the image generation process. -Refer to imageGen.py for the main image generation script. +Generate trace data for image generation process analysis and debugging. + +This script creates detailed trace information for the synthetic image generation +pipeline, useful for debugging and analyzing the interpolation process. +See imageGen.py for the main generation script. """ from collections import Counter @@ -11,6 +14,7 @@ from tools import classes, data, synth +# Enable performance optimizations torch.backends.cudnn.benchmark = True torch.backends.cuda.matmul.allow_tf32 = True diff --git a/tools/data.py b/tools/data.py index f961a5c..94812e1 100644 --- a/tools/data.py +++ b/tools/data.py @@ -1,25 +1,8 @@ """ -Helper functions for data processing and caching. - -Functions: - - get_module - - ClassificationPresetTrain - - ClassificationPresetEval - - cacheTrainData - - cacheValData - - loadData - - cacheGenData - - getSubsetIndicies - - getSubsetsFromIndicies - - getSubsets - - cacheAllInterpolation - - loadAllInterpolation - - getSubsetsByInterpolation - - getValSampler - - getSubsetLoader - - getSubsetLoaderbyInterpolation - - mix_Data - - synthetic_only_Data +Data processing and caching utilities for image classification. + +This module provides functions for loading, caching, and transforming datasets, +as well as creating data loaders and managing synthetic data organization. """ import os @@ -31,13 +14,11 @@ from torchvision import transforms from torchvision.transforms.functional import InterpolationMode -# Set default cache path - -cache_path = "./data" - -# Set default parameters for training and validation transformations +# Default configuration +DEFAULT_CACHE_PATH = "./data" -train_param = { +# Default training transformation parameters +DEFAULT_TRAIN_PARAMS = { "train_crop_size": 224, "interpolation": "bilinear", "auto_augment": None, @@ -48,7 +29,8 @@ "use_v2": False, } -val_param = { +# Default validation transformation parameters +DEFAULT_VAL_PARAMS = { "val_resize_size": 256, "val_crop_size": 224, "interpolation": "bilinear", @@ -59,29 +41,25 @@ def get_module(use_v2): """ - Returns the appropriate module from the torchvision.transforms package based on the value of use_v2. + Get appropriate torchvision transforms module. - Parameters: - use_v2 (bool): If True, returns the v2 module from torchvision.transforms.v2. If False, returns the module from torchvision.transforms. + Args: + use_v2 (bool): If True, use v2 transforms; otherwise use v1. Returns: - module: The selected module from torchvision.transforms or torchvision.transforms.v2. - """ + module: Selected transforms module. - ############################ - # Reference: - # TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. - ############################ - - # We need a protected import to avoid the V2 warning in case just V1 is used + Reference: + TorchVision maintainers and contributors (2016). + """ if use_v2: import torchvision.transforms.v2 return torchvision.transforms.v2 - else: - import torchvision.transforms - return torchvision.transforms + import torchvision.transforms + + return torchvision.transforms class ClassificationPresetTrain: @@ -248,32 +226,30 @@ def __call__(self, img): def cacheTrainData( train_dir, ds_name, - save_path=cache_path, - default_param=train_param, - stats={"mean": (0.485, 0.456, 0.406), "std": (0.229, 0.224, 0.225)}, + save_path=DEFAULT_CACHE_PATH, + default_param=DEFAULT_TRAIN_PARAMS, + stats=None, ): """ - Caches the ImageNet dataset for training. + Cache training dataset with transformations. Args: - train_dir (str): The directory path containing the training images. - ds_name (str): The name of the dataset. - save_path (str, optional): The directory path to save the cached dataset. Defaults to cache_path. - default_param (dict, optional): The default parameters for the dataset. Defaults to train_param. - stats (dict, optional): The statistics for normalization. Defaults to {"mean": (0.485, 0.456, 0.406), "std": (0.229, 0.224, 0.225)}. + train_dir (str): Directory containing training images. + ds_name (str): Dataset name for saving. + save_path (str): Directory to save cached dataset. + default_param (dict): Transformation parameters. + stats (dict): Normalization statistics (mean, std). Returns: None """ - ############################ - # Reference: - # TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. - ############################ + if stats is None: + stats = {"mean": (0.485, 0.456, 0.406), "std": (0.229, 0.224, 0.225)} - # Load the ImageNet dataset statistics mean = stats["mean"] std = stats["std"] + # Create dataset with training transformations dataset = torchvision.datasets.ImageFolder( train_dir, ClassificationPresetTrain( @@ -292,42 +268,39 @@ def cacheTrainData( ), ) - # Create directory if not exist + # Save cached dataset os.makedirs(save_path, exist_ok=True) - # Save the dataset to tensor file torch.save(dataset, os.path.join(save_path, f"{ds_name}.pt")) print(f"{ds_name}.pt dataset saved to {save_path}") - return None def cacheValData( valdir, ds_name, - save_path=cache_path, - default_param=val_param, - stats={"mean": (0.485, 0.456, 0.406), "std": (0.229, 0.224, 0.225)}, + save_path=DEFAULT_CACHE_PATH, + default_param=DEFAULT_VAL_PARAMS, + stats=None, ): """ - Caches the ImageNet validation dataset to a tensor file. + Cache validation dataset with transformations. Args: - valdir (str): Path to the validation dataset directory. - ds_name (str): Name of the dataset. - save_path (str, optional): Path to save the cached dataset. Defaults to cache_path. - default_param (dict, optional): Default parameters for dataset creation. Defaults to val_param. - stats (dict, optional): Dictionary containing the mean and standard deviation values for normalization. - Defaults to {"mean": (0.485, 0.456, 0.406), "std": (0.229, 0.224, 0.225)}. + valdir (str): Directory containing validation images. + ds_name (str): Dataset name for saving. + save_path (str): Directory to save cached dataset. + default_param (dict): Transformation parameters. + stats (dict): Normalization statistics (mean, std). Returns: None """ - ############################ - # Reference: - # TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. - ############################ + if stats is None: + stats = {"mean": (0.485, 0.456, 0.406), "std": (0.229, 0.224, 0.225)} + mean = stats["mean"] std = stats["std"] - # Load the ImageNet dataset + + # Create dataset with validation transformations dataset = torchvision.datasets.ImageFolder( valdir, ClassificationPresetEval( @@ -343,75 +316,72 @@ def cacheValData( ), ) - # Create directory if not exist + # Save cached dataset os.makedirs(save_path, exist_ok=True) - # Save the dataset to tensor file torch.save(dataset, os.path.join(save_path, f"{ds_name}.pt")) print(f"{ds_name}.pt dataset saved to {save_path}") - return None -def loadData(file_name, cache_path=cache_path): +def loadData(file_name, cache_path=DEFAULT_CACHE_PATH): """ - Load a dataset from a cache path if it exists, otherwise print a message indicating that the dataset was not found. + Load cached dataset from disk. Args: - file_name (str): The name of the dataset file. - cache_path (str, optional): The path to the cache directory. Defaults to cache_path. + file_name (str): Name of the dataset file (without .pt extension). + cache_path (str): Path to cache directory. Returns: - dataset: The loaded dataset if it exists, otherwise None. + dataset: Loaded dataset or None if not found. """ - if os.path.exists(cache_path): - dataset = torch.load( - os.path.join(cache_path, f"{file_name}.pt"), weights_only=False - ) + if not os.path.exists(cache_path): + print(f"Cache path {cache_path} does not exist") + return None + + dataset_path = os.path.join(cache_path, f"{file_name}.pt") + if os.path.exists(dataset_path): + dataset = torch.load(dataset_path, weights_only=False) print(f"{file_name}.pt dataset loaded from {cache_path}") - else: - print(f"{file_name}.pt not found in {cache_path}") - dataset = None - return dataset + return dataset + + print(f"{file_name}.pt not found in {cache_path}") + return None def cacheGenData( genInput_dir, ds_name, - save_path=cache_path, + save_path=DEFAULT_CACHE_PATH, resize=(256, 256), do_rescale=True, ): """ - Caches the ImageNet dataset generated from the given directory. + Cache generated dataset for synthetic image pipeline. Args: - genInput_dir (str): The directory containing the generated dataset. - ds_name (str): The name of the dataset. - save_path (str, optional): The path to save the cached dataset. Defaults to cache_path. - resize (tuple, optional): The size to resize the images to. Defaults to (256, 256). - do_rescale (bool, optional): Whether to rescale the images or not. Defaults to True. + genInput_dir (str): Directory containing generated images. + ds_name (str): Dataset name for saving. + save_path (str): Directory to save cached dataset. + resize (tuple): Target size for image resizing. + do_rescale (bool): Whether to rescale images. Returns: None """ - # Load the ImageNet dataset + # Define transformation based on rescale option if do_rescale: - dataset = torchvision.datasets.ImageFolder( - genInput_dir, - transform=transforms.Compose( - [transforms.ToTensor(), transforms.Resize(resize)] - ), + transform = transforms.Compose( + [transforms.ToTensor(), transforms.Resize(resize)] ) else: - dataset = torchvision.datasets.ImageFolder( - genInput_dir, transform=transforms.Compose([transforms.PILToTensor()]) - ) + transform = transforms.Compose([transforms.PILToTensor()]) - # Create directory if not exist + # Create dataset + dataset = torchvision.datasets.ImageFolder(genInput_dir, transform=transform) + + # Save cached dataset os.makedirs(save_path, exist_ok=True) - # Save the dataset to tensor file torch.save(dataset, os.path.join(save_path, f"{ds_name}.pt")) print(f"{ds_name}.pt dataset saved to {save_path}") - return None def getSubsetIndicies(dataset, data_dir): diff --git a/tools/synth.py b/tools/synth.py index e3b31a7..0721bc1 100644 --- a/tools/synth.py +++ b/tools/synth.py @@ -40,77 +40,75 @@ def get_top_misclassified(val_classifier_json): """ - Retrieves the top misclassified classes from a validation classifier JSON file. + Retrieve top misclassified classes from validation metrics. Args: - val_classifier_json (str): The path to the validation classifier JSON file. + val_classifier_json (str): Path to validation classifier JSON file. Returns: - dict: A dictionary containing the top misclassified classes, where the keys are the class names - and the values are the number of misclassifications. + dict: Class names mapped to their misclassification counts. """ with open(val_classifier_json) as f: val_output = json.load(f) + val_metrics_df = pd.DataFrame.from_dict( val_output["val_metrics_details"], orient="index" ) - class_dict = dict() - for k, v in val_metrics_df["top_n_classes"].items(): - class_dict[k] = v - return class_dict + + return dict(val_metrics_df["top_n_classes"].items()) def get_class_list(val_classifier_json): """ - Retrieves the list of classes from the given validation classifier JSON file. + Get sorted list of classes from validation metrics. Args: - val_classifier_json (str): The path to the validation classifier JSON file. + val_classifier_json (str): Path to validation classifier JSON file. Returns: - list: A sorted list of class names extracted from the JSON file. + list: Sorted list of class names. """ with open(val_classifier_json, "r") as f: data = json.load(f) - return sorted(list(data["val_metrics_details"].keys())) + return sorted(data["val_metrics_details"].keys()) def generateClassPairs(val_classifier_json): """ - Generate pairs of misclassified classes from the given validation classifier JSON. + Generate pairs of misclassified classes from validation metrics. Args: - val_classifier_json (str): The path to the validation classifier JSON file. + val_classifier_json (str): Path to validation classifier JSON file. Returns: - list: A sorted list of pairs of misclassified classes. + list: Sorted list of misclassified class pairs. """ pairs = set() misclassified_classes = get_top_misclassified(val_classifier_json) - for key, value in misclassified_classes.items(): - for v in value: - pairs.add(tuple(sorted([key, v]))) - return sorted(list(pairs)) + + for key, values in misclassified_classes.items(): + for value in values: + pairs.add(tuple(sorted([key, value]))) + + return sorted(pairs) def outputDirectory(class_pairs, synth_path, metadata_path): """ - Creates the output directory structure for the synthesized data. + Create output directory structure for synthesized data. Args: - class_pairs (list): A list of class pairs. - synth_path (str): The path to the directory where the synthesized data will be stored. - metadata_path (str): The path to the directory where the metadata will be stored. - - Returns: - None + class_pairs (list): List of class pairs. + synth_path (str): Path for synthesized data storage. + metadata_path (str): Path for metadata storage. """ - for id in class_pairs: - class_folder = f"{synth_path}/{id}" - if not (os.path.exists(class_folder)): - os.makedirs(class_folder) - if not (os.path.exists(metadata_path)): - os.makedirs(metadata_path) + # Create directories for each class + for class_id in class_pairs: + class_folder = f"{synth_path}/{class_id}" + os.makedirs(class_folder, exist_ok=True) + + # Create metadata directory + os.makedirs(metadata_path, exist_ok=True) print("Info: Output directory ready.") @@ -221,6 +219,96 @@ def createPrompts( return prompts, negative_prompts +def _slerp(v0, v1, num, t0=0, t1=1): + """ + Perform spherical linear interpolation between two vectors. + + Args: + v0 (torch.Tensor): Starting vector. + v1 (torch.Tensor): Ending vector. + num (int): Number of interpolation points. + t0 (float): Starting time (default: 0). + t1 (float): Ending time (default: 1). + + Returns: + torch.Tensor: Interpolated vectors. + + Reference: + Karpathy, A. (2022) 'hacky stablediffusion code for generating videos', Gist. + """ + v0 = v0.detach().cpu().numpy() + v1 = v1.detach().cpu().numpy() + + def interpolation(t, v0, v1, DOT_THRESHOLD=0.9995): + """Helper function to spherically interpolate two arrays.""" + dot = np.sum(v0 * v1 / (np.linalg.norm(v0) * np.linalg.norm(v1))) + if np.abs(dot) > DOT_THRESHOLD: + v2 = (1 - t) * v0 + t * v1 + else: + theta_0 = np.arccos(dot) + sin_theta_0 = np.sin(theta_0) + theta_t = theta_0 * t + sin_theta_t = np.sin(theta_t) + s0 = np.sin(theta_0 - theta_t) / sin_theta_0 + s1 = sin_theta_t / sin_theta_0 + v2 = s0 * v0 + s1 * v1 + return v2 + + t = np.linspace(t0, t1, num) + v3 = torch.tensor(np.array([interpolation(t[i], v0, v1) for i in range(num)])) + return v3 + + +def _get_middle_elements(lst, n): + """ + Get middle n elements from a list. + + Args: + lst (list): Input list. + n (int): Number of middle elements to extract. + + Returns: + tuple: (middle elements sublist, range of indices). + """ + if n % 2 == 0: # Even number of elements + middle_index = len(lst) // 2 - 1 + start = middle_index - n // 2 + 1 + end = middle_index + n // 2 + 1 + else: # Odd number of elements + middle_index = len(lst) // 2 + start = middle_index - n // 2 + end = middle_index + n // 2 + 1 + + return lst[start:end], range(start, end) + + +def _remove_middle(data, n): + """ + Remove middle n elements from a list. + + Args: + data (list): Input list. + n (int): Number of middle elements to remove. + + Returns: + list: List with middle n elements removed. + + Raises: + ValueError: If n is invalid. + """ + if n < 0 or n > len(data): + raise ValueError("Invalid n: must be non-negative and less than list length") + + middle = len(data) // 2 + + if n == 1: + return data[:middle] + data[middle + 1 :] + elif n % 2 == 0: + return data[: middle - n // 2] + data[middle + n // 2 :] + else: + return data[: middle - n // 2] + data[middle + n // 2 + 1 :] + + def interpolatePrompts( prompts, pipeline, @@ -230,139 +318,33 @@ def interpolatePrompts( device="cuda", ): """ - Interpolates prompts by generating intermediate embeddings between pairs of prompts. + Interpolate prompts by generating intermediate embeddings between pairs. Args: - prompts (List[str]): A list of prompts to be interpolated. - pipeline: The pipeline object containing the tokenizer and text encoder. - num_interpolation_steps (int): The number of interpolation steps between each pair of prompts. - sample_mid_interpolation (int): The number of intermediate embeddings to sample from the middle of the interpolated prompts. - remove_n_middle (int, optional): The number of middle embeddings to remove from the interpolated prompts. Defaults to 0. - device (str, optional): The device to run the interpolation on. Defaults to "cuda". + prompts (List[str]): List of prompts to interpolate. + pipeline: Pipeline with tokenizer and text encoder. + num_interpolation_steps (int): Number of interpolation steps. + sample_mid_interpolation (int): Number of mid samples to keep. + remove_n_middle (int): Number of middle embeddings to remove. + device (str): Device for computation (default: "cuda"). Returns: - interpolated_prompt_embeds (torch.Tensor): The interpolated prompt embeddings. - prompt_metadata (dict): Metadata about the interpolation process, including similarity scores and nearest class information. + tuple: (interpolated_prompt_embeds, prompt_metadata) - e.g. if num_interpolation_steps = 10, sample_mid_interpolation = 6, remove_n_middle = 2 - Interpolated: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - Sampled: [2, 3, 4, 5, 6, 7] - Removed: x x - Returns: [2, 3, 6, 7] - """ + Example: + num_interpolation_steps = 10, sample_mid_interpolation = 6, remove_n_middle = 2 + Interpolated: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + Sampled: [2, 3, 4, 5, 6, 7] + Removed: x x + Returns: [2, 3, 6, 7] - ############################### - # Reference: - # Akimov, R. (2024) Images Interpolation with Stable Diffusion - Hugging Face Open-Source AI Cookbook. Available at: https://huggingface.co/learn/cookbook/en/stable_diffusion_interpolation (Accessed: 4 June 2024). - ############################### - - def slerp(v0, v1, num, t0=0, t1=1): - """ - Performs spherical linear interpolation between two vectors. - - Args: - v0 (torch.Tensor): The starting vector. - v1 (torch.Tensor): The ending vector. - num (int): The number of interpolation points. - t0 (float, optional): The starting time. Defaults to 0. - t1 (float, optional): The ending time. Defaults to 1. - - Returns: - torch.Tensor: The interpolated vectors. - - """ - ############################### - # Reference: - # Karpathy, A. (2022) hacky stablediffusion code for generating videos, Gist. Available at: https://gist.github.com/karpathy/00103b0037c5aaea32fe1da1af553355 (Accessed: 4 June 2024). - ############################### - v0 = v0.detach().cpu().numpy() - v1 = v1.detach().cpu().numpy() - - def interpolation(t, v0, v1, DOT_THRESHOLD=0.9995): - """helper function to spherically interpolate two arrays v1 v2""" - dot = np.sum(v0 * v1 / (np.linalg.norm(v0) * np.linalg.norm(v1))) - if np.abs(dot) > DOT_THRESHOLD: - v2 = (1 - t) * v0 + t * v1 - else: - theta_0 = np.arccos(dot) - sin_theta_0 = np.sin(theta_0) - theta_t = theta_0 * t - sin_theta_t = np.sin(theta_t) - s0 = np.sin(theta_0 - theta_t) / sin_theta_0 - s1 = sin_theta_t / sin_theta_0 - v2 = s0 * v0 + s1 * v1 - return v2 - - t = np.linspace(t0, t1, num) - - v3 = torch.tensor(np.array([interpolation(t[i], v0, v1) for i in range(num)])) - - return v3 - - def get_middle_elements(lst, n): - """ - Returns a tuple containing a sublist of the middle elements of the given list `lst` and a range of indices of those elements. - - Args: - lst (list): The list from which to extract the middle elements. - n (int): The number of middle elements to extract. - - Returns: - tuple: A tuple containing the sublist of middle elements and a range of indices. - - Raises: - None - - Examples: - lst = [1, 2, 3, 4, 5] - get_middle_elements(lst, 3) - ([2, 3, 4], range(2, 5)) - """ - if n % 2 == 0: # Even number of elements - middle_index = len(lst) // 2 - 1 - start = middle_index - n // 2 + 1 - end = middle_index + n // 2 + 1 - return lst[start:end], range(start, end) - else: # Odd number of elements - middle_index = len(lst) // 2 - start = middle_index - n // 2 - end = middle_index + n // 2 + 1 - return lst[start:end], range(start, end) - - def remove_middle(data, n): - """ - Remove the middle n elements from a list. - - Args: - data (list): The input list. - n (int): The number of elements to remove from the middle of the list. - - Returns: - list: The modified list with the middle n elements removed. - - Raises: - ValueError: If n is negative or greater than the length of the list. - - """ - if n < 0 or n > len(data): - raise ValueError( - "Invalid value for n. It should be non-negative and less than half the list length" - ) - - # Find the middle index - middle = len(data) // 2 - - # Create slices to exclude the middle n elements - if n == 1: - return data[:middle] + data[middle + 1 :] - elif n % 2 == 0: - return data[: middle - n // 2] + data[middle + n // 2 :] - else: - return data[: middle - n // 2] + data[middle + n // 2 + 1 :] + Reference: + Akimov, R. (2024) 'Images Interpolation with Stable Diffusion'. + """ batch_size = len(prompts) - # Tokenizing and encoding prompts into embeddings. + # Tokenize and encode prompts into embeddings prompts_tokens = pipeline.tokenizer( prompts, padding="max_length", @@ -372,27 +354,33 @@ def remove_middle(data, n): ) prompts_embeds = pipeline.text_encoder(prompts_tokens.input_ids.to(device))[0] - # Interpolating between embeddings pairs for the given number of interpolation steps. + # Interpolate between embedding pairs interpolated_prompt_embeds = [] - for i in range(batch_size - 1): interpolated_prompt_embeds.append( - slerp(prompts_embeds[i], prompts_embeds[i + 1], num_interpolation_steps) + _slerp(prompts_embeds[i], prompts_embeds[i + 1], num_interpolation_steps) ) + # Keep full interpolations for metadata calculation full_interpolated_prompt_embeds = interpolated_prompt_embeds[:] - interpolated_prompt_embeds[0], sample_range = get_middle_elements( + + # Sample middle elements + interpolated_prompt_embeds[0], sample_range = _get_middle_elements( interpolated_prompt_embeds[0], sample_mid_interpolation ) + # Remove center elements if requested if remove_n_middle > 0: - interpolated_prompt_embeds[0] = remove_middle( + interpolated_prompt_embeds[0] = _remove_middle( interpolated_prompt_embeds[0], remove_n_middle ) - prompt_metadata = dict() + # Calculate similarity metadata + prompt_metadata = {} similarity = nn.CosineSimilarity(dim=-1, eps=1e-6) + for i in range(num_interpolation_steps): + # Calculate similarities to both class embeddings class1_sim = ( similarity( full_interpolated_prompt_embeds[0][0], @@ -409,6 +397,8 @@ def remove_middle(data, n): .mean() .item() ) + + # Calculate relative distance relative_distance = class1_sim / (class1_sim + class2_sim) prompt_metadata[i] = { @@ -422,6 +412,7 @@ def remove_middle(data, n): "nearest_class": int(relative_distance < 0.5), } + # Concatenate and return interpolated_prompt_embeds = torch.cat(interpolated_prompt_embeds, dim=0).to(device) return interpolated_prompt_embeds, prompt_metadata diff --git a/tools/transforms.py b/tools/transforms.py index 7dd9d28..ffd7950 100644 --- a/tools/transforms.py +++ b/tools/transforms.py @@ -1,19 +1,12 @@ """ -Helper script to implement MixUp and CutMix augmentations. -The script is based on the implementation in the TorchVision library. - -Functions: - - get_module - - get_mixup_cutmix - - RandomMixUp - - RandomCutMix - - getCollateFn -""" +MixUp and CutMix augmentation implementations. + +This module provides data augmentation techniques for improving model generalization, +based on the TorchVision library implementations. -############################ -# Reference: -# TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. -############################ +Reference: +TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. +""" import math from typing import Tuple @@ -26,68 +19,66 @@ def get_module(use_v2): """ - Returns the appropriate module for torchvision transforms based on the value of `use_v2`. + Get appropriate torchvision transforms module. - Parameters: - - use_v2 (bool): If True, returns the v2 module of torchvision.transforms. If False, returns the default module. + Args: + use_v2 (bool): If True, use v2 transforms; otherwise use v1. Returns: - - module: The module for torchvision transforms. - - References: - - TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. + module: Selected transforms module. """ - ############################ - # Reference: - # TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. - ############################ - # We need a protected import to avoid the V2 warning in case just V1 is used if use_v2: import torchvision.transforms.v2 return torchvision.transforms.v2 - else: - import torchvision.transforms - return torchvision.transforms + import torchvision.transforms + + return torchvision.transforms def get_mixup_cutmix(*, mixup_alpha, cutmix_alpha, num_classes, use_v2): """ - Returns a random transformation from MixUp and CutMix based on the provided parameters. + Create MixUp and/or CutMix augmentation transforms. Args: - mixup_alpha (float): The alpha value for MixUp transformation. - cutmix_alpha (float): The alpha value for CutMix transformation. - num_classes (int): The number of classes in the dataset. - use_v2 (bool): A flag indicating whether to use the v2 version of the transforms module. + mixup_alpha (float): Alpha parameter for MixUp. + cutmix_alpha (float): Alpha parameter for CutMix. + num_classes (int): Number of classes in dataset. + use_v2 (bool): Whether to use v2 transforms. Returns: - A random transformation from MixUp and CutMix, or None if both mixup_alpha and cutmix_alpha are 0. + RandomChoice transform or None if both alphas are 0. """ - ############################ - # Reference: - # TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. - ############################ transforms_module = get_module(use_v2) + transforms_list = [] - mixup_cutmix = [] + # Add MixUp if alpha > 0 if mixup_alpha > 0: - mixup_cutmix.append( - transforms_module.MixUp(alpha=mixup_alpha, num_classes=num_classes) - if use_v2 - else RandomMixUp(num_classes=num_classes, p=1.0, alpha=mixup_alpha) - ) + if use_v2: + transforms_list.append( + transforms_module.MixUp(alpha=mixup_alpha, num_classes=num_classes) + ) + else: + transforms_list.append( + RandomMixUp(num_classes=num_classes, p=1.0, alpha=mixup_alpha) + ) + + # Add CutMix if alpha > 0 (note: using cutmix_alpha, not mixup_alpha) if cutmix_alpha > 0: - mixup_cutmix.append( - transforms_module.CutMix(alpha=mixup_alpha, num_classes=num_classes) - if use_v2 - else RandomCutMix(num_classes=num_classes, p=1.0, alpha=mixup_alpha) - ) - if not mixup_cutmix: + if use_v2: + transforms_list.append( + transforms_module.CutMix(alpha=cutmix_alpha, num_classes=num_classes) + ) + else: + transforms_list.append( + RandomCutMix(num_classes=num_classes, p=1.0, alpha=cutmix_alpha) + ) + + if not transforms_list: return None - return transforms_module.RandomChoice(mixup_cutmix) + return transforms_module.RandomChoice(transforms_list) class RandomMixUp(torch.nn.Module): @@ -299,33 +290,24 @@ def __repr__(self) -> str: def getCollateFn(num_classes, setup: dict, get_mixup_cutmix=get_mixup_cutmix): """ - Returns a collate function for data batching that applies mixup or cutmix transformations. + Create collate function with optional MixUp/CutMix transforms. Args: - num_classes (int): The number of classes in the dataset. - setup (dict): A dictionary containing setup parameters for mixup and cutmix. - get_mixup_cutmix (function, optional): A function that returns mixup or cutmix transformation. - Defaults to get_mixup_cutmix. + num_classes (int): Number of classes in dataset. + setup (dict): Configuration with mixup_alpha, cutmix_alpha, and use_v2. + get_mixup_cutmix (function): Function to create transforms. Returns: - collate_fn (function): The collate function for data batching. + function: Collate function for DataLoader. """ - ############################ - # Reference: - # TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. - ############################ mixup_cutmix = get_mixup_cutmix( mixup_alpha=setup["mixup_alpha"], cutmix_alpha=setup["cutmix_alpha"], num_classes=num_classes, use_v2=setup["use_v2"], ) - if mixup_cutmix is not None: - - def collate_fn(batch): - return mixup_cutmix(*default_collate(batch)) - else: - collate_fn = default_collate + if mixup_cutmix is not None: + return lambda batch: mixup_cutmix(*default_collate(batch)) - return collate_fn + return default_collate diff --git a/tools/utils.py b/tools/utils.py index 043a884..9311a18 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -22,12 +22,11 @@ import torch import torch.distributed as dist import torch.utils.data +import wandb from torch import Tensor, nn from torchvision.transforms import functional as F from tqdm import tqdm -import wandb - def setup_for_distributed(is_master): """ diff --git a/trainClassifier.py b/trainClassifier.py index bc62036..a12dcbb 100644 --- a/trainClassifier.py +++ b/trainClassifier.py @@ -1,22 +1,14 @@ """ -This script trains a classifier on a specified dataset with PyTorch. +Train a classifier on specified dataset with PyTorch. -Most of the code is adapted from the TorchVision reference scripts: -https://github.com/pytorch/vision/tree/main/references/classification - -The script was modified to include additional features such as: -- ScheduleFree Optimizer (https://arxiv.org/abs/2405.15682) -- Experiment Tracking with Weights and Biases (https://wandb.ai/) -- Learning Rate Milestones (MultiStepLR) for LR Scheduler (https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.MultiStepLR.html) - -During the development of the script, a contribution was made to the TorchVision repository: -- Title: Fix CutMix and MixUp arguments in transforms.py -- Description: Fixed the CutMix and MixUp arguments in the transforms.py reference script to train a classifier. -- Commit: https://github.com/pytorch/vision/commit/3b5e6fc42403cc20f8cfbafb52a75b30bd00c226 -- Pull Request: https://github.com/pytorch/vision/pull/8287 +Adapted from TorchVision reference scripts with additional features: +- ScheduleFree Optimizer for improved convergence +- Weights & Biases integration for experiment tracking +- MultiStepLR scheduler with configurable milestones Reference: -TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. GitHub. Available at: https://github.com/pytorch/vision. +TorchVision maintainers and contributors (2016) 'TorchVision: PyTorch's computer vision library'. +GitHub. Available at: https://github.com/pytorch/vision. """ import torch @@ -25,18 +17,33 @@ from tools import data, utils from tools import transforms as trnf -if __name__ == "__main__": - torch.backends.cuda.matmul.allow_tf32 = True # enable TensorFloat32 for matmul + +def main(): + """Main training function.""" + # Enable performance optimizations + torch.backends.cuda.matmul.allow_tf32 = True + + # Training configuration setup = { - "entity": "zhulinchng", # the wandb username - "project": "demo", # the wandb project name - "data_path": "./data/imagenette", # ./data/imagenette for imagenette; ./data/woof for imagewoof; ./stanford-dogs for stanford-dogs + # Experiment tracking + "entity": "zhulinchng", # Wandb username + "project": "demo", # Wandb project name + "wandb_id": "", + # Dataset configuration + "data_path": "./data/imagenette", # imagenette, woof, or stanford-dogs + "cache_dataset": False, + "workers": 12, # Number of data loading workers + # Model configuration + "model": "resnet18", # resnet18 or mobilenet_v2 + "weights": None, # Path to pretrained weights or None "device": "cuda", - "model": "resnet18", # resnet18, mobilenet_v2 - "weights": None, # path to model weights, None to train from scratch - "batch_size": 256, # batch size 256 for resnet18, 128 for mobilenet_v2 to fit 12GB of GPU memory optimally + # Training parameters "epochs": 200, - "workers": 12, # number of data loading workers, CPU cores + "batch_size": 256, # 256 for resnet18, 128 for mobilenet_v2 (12GB GPU) + "start_epoch": 0, + "resume": "", # Path to checkpoint to resume from + "test_only": False, + # Optimizer configuration "opt": "sgd", "lr": 0.1, "momentum": 0.9, @@ -44,75 +51,75 @@ "norm_weight_decay": None, "bias_weight_decay": None, "transformer_embedding_decay": None, - "label_smoothing": 0.0, - "mixup_alpha": 0.0, - "cutmix_alpha": 0.0, + # Learning rate schedule "lr_scheduler": "multisteplr", + "lr_milestones": [100, 150, 201, 251], + "lr_gamma": 0.1, + "lr_min": 0.0, + "lr_step_size": 50, "lr_warmup_epochs": 0, "lr_warmup_method": "constant", "lr_warmup_decay": 0.01, - "lr_step_size": 50, - "lr_gamma": 0.1, - "lr_min": 0.0, - "lr_milestones": [ - 100, - 150, - 201, - 251, - ], # if using multisteplr, else milestones are not used - "print_freq": 20, - "output_dir": "./output/imagenette", # output directory for model checkpoints - "resume": "", # path to model to resume training, empty to train from scratch - "start_epoch": 0, - "cache_dataset": False, - "sync_bn": False, - "test_only": False, # set to True to only test the model + # Data augmentation "auto_augment": None, "ra_magnitude": 9, "augmix_severity": 3, "random_erase": 0.0, - "amp": False, - "world_size": 1, - "dist_url": "env://", - "model_ema": False, - "model_ema_steps": 32, - "model_ema_decay": 0.99998, - "use_deterministic_algorithms": False, + "mixup_alpha": 0.0, + "cutmix_alpha": 0.0, + "label_smoothing": 0.0, + # Training settings "interpolation": "bilinear", "val_resize_size": 256, "val_crop_size": 224, "train_crop_size": 224, "clip_grad_norm": None, + "print_freq": 20, + "output_dir": "./output/imagenette", + # Advanced features + "amp": False, # Automatic mixed precision + "model_ema": False, # Exponential moving average + "model_ema_steps": 32, + "model_ema_decay": 0.99998, + "sync_bn": False, + "use_deterministic_algorithms": False, + # Distributed training + "world_size": 1, + "dist_url": "env://", + # Miscellaneous "ra_sampler": False, "ra_reps": 3, "backend": "PIL", "use_v2": False, - "wandb_id": "", } - ### Cache Dataset for Faster Loading + # Prepare dataset paths train_dir = f'{setup["data_path"].replace("./","")}/train' - valdir = f'{setup["data_path"].replace("./","")}/val' + val_dir = f'{setup["data_path"].replace("./","")}/val' + # Cache datasets for faster loading data.cacheTrainData(train_dir, "train_cache", save_path=setup["data_path"]) - data.cacheValData(valdir, "val_cache", save_path=setup["data_path"]) - - ### Training + data.cacheValData(val_dir, "val_cache", save_path=setup["data_path"]) + # Initialize distributed training setup = utils.init_distributed_mode(setup) device = torch.device(setup["device"]) + + # Configure deterministic behavior if setup["use_deterministic_algorithms"]: torch.backends.cudnn.benchmark = False torch.use_deterministic_algorithms(True) else: torch.backends.cudnn.benchmark = True - # Load training set + + # Load and validate training dataset train_dataset = data.loadData("train_cache", cache_path=setup["data_path"]) num_classes = len(train_dataset.classes) + assert ( + num_classes == 10 + ), f"Expected 10 classes, got {num_classes}" # 10 for imagenette/woof, 120 for stanford-dogs - # Check if the number of classes is correct - assert num_classes == 10 # 10 for imagenette and imagewoof , 120 for stanford-dogs - + # Create training data loader train_sampler = utils.getTrainSampler(train_dataset, setup) collate_fn = trnf.getCollateFn(num_classes=num_classes, setup=setup) data_loader = torch.utils.data.DataLoader( @@ -123,12 +130,14 @@ pin_memory=True, collate_fn=collate_fn, ) - val_dataset = data.loadData("val_cache", cache_path=setup["data_path"]) - # Check if the number of classes is correct - # 10 for imagenette and imagewoof , 120 for stanford-dogs - assert len(val_dataset.classes) == 10 + # Load and validate validation dataset + val_dataset = data.loadData("val_cache", cache_path=setup["data_path"]) + assert ( + len(val_dataset.classes) == 10 + ), f"Expected 10 classes, got {len(val_dataset.classes)}" + # Create validation data loader val_sampler = utils.getValSampler(val_dataset, setup) data_loader_test = torch.utils.data.DataLoader( val_dataset, @@ -139,14 +148,20 @@ shuffle=False, ) + # Initialize model model = torchvision.models.get_model( setup["model"], weights=setup["weights"], num_classes=num_classes ) model.to(device) + + # Apply synchronized batch normalization if distributed if setup["distributed"] and setup["sync_bn"]: model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) - # Train Setup + # Setup training components + training_components = utils.trainSetup( + model=model, setup=setup, val_dataloader=data_loader_test, device=device + ) ( model, criterion, @@ -157,11 +172,9 @@ model_without_ddp, lr_scheduler, run, - ) = utils.trainSetup( - model=model, setup=setup, val_dataloader=data_loader_test, device=device - ) + ) = training_components - # Train + # Train the model model, run = utils.train( model, model_without_ddp, @@ -178,10 +191,11 @@ run, ) - # Evaluate + # Final evaluation torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True model.eval() + metrics = utils.evaluate( model, criterion, @@ -190,8 +204,12 @@ print_freq=setup["print_freq"], ) - # Log metrics to wandb + # Log final metrics to wandb run.summary["acc1"] = metrics["acc1"] run.summary["acc5"] = metrics["acc5"] run.summary["loss"] = metrics["loss"] run.finish() + + +if __name__ == "__main__": + main() diff --git a/validateClassifier.ipynb b/validateClassifier.ipynb index ba53bf0..2c7f56b 100644 --- a/validateClassifier.ipynb +++ b/validateClassifier.ipynb @@ -120,11 +120,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "device = torch.device(setup['device'])" + "device = torch.device(setup[\"device\"])" ] }, { @@ -302,11 +302,11 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "criterion = nn.CrossEntropyLoss(label_smoothing=setup['label_smoothing'])" + "criterion = nn.CrossEntropyLoss(label_smoothing=setup[\"label_smoothing\"])" ] }, { From d7bc93c09e749bce1871bc093e291ea12cf953f2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 14 Jan 2026 01:16:06 +0000 Subject: [PATCH 2/6] Initial plan From 19cda36ccdd0608a4db6eeb2183471be02f016bb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 14 Jan 2026 01:20:04 +0000 Subject: [PATCH 3/6] Remove hardcoded class assertions to support multiple datasets Co-authored-by: zhulinchng <24189730+zhulinchng@users.noreply.github.com> --- trainClassifier.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/trainClassifier.py b/trainClassifier.py index a12dcbb..0bba0ce 100644 --- a/trainClassifier.py +++ b/trainClassifier.py @@ -112,12 +112,9 @@ def main(): else: torch.backends.cudnn.benchmark = True - # Load and validate training dataset + # Load training dataset train_dataset = data.loadData("train_cache", cache_path=setup["data_path"]) num_classes = len(train_dataset.classes) - assert ( - num_classes == 10 - ), f"Expected 10 classes, got {num_classes}" # 10 for imagenette/woof, 120 for stanford-dogs # Create training data loader train_sampler = utils.getTrainSampler(train_dataset, setup) @@ -131,11 +128,8 @@ def main(): collate_fn=collate_fn, ) - # Load and validate validation dataset + # Load validation dataset val_dataset = data.loadData("val_cache", cache_path=setup["data_path"]) - assert ( - len(val_dataset.classes) == 10 - ), f"Expected 10 classes, got {len(val_dataset.classes)}" # Create validation data loader val_sampler = utils.getValSampler(val_dataset, setup) From 2c4f215bfb6adc97f00c38b8fc5929d87c7ee057 Mon Sep 17 00:00:00 2001 From: Zhu Lin Date: Wed, 14 Jan 2026 09:54:37 +0800 Subject: [PATCH 4/6] Update trainClassifier.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- trainClassifier.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/trainClassifier.py b/trainClassifier.py index 0bba0ce..acedd71 100644 --- a/trainClassifier.py +++ b/trainClassifier.py @@ -130,6 +130,10 @@ def main(): # Load validation dataset val_dataset = data.loadData("val_cache", cache_path=setup["data_path"]) + assert len(val_dataset.classes) == num_classes, ( + f"Training dataset has {num_classes} classes but validation dataset has " + f"{len(val_dataset.classes)} classes" + ) # Create validation data loader val_sampler = utils.getValSampler(val_dataset, setup) From b45c4e105af2bd506c397f378cf5507f861843e1 Mon Sep 17 00:00:00 2001 From: Zhu Lin Date: Wed, 14 Jan 2026 09:55:13 +0800 Subject: [PATCH 5/6] Update tools/synth.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tools/synth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/synth.py b/tools/synth.py index 0721bc1..930bf29 100644 --- a/tools/synth.py +++ b/tools/synth.py @@ -297,7 +297,7 @@ def _remove_middle(data, n): ValueError: If n is invalid. """ if n < 0 or n > len(data): - raise ValueError("Invalid n: must be non-negative and less than list length") + raise ValueError("Invalid n: must be non-negative and not exceed list length") middle = len(data) // 2 From ff4bfc5263fc63ebba90fb883806a0e38c8989f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 14 Jan 2026 01:55:47 +0000 Subject: [PATCH 6/6] Initial plan