diff --git a/README.md b/README.md index ddb1615..f80d73d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -

LLaMEA-SAGE: Guiding Automated Algorithm Design with +

LLaMEA-CFG: Guiding Automated Algorithm Design with Structural Feedback from Explainable AI

## Introduction -This is the reproducability repository for the paper "LLaMEA-SAGE: Guiding Automated Algorithm Design with Structural Feedback from Explainable AI". +This is the reproducability repository for the paper "LLaMEA-CFG: Guiding Automated Algorithm Design with Structural Feedback from Explainable AI". @@ -44,14 +44,14 @@ make sure you have `uv` installed. ```bash uv run python run_archive_guided.py ``` - This will run Experiment 1 (LLaMEA and LLaMEA-SAGE on SBOX-COST). + This will run Experiment 1 (LLaMEA and LLaMEA-CFG on SBOX-COST). ```bash uv run python run_archive_guided-2.py uv run python run_archive_guided-2-1.py uv run python run_archive_guided-2-2.py ``` - This will run Experiment 2 (LLaMEA and LLaMEA-SAGE on MA-BBOB), baselines and the abblation using Gemini respectively. + This will run Experiment 2 (LLaMEA and LLaMEA-CFG on MA-BBOB), baselines and the abblation using Gemini respectively. 3. Produce paper artifacts. diff --git a/examples/visualize_kerneltuner.ipynb b/examples/visualize_kerneltuner.ipynb index aca4c06..d53b951 100644 --- a/examples/visualize_kerneltuner.ipynb +++ b/examples/visualize_kerneltuner.ipynb @@ -122,7 +122,7 @@ "\n", "token_df = token_df1\n", "\n", - "#token_df['method_name'].replace({'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-SAGE', 'LHNS:vns': \"LHNS\"}, inplace=True)\n", + "#token_df['method_name'].replace({'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-CFG', 'LHNS:vns': \"LHNS\"}, inplace=True)\n", "\n", "summary = (\n", " token_df.groupby([\"problem_name\", \"method_name\"])[\"tokens\"].mean().reset_index()\n", @@ -190,7 +190,7 @@ "\n", "token_df = token_df1\n", "\n", - "# token_df['method_name'].replace({'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-SAGE', 'LHNS:vns': \"LHNS\"}, inplace=True)\n", + "# token_df['method_name'].replace({'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-CFG', 'LHNS:vns': \"LHNS\"}, inplace=True)\n", "\n", "summary = (\n", " token_df.groupby([\"problem_name\", \"method_name\"])[\"tokens\"]\n", diff --git a/final_validation.ipynb b/final_validation.ipynb index 6c42468..e9e62ba 100644 --- a/final_validation.ipynb +++ b/final_validation.ipynb @@ -425,7 +425,7 @@ " if label == \"ES\":\n", " handle.set_label(\"LLaMEA\")\n", " if label == \"ES-guided\":\n", - " handle.set_label(\"LLaMEA-SAGE\")\n", + " handle.set_label(\"LLaMEA-CFG\")\n", " if label == \"LHNS:vns\":\n", " handle.set_label(\"LHNS\")\n", "\n", @@ -502,7 +502,7 @@ " if label == \"ES\":\n", " handle.set_label(\"LLaMEA\")\n", " if label == \"ES-guided\":\n", - " handle.set_label(\"LLaMEA-SAGE\")\n", + " handle.set_label(\"LLaMEA-CFG\")\n", " if label == \"LHNS:vns\":\n", " handle.set_label(\"LHNS\")\n", "\n", @@ -581,7 +581,7 @@ " if label == \"ES\":\n", " handle.set_label(\"LLaMEA\")\n", " if label == \"ES-guided\":\n", - " handle.set_label(\"LLaMEA-SAGE\")\n", + " handle.set_label(\"LLaMEA-CFG\")\n", " if label == \"LHNS:vns\":\n", " handle.set_label(\"LHNS\")\n", "\n", diff --git a/iohblade/plots.py b/iohblade/plots.py index 509483c..445eb6a 100644 --- a/iohblade/plots.py +++ b/iohblade/plots.py @@ -452,9 +452,9 @@ def plot_experiment_CEG( if method == "ES": method_title = "LLaMEA" elif method == "ES-guided": - method_title = "LLaMEA-SAGE" + method_title = "LLaMEA-CFG" elif method == "ES-guided-new": - method_title = "LLaMEA-SAGE" + method_title = "LLaMEA-CFG" ax.set_title(f"{method_title} run:{seed}") if seed_i > 0: ax.set_ylabel(None) diff --git a/run_archive_guided-2.py b/run_archive_guided-2.py index 14e2d30..f490ae3 100644 --- a/run_archive_guided-2.py +++ b/run_archive_guided-2.py @@ -29,7 +29,7 @@ ] LLaMEA_1 = LLaMEA(llm, budget=budget, name="LLaMEA", mutation_prompts=mutation_prompts, n_parents=4, n_offspring=16, elitism=True) - LLaMEA_2 = LLaMEA(llm, budget=budget, name="LLaMEA-SAGE", mutation_prompts=mutation_prompts, n_parents=4, n_offspring=16, elitism=True, feature_guided_mutation=True, parent_selection="tournament", tournament_size=2) + LLaMEA_2 = LLaMEA(llm, budget=budget, name="LLaMEA-CFG", mutation_prompts=mutation_prompts, n_parents=4, n_offspring=16, elitism=True, feature_guided_mutation=True, parent_selection="tournament", tournament_size=2) methods = [LLaMEA_1, LLaMEA_2] diff --git a/run_archive_guided-3.py b/run_archive_guided-3.py index d4b6cff..6c22030 100644 --- a/run_archive_guided-3.py +++ b/run_archive_guided-3.py @@ -29,7 +29,7 @@ ] LLaMEA_1 = LLaMEA(llm, budget=budget, name="LLaMEA", mutation_prompts=mutation_prompts, n_parents=4, n_offspring=16, elitism=True) - LLaMEA_2 = LLaMEA(llm, budget=budget, name="LLaMEA-SAGE", mutation_prompts=mutation_prompts, n_parents=4, n_offspring=16, elitism=True, feature_guided_mutation=True, parent_selection="tournament", tournament_size=2) + LLaMEA_2 = LLaMEA(llm, budget=budget, name="LLaMEA-CFG", mutation_prompts=mutation_prompts, n_parents=4, n_offspring=16, elitism=True, feature_guided_mutation=True, parent_selection="tournament", tournament_size=2) methods = [LLaMEA_1, LLaMEA_2] diff --git a/run_archive_guided.py b/run_archive_guided.py index 7603e3c..e407f67 100644 --- a/run_archive_guided.py +++ b/run_archive_guided.py @@ -27,7 +27,7 @@ ] LLaMEA_1 = LLaMEA(llm, budget=budget, name="LLaMEA", mutation_prompts=mutation_prompts, n_parents=8, n_offspring=8, elitism=True) - LLaMEA_2 = LLaMEA(llm, budget=budget, name="LLaMEA-SAGE", mutation_prompts=mutation_prompts, n_parents=8, n_offspring=8, elitism=True, feature_guided_mutation=True) + LLaMEA_2 = LLaMEA(llm, budget=budget, name="LLaMEA-CFG", mutation_prompts=mutation_prompts, n_parents=8, n_offspring=8, elitism=True, feature_guided_mutation=True) methods = [LLaMEA_1, LLaMEA_2] diff --git a/visualize_1.ipynb b/visualize_1.ipynb index 3c53c1c..c4ce812 100644 --- a/visualize_1.ipynb +++ b/visualize_1.ipynb @@ -666,7 +666,7 @@ "new_labels = []\n", "for label in previous_labels:\n", " if label == \"ES-guided-new\":\n", - " new_labels.append(\"LLaMEA-SAGE\")\n", + " new_labels.append(\"LLaMEA-CFG\")\n", " elif label == \"ES\":\n", " new_labels.append(\"LLaMEA\")\n", " else:\n", @@ -704,7 +704,7 @@ "new_labels = []\n", "for label in previous_labels:\n", " if label == \"ES-guided-new vs ES\":\n", - " new_labels.append(\"LLaMEA-SAGE vs LLaMEA\")\n", + " new_labels.append(\"LLaMEA-CFG vs LLaMEA\")\n", " else:\n", " new_labels.append(label)\n", "ax.legend(handles=handles, labels=new_labels)\n", diff --git a/visualize_2.ipynb b/visualize_2.ipynb index 409510b..f94ba36 100644 --- a/visualize_2.ipynb +++ b/visualize_2.ipynb @@ -184,7 +184,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "['LLaMEA', 'LLaMEA-SAGE', 'MCTS_AHD', 'ES-guided', 'LHNS:vns', 'ES']\n", + "['LLaMEA', 'LLaMEA-CFG', 'MCTS_AHD', 'ES-guided', 'LHNS:vns', 'ES']\n", "['MA_BBOB']\n" ] } @@ -210,7 +210,7 @@ "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n", "\n", "\n", - " token_df['method_name'].replace({'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-SAGE', 'LHNS:vns': \"LHNS\"}, inplace=True)\n" + " token_df['method_name'].replace({'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-CFG', 'LHNS:vns': \"LHNS\"}, inplace=True)\n" ] } ], @@ -226,7 +226,7 @@ "\n", "token_df = pd.concat([token_df1, token_df2, token_df3], axis=0)\n", "\n", - "token_df['method_name'].replace({'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-SAGE', 'LHNS:vns': \"LHNS\"}, inplace=True)\n", + "token_df['method_name'].replace({'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-CFG', 'LHNS:vns': \"LHNS\"}, inplace=True)\n", "\n", "\n" ] @@ -261,7 +261,7 @@ "source": [ "\n", "filtered = token_df[\n", - " token_df[\"method_name\"].isin([\"LLaMEA\", \"LLaMEA-SAGE\", \"LHNS\", \"MCTS_AHD\"])\n", + " token_df[\"method_name\"].isin([\"LLaMEA\", \"LLaMEA-CFG\", \"LHNS\", \"MCTS_AHD\"])\n", "]\n", "\n", "\n", @@ -313,7 +313,7 @@ } ], "source": [ - "fig = plot_convergence(logger, metric=\"AOCC\", aggregation=\"mean\", methods=[\"LLaMEA\", \"LLaMEA-SAGE\", \"LHNS:vns\",\"MCTS_AHD\"], replace_labels={'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-SAGE'}, save=False, return_fig=True, budget=200, separate_lines=False, variance_aggregation=\"sem\")\n", + "fig = plot_convergence(logger, metric=\"AOCC\", aggregation=\"mean\", methods=[\"LLaMEA\", \"LLaMEA-CFG\", \"LHNS:vns\",\"MCTS_AHD\"], replace_labels={'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-CFG'}, save=False, return_fig=True, budget=200, separate_lines=False, variance_aggregation=\"sem\")\n", "ax = fig.axes[0] # or however you're getting your axis\n", "fig.set_size_inches(6, 5)\n", "handles, previous_labels = ax.get_legend_handles_labels()\n", @@ -322,7 +322,7 @@ " if label == \"ES\":\n", " new_labels.append(\"LLaMEA\")\n", " elif label == \"ES-guided\":\n", - " new_labels.append(\"LLaMEA-SAGE\")\n", + " new_labels.append(\"LLaMEA-CFG\")\n", " elif label == \"LHNS:vns\":\n", " new_labels.append(\"LHNS\")\n", " else:\n", @@ -367,7 +367,7 @@ } ], "source": [ - "fig = plot_speedup(logger, method_fast=\"LLaMEA-SAGE\", method_slow=\"LLaMEA\", save=False, return_fig=True, budget=200, replace_labels={'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-SAGE'})\n", + "fig = plot_speedup(logger, method_fast=\"LLaMEA-CFG\", method_slow=\"LLaMEA\", save=False, return_fig=True, budget=200, replace_labels={'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-CFG'})\n", "ax = fig.axes[0] # or however you're getting your axis\n", "fig.set_size_inches(6, 5)\n", "ax.set_title(None)\n", @@ -375,7 +375,7 @@ "new_labels = []\n", "for label in previous_labels:\n", " if label == \"ES-guided vs ES\":\n", - " new_labels.append(\"LLaMEA-SAGE vs LLaMEA\")\n", + " new_labels.append(\"LLaMEA-CFG vs LLaMEA\")\n", " else:\n", " new_labels.append(label)\n", "ax.legend(handles=handles, labels=new_labels)\n", @@ -447,7 +447,7 @@ " \n", " 0\n", " MA_BBOB\n", - " LLaMEA-SAGE\n", + " LLaMEA-CFG\n", " MCTS_AHD\n", " 5\n", " 144.165807\n", @@ -469,7 +469,7 @@ ], "text/plain": [ " problem method_a method_b n_seeds mean_auc_a mean_auc_b \\\n", - "0 MA_BBOB LLaMEA-SAGE MCTS_AHD 5 144.165807 112.480177 \n", + "0 MA_BBOB LLaMEA-CFG MCTS_AHD 5 144.165807 112.480177 \n", "\n", " median_auc_a median_auc_b mean_auc_diff ci_low ci_high cliffs_delta \\\n", "0 155.035198 98.100125 31.68563 7.1042 58.820452 0.6 \n", @@ -485,12 +485,12 @@ ], "source": [ "compare_auc(logger, \n", - " method_a= \"LLaMEA-SAGE\",\n", + " method_a= \"LLaMEA-CFG\",\n", " method_b=\"LHNS:vns\",\n", " budget=200,\n", " metric=\"fitness\",\n", " test=\"wilcoxon\",\n", - " replace_labels={'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-SAGE'})" + " replace_labels={'ES': 'LLaMEA', 'ES-guided': 'LLaMEA-CFG'})" ] }, { diff --git a/visualize_3.ipynb b/visualize_3.ipynb index 365091a..1e79b3c 100644 --- a/visualize_3.ipynb +++ b/visualize_3.ipynb @@ -225,7 +225,7 @@ " if label == \"ES\":\n", " new_labels.append(\"LLaMEA\")\n", " elif label == \"ES-guided\":\n", - " new_labels.append(\"LLaMEA-SAGE\")\n", + " new_labels.append(\"LLaMEA-CFG\")\n", " elif label == \"LHNS:vns\":\n", " new_labels.append(\"LHNS\")\n", " else:\n",