FEAT Restructuring Attacks (microsoft#1059)

Co-authored-by: Bashir Partovi <bpartovi@microsoft.com>

Author: bashirpartovi

doc/api.rst

@@ -17,7 +17,7 @@ API Reference
 :py:mod:`pyrit.attacks`
 ============================
 
-.. automodule:: pyrit.attacks
+.. automodule:: pyrit.executor.attack
     :no-members:
     :no-inherited-members:
 
@@ -31,8 +31,6 @@ API Reference
     AttackExecutor
     AttackScoringConfig
     AttackStrategy
-    AttackStrategyLogAdapter
-    ContextT
     ContextComplianceAttack
     ConversationSession
     CrescendoAttack
@@ -47,9 +45,6 @@ API Reference
     TAPAttack
     TAPAttackContext
     TAPAttackResult
-    FuzzerAttack
-    FuzzerAttackContext
-    FuzzerAttackResult
     TreeOfAttacksWithPruningAttack
     SkeletonKeyAttack
     ConsoleAttackResultPrinter

doc/code/attacks/1_prompt_sending_attack.ipynb

@@ -48,8 +48,8 @@
     }
    ],
    "source": [
-    "from pyrit.attacks import ConsoleAttackResultPrinter, PromptSendingAttack\n",
     "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
+    "from pyrit.executor.attack import ConsoleAttackResultPrinter, PromptSendingAttack\n",
     "from pyrit.prompt_target import OpenAIChatTarget\n",
     "\n",
     "initialize_pyrit(memory_db_type=IN_MEMORY)\n",
@@ -206,13 +206,13 @@
    "source": [
     "import pathlib\n",
     "\n",
-    "from pyrit.attacks import (\n",
+    "from pyrit.common.path import DATASETS_PATH\n",
+    "from pyrit.executor.attack import (\n",
     "    AttackConverterConfig,\n",
     "    AttackExecutor,\n",
     "    ConsoleAttackResultPrinter,\n",
     "    PromptSendingAttack,\n",
     ")\n",
-    "from pyrit.common.path import DATASETS_PATH\n",
     "from pyrit.models import SeedPromptDataset\n",
     "from pyrit.prompt_converter import Base64Converter\n",
     "from pyrit.prompt_normalizer import PromptConverterConfiguration\n",
@@ -329,7 +329,7 @@
    "source": [
     "import pathlib\n",
     "\n",
-    "from pyrit.attacks import (\n",
+    "from pyrit.executor.attack import (\n",
     "    ConsoleAttackResultPrinter,\n",
     "    PromptSendingAttack,\n",
     "    SingleTurnAttackContext,\n",
@@ -432,7 +432,7 @@
    "source": [
     "from azure.ai.contentsafety.models import TextCategory\n",
     "\n",
-    "from pyrit.attacks import AttackScoringConfig, PromptSendingAttack\n",
+    "from pyrit.executor.attack import AttackScoringConfig, PromptSendingAttack\n",
     "from pyrit.prompt_target import OpenAIChatTarget\n",
     "from pyrit.score import (\n",
     "    AzureContentFilterScorer,\n",
@@ -623,8 +623,8 @@
     }
    ],
    "source": [
-    "from pyrit.attacks import AttackExecutor, PromptSendingAttack\n",
     "from pyrit.datasets import TextJailBreak\n",
+    "from pyrit.executor.attack import AttackExecutor, PromptSendingAttack\n",
     "from pyrit.models.prompt_request_response import PromptRequestResponse\n",
     "from pyrit.prompt_target import OpenAIChatTarget\n",
     "\n",
@@ -725,8 +725,8 @@
    "source": [
     "import uuid\n",
     "\n",
-    "from pyrit.attacks import PromptSendingAttack\n",
     "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
+    "from pyrit.executor.attack import PromptSendingAttack\n",
     "from pyrit.models import SeedPrompt, SeedPromptGroup\n",
     "from pyrit.prompt_target import OpenAIChatTarget\n",
     "\n",

doc/code/attacks/1_prompt_sending_attack.py

@@ -31,9 +31,10 @@
 # > It is required to manually set the memory instance using `initialize_pyrit`. For details, see the [Memory Configuration Guide](../memory/0_memory.md).
 #
 
-# %%
-from pyrit.attacks import ConsoleAttackResultPrinter, PromptSendingAttack
 from pyrit.common import IN_MEMORY, initialize_pyrit
+
+# %%
+from pyrit.executor.attack import ConsoleAttackResultPrinter, PromptSendingAttack
 from pyrit.prompt_target import OpenAIChatTarget
 
 initialize_pyrit(memory_db_type=IN_MEMORY)
@@ -57,13 +58,13 @@
 # %%
 import pathlib
 
-from pyrit.attacks import (
+from pyrit.common.path import DATASETS_PATH
+from pyrit.executor.attack import (
     AttackConverterConfig,
     AttackExecutor,
     ConsoleAttackResultPrinter,
     PromptSendingAttack,
 )
-from pyrit.common.path import DATASETS_PATH
 from pyrit.models import SeedPromptDataset
 from pyrit.prompt_converter import Base64Converter
 from pyrit.prompt_normalizer import PromptConverterConfiguration
@@ -99,7 +100,7 @@
 # %%
 import pathlib
 
-from pyrit.attacks import (
+from pyrit.executor.attack import (
     ConsoleAttackResultPrinter,
     PromptSendingAttack,
     SingleTurnAttackContext,
@@ -135,7 +136,7 @@
 # %%
 from azure.ai.contentsafety.models import TextCategory
 
-from pyrit.attacks import AttackScoringConfig, PromptSendingAttack
+from pyrit.executor.attack import AttackScoringConfig, PromptSendingAttack
 from pyrit.prompt_target import OpenAIChatTarget
 from pyrit.score import (
     AzureContentFilterScorer,
@@ -169,13 +170,14 @@
 result = await attack.execute_async(objective=objective)  # type: ignore
 await printer.print_conversation_async(result=result, include_auxiliary_scores=True)  # type: ignore
 
+from pyrit.datasets import TextJailBreak
+
 # %% [markdown]
 # ## Prepending Conversations
 #
 # If you prepend all or part of a conversation with `PromptSendingAttack`, that is also supported. You can call `set_prepended_conversation` to customize the beginning part of any message. For example, you could use this to do a multi-turn conversation. Below sets the system prompt for many messages.
 # %%
-from pyrit.attacks import AttackExecutor, PromptSendingAttack
-from pyrit.datasets import TextJailBreak
+from pyrit.executor.attack import AttackExecutor, PromptSendingAttack
 from pyrit.models.prompt_request_response import PromptRequestResponse
 from pyrit.prompt_target import OpenAIChatTarget
 
@@ -207,8 +209,8 @@
 # %%
 import uuid
 
-from pyrit.attacks import PromptSendingAttack
 from pyrit.common import IN_MEMORY, initialize_pyrit
+from pyrit.executor.attack import PromptSendingAttack
 from pyrit.models import SeedPrompt, SeedPromptGroup
 from pyrit.prompt_target import OpenAIChatTarget
 

doc/code/auxiliary_attacks/0_auxiliary_attacks.ipynb

@@ -65,12 +65,12 @@
     }
    ],
    "source": [
-    "from pyrit.attacks import (\n",
+    "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
+    "from pyrit.executor.attack import (\n",
     "    AttackScoringConfig,\n",
     "    ConsoleAttackResultPrinter,\n",
     "    PromptSendingAttack,\n",
     ")\n",
-    "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
     "from pyrit.prompt_target import AzureMLChatTarget, OpenAIChatTarget\n",
     "from pyrit.score import SelfAskRefusalScorer, TrueFalseInverterScorer\n",
     "\n",
@@ -188,7 +188,7 @@
     }
    ],
    "source": [
-    "from pyrit.attacks import AttackConverterConfig\n",
+    "from pyrit.executor.attack import AttackConverterConfig\n",
     "from pyrit.prompt_converter import SuffixAppendConverter\n",
     "from pyrit.prompt_normalizer import PromptConverterConfiguration\n",
     "\n",

doc/code/auxiliary_attacks/0_auxiliary_attacks.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.0
+#       jupytext_version: 1.17.2
 #   kernelspec:
 #     display_name: pyrit-dev
 #     language: python
@@ -29,13 +29,14 @@
 #
 # First, we send a harmful prompt to Phi-3-mini without a GCG suffix. If the environment variables `PHI3_MINI_ENDPOINT` and `PHI3_MINI_KEY` are not set in your .env file, the target will default to the model with `AZURE_ML_MANAGED_ENDPOINT` and `AZURE_ML_MANAGED_KEY`.
 
+from pyrit.common import IN_MEMORY, initialize_pyrit
+
 # %%
-from pyrit.attacks import (
+from pyrit.executor.attack import (
     AttackScoringConfig,
     ConsoleAttackResultPrinter,
     PromptSendingAttack,
 )
-from pyrit.common import IN_MEMORY, initialize_pyrit
 from pyrit.prompt_target import AzureMLChatTarget, OpenAIChatTarget
 from pyrit.score import SelfAskRefusalScorer, TrueFalseInverterScorer
 
@@ -58,7 +59,7 @@
 # Next, let's apply a GCG suffix trained on Phi-3-mini to the base prompt using the `SuffixAppendConverter`.
 
 # %%
-from pyrit.attacks import AttackConverterConfig
+from pyrit.executor.attack import AttackConverterConfig
 from pyrit.prompt_converter import SuffixAppendConverter
 from pyrit.prompt_normalizer import PromptConverterConfiguration
 

doc/code/converters/2_using_converters.ipynb

@@ -58,12 +58,12 @@
     }
    ],
    "source": [
-    "from pyrit.attacks import (\n",
+    "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
+    "from pyrit.executor.attack import (\n",
     "    AttackConverterConfig,\n",
     "    ConsoleAttackResultPrinter,\n",
     "    PromptSendingAttack,\n",
     ")\n",
-    "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
     "from pyrit.prompt_converter import StringJoinConverter, VariationConverter\n",
     "from pyrit.prompt_normalizer import PromptConverterConfiguration\n",
     "from pyrit.prompt_target import OpenAIChatTarget, TextTarget\n",
@@ -91,7 +91,7 @@
     "result = await attack.execute_async(objective=objective)  # type: ignore\n",
     "\n",
     "printer = ConsoleAttackResultPrinter()\n",
-    "await printer.print_conversation_async(result=result) # type: ignore"
+    "await printer.print_conversation_async(result=result)  # type: ignore"
    ]
   }
  ],

doc/code/converters/2_using_converters.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.0
+#       jupytext_version: 1.17.2
 #   kernelspec:
 #     display_name: pyrit-dev
 #     language: python
@@ -26,13 +26,14 @@
 # In this example the converters are stacked. First a variation is found using `VariationConverter`, and then `StringJoinConverter` is used to add a dash between letters. Remember that order matters. If `StringJoinConverter` came first, we would be asking the LLM to make variations of the prompt:
 # "t-e-l-l- - m-e- -h-o-w- -t-o- -c-u-t- -d-o-w-n - a- -t-r-e-e"
 
+from pyrit.common import IN_MEMORY, initialize_pyrit
+
 # %%
-from pyrit.attacks import (
+from pyrit.executor.attack import (
     AttackConverterConfig,
     ConsoleAttackResultPrinter,
     PromptSendingAttack,
 )
-from pyrit.common import IN_MEMORY, initialize_pyrit
 from pyrit.prompt_converter import StringJoinConverter, VariationConverter
 from pyrit.prompt_normalizer import PromptConverterConfiguration
 from pyrit.prompt_target import OpenAIChatTarget, TextTarget

doc/code/converters/5_selectively_converting.ipynb

@@ -51,12 +51,12 @@
     }
    ],
    "source": [
-    "from pyrit.attacks import (\n",
+    "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
+    "from pyrit.executor.attack import (\n",
     "    AttackConverterConfig,\n",
     "    ConsoleAttackResultPrinter,\n",
     "    PromptSendingAttack,\n",
     ")\n",
-    "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
     "from pyrit.prompt_converter import Base64Converter\n",
     "from pyrit.prompt_normalizer import PromptConverterConfiguration\n",
     "from pyrit.prompt_target import TextTarget\n",

doc/code/converters/5_selectively_converting.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.0
+#       jupytext_version: 1.17.2
 #   kernelspec:
 #     display_name: pyrit-dev
 #     language: python
@@ -17,13 +17,14 @@
 #
 # You can selectively convert strings from text converters using most orchestrators or the `convert_tokens_async` function. This function uses a `start_token` and `end_token` to determine where to do the converting (by default these are the unicode characters ⟪ and ⟫). Here is an example that uses `PromptSendingAttack` to convert pieces of the text to base64.
 
+from pyrit.common import IN_MEMORY, initialize_pyrit
+
 # %%
-from pyrit.attacks import (
+from pyrit.executor.attack import (
     AttackConverterConfig,
     ConsoleAttackResultPrinter,
     PromptSendingAttack,
 )
-from pyrit.common import IN_MEMORY, initialize_pyrit
 from pyrit.prompt_converter import Base64Converter
 from pyrit.prompt_normalizer import PromptConverterConfiguration
 from pyrit.prompt_target import TextTarget

doc/code/converters/ansi_attack_converter.ipynb

@@ -257,13 +257,13 @@
     }
    ],
    "source": [
-    "from pyrit.attacks import (\n",
+    "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
+    "from pyrit.executor.attack import (\n",
     "    AttackConverterConfig,\n",
     "    AttackExecutor,\n",
     "    ConsoleAttackResultPrinter,\n",
     "    PromptSendingAttack,\n",
     ")\n",
-    "from pyrit.common import IN_MEMORY, initialize_pyrit\n",
     "from pyrit.prompt_converter import AnsiAttackConverter\n",
     "from pyrit.prompt_normalizer import PromptConverterConfiguration\n",
     "from pyrit.prompt_target import OpenAIChatTarget\n",