Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions .pyrit_conf_example
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,27 @@ memory_db_type: sqlite
#
# Each initializer can be specified as:
# - A simple string (name only)
# - A dictionary with 'name' and optional 'args' for constructor arguments
# - A dictionary with 'name' and optional 'args' for parameters
#
# Parameters are lists of strings. Use the CLI command
# `pyrit_scan --list-initializers` to see available parameters.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we implement something like pyrit_scan --list-initializers --help or pyrit_scan --list-parameters simple instead to see available parameters? I imagine that users would want a dedicated interface for figuring out parameters once they choose one or more initializers and don't need to list them all

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the idea, but lots of things to do in the CLI, I'd rather punt :)

#
# Example:
# initializers:
# - simple
# - name: airt
# - name: target
# args:
# some_param: value
# tags:
# - default
# - scorer
initializers:
- simple
- name: simple
- name: scorer
- name: target
args:
tags:
- default
- scorer

# Operator and Operation Labels
# ------------------------------
Expand Down
4 changes: 3 additions & 1 deletion build_scripts/evaluate_scorers.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,11 @@ async def evaluate_scorers() -> None:
5. Save results to scorer_evals directory
"""
print("Initializing PyRIT...")
target_init = TargetInitializer()
target_init.params = {"tags": ["default", "scorer"]}
await initialize_pyrit_async(
memory_db_type=IN_MEMORY,
initializers=[TargetInitializer(tags=["default", "scorer"]), ScorerInitializer()],
initializers=[target_init, ScorerInitializer()],
)

registry = ScorerRegistry.get_registry_singleton()
Expand Down
4 changes: 2 additions & 2 deletions doc/code/setup/pyrit_initializer.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
" def execution_order(self) -> int:\n",
" return 2 # Lower numbers run first (default is 1)\n",
"\n",
" async def initialize_async(self) -> None:\n",
" async def initialize_async(self, *, params=None) -> None:\n",
" set_default_value(class_type=OpenAIChatTarget, parameter_name=\"temperature\", value=0.9)\n",
"\n",
" @property\n",
Expand Down Expand Up @@ -159,7 +159,7 @@
" def execution_order(self) -> int:\n",
" return 2 # Lower numbers run first (default is 1)\n",
"\n",
" async def initialize_async(self) -> None:\n",
" async def initialize_async(self, *, params=None) -> None:\n",
" set_default_value(class_type=OpenAIChatTarget, parameter_name=\"temperature\", value=0.9)\n",
"\n",
" @property\n",
Expand Down
4 changes: 2 additions & 2 deletions doc/code/setup/pyrit_initializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def name(self) -> str:
def execution_order(self) -> int:
return 2 # Lower numbers run first (default is 1)

async def initialize_async(self) -> None:
async def initialize_async(self, *, params=None) -> None:
set_default_value(class_type=OpenAIChatTarget, parameter_name="temperature", value=0.9)

@property
Expand Down Expand Up @@ -107,7 +107,7 @@ def name(self) -> str:
def execution_order(self) -> int:
return 2 # Lower numbers run first (default is 1)

async def initialize_async(self) -> None:
async def initialize_async(self, *, params=None) -> None:
set_default_value(class_type=OpenAIChatTarget, parameter_name="temperature", value=0.9)

@property
Expand Down
6 changes: 4 additions & 2 deletions doc/setup/pyrit_conf.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ A list of built-in initializers to run during PyRIT initialization. Initializers
Each entry can be:

- **A simple string** — just the initializer name
- **A dictionary** — with `name` and optional `args` for constructor arguments
- **A dictionary** — with `name` and optional `args` (each arg is a list of strings passed to `initialize_async`)

Example:

Expand All @@ -53,7 +53,9 @@ initializers:
- simple
- name: airt
args:
some_param: value
tags:
- default
- scorer
```

Use `pyrit list initializers` in the CLI to see all registered initializers. See the [initializer documentation notebook](../code/setup/pyrit_initializer.ipynb) for reference.
Expand Down
2 changes: 1 addition & 1 deletion docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,5 +183,5 @@ The JupyterLab instance is configured to run without authentication by default f

- 📖 **[Docker Installation Guide](./../doc/setup/1b_install_docker.md)** - Complete user-friendly installation instructions
- 🚀 **[PyRIT Documentation](https://azure.github.io/PyRIT/)** - Full documentation site
- 🔧 **[Contributing Guide](https://azure.github.io/PyRIT/contributing/README.html)** - For developers and contributors
- 🔧 **[Contributing Guide](https://azure.github.io/PyRIT/contributing/readme/)** - For developers and contributors
- 🐛 **[Issues](https://github.com/Azure/PyRIT/issues)** - Report bugs or request features
87 changes: 74 additions & 13 deletions pyrit/cli/frontend_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def __init__(
config_file: Optional[Path] = None,
database: Optional[str] = None,
initialization_scripts: Optional[list[Path]] = None,
initializer_names: Optional[list[str]] = None,
initializer_names: Optional[list[Any]] = None,
env_files: Optional[list[Path]] = None,
log_level: Optional[int] = None,
):
Expand All @@ -94,7 +94,9 @@ def __init__(
The file uses .pyrit_conf extension but is YAML format.
database: Database type (InMemory, SQLite, or AzureSQL).
initialization_scripts: Optional list of initialization script paths.
initializer_names: Optional list of built-in initializer names to run.
initializer_names: Optional list of initializer entries. Each entry can be
a string name (e.g., "simple") or a dict with 'name' and optional 'args'
(e.g., {"name": "target", "args": {"tags": "default,scorer"}}).
env_files: Optional list of environment file paths to load in order.
log_level: Logging level constant (e.g., logging.WARNING). Defaults to logging.WARNING.

Expand Down Expand Up @@ -130,9 +132,7 @@ def __init__(
# Use canonical mapping from configuration_loader
self._database = _MEMORY_DB_TYPE_MAP[config.memory_db_type]
self._initialization_scripts = config._resolve_initialization_scripts()
self._initializer_names = (
[ic.name for ic in config._initializer_configs] if config._initializer_configs else None
)
self._initializer_configs = config._initializer_configs if config._initializer_configs else None
self._env_files = config._resolve_env_files()
self._operator = config.operator
self._operation = config.operation
Expand Down Expand Up @@ -289,15 +289,20 @@ async def run_scenario_async(

# Run initializers before scenario
initializer_instances = None
if context._initializer_names:
print(f"Running {len(context._initializer_names)} initializer(s)...")
if context._initializer_configs:
print(f"Running {len(context._initializer_configs)} initializer(s)...")
sys.stdout.flush()

initializer_instances = []

for name in context._initializer_names:
initializer_class = context.initializer_registry.get_class(name)
initializer_instances.append(initializer_class())
for config in context._initializer_configs:
initializer_class = context.initializer_registry.get_class(config.name)
instance = initializer_class()
if config.args:
instance.params = {
k: [str(i) for i in v] if isinstance(v, list) else [str(v)] for k, v in config.args.items()
}
initializer_instances.append(instance)

# Re-initialize PyRIT with the scenario-specific initializers
# This resets memory and applies initializer defaults
Expand Down Expand Up @@ -479,6 +484,13 @@ def format_initializer_metadata(*, initializer_metadata: InitializerMetadata) ->
else:
print(" Required Environment Variables: None")

if initializer_metadata.supported_parameters:
print(" Supported Parameters:")
for param_name, param_desc, param_required, param_default in initializer_metadata.supported_parameters:
req_str = " (required)" if param_required else ""
default_str = f" [default: {param_default}]" if param_default else ""
print(f" - {param_name}{req_str}{default_str}: {param_desc}")

if initializer_metadata.class_description:
print(" Description:")
print(_format_wrapped_text(text=initializer_metadata.class_description, indent=" "))
Expand Down Expand Up @@ -775,7 +787,11 @@ async def print_initializers_list_async(*, context: FrontendCore, discovery_path
"initialization scripts, and env files. CLI arguments override config file values. "
"If not specified, ~/.pyrit/.pyrit_conf is loaded if it exists."
),
"initializers": "Built-in initializer names to run before the scenario (e.g., openai_objective_target)",
"initializers": (
"Built-in initializer names to run before the scenario. "
"Supports optional params with name:key=val syntax "
"(e.g., target:tags=default,scorer dataset:mode=strict)"
),
"initialization_scripts": "Paths to custom Python initialization scripts to run before the scenario",
"env_files": "Paths to environment files to load in order (e.g., .env.production .env.local). Later files "
"override earlier ones.",
Expand All @@ -792,6 +808,51 @@ async def print_initializers_list_async(*, context: FrontendCore, discovery_path
}


def _parse_initializer_arg(arg: str) -> dict[str, Any]:
"""
Parse an initializer CLI argument into a dict for ConfigurationLoader.

Supports two formats:
- Simple name: "simple" → {"name": "simple"}
- Name with params: "target:tags=default,scorer" → {"name": "target", "args": {"tags": "default,scorer"}}

For multiple params on one initializer, separate with semicolons: "name:key1=val1;key2=val2"
For multiple initializers with params, space-separate them: "target:tags=a,b dataset:mode=strict"

Args:
arg: The CLI argument string.

Returns:
dict: A dict with 'name' and optionally 'args' keys.

Raises:
ValueError: If the argument format is invalid.
"""
if ":" not in arg:
return arg # type: ignore[return-value]

name, params_str = arg.split(":", 1)
if not name:
raise ValueError(f"Invalid initializer argument '{arg}': missing name before ':'")

args: dict[str, list[str]] = {}
for pair in params_str.split(";"):
pair = pair.strip()
if not pair:
continue
if "=" not in pair:
raise ValueError(f"Invalid initializer parameter '{pair}' in '{arg}': expected key=value format")
key, value = pair.split("=", 1)
key = key.strip()
if not key:
raise ValueError(f"Invalid initializer parameter in '{arg}': empty key")
args[key] = [v.strip() for v in value.split(",")]

if args:
return {"name": name, "args": args}
return name # type: ignore[return-value]


def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
"""
Parse run command arguments from a string (for shell mode).
Expand Down Expand Up @@ -839,11 +900,11 @@ def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
i = 1
while i < len(parts):
if parts[i] == "--initializers":
# Collect initializers until next flag
# Collect initializers until next flag, parsing name:key=val syntax
result["initializers"] = []
i += 1
while i < len(parts) and not parts[i].startswith("--"):
result["initializers"].append(parts[i])
result["initializers"].append(_parse_initializer_arg(parts[i]))
i += 1
elif parts[i] == "--initialization-scripts":
# Collect script paths until next flag
Expand Down
21 changes: 13 additions & 8 deletions pyrit/cli/pyrit_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def parse_args(*, args: Optional[list[str]] = None) -> Namespace:

parser.add_argument(
"--initializers",
type=str,
type=frontend_core._parse_initializer_arg,
nargs="+",
help=frontend_core.ARG_HELP["initializers"],
)
Expand Down Expand Up @@ -165,12 +165,17 @@ async def initialize_and_run_async(*, parsed_args: Namespace) -> int:

# Run initializers up-front (backend runs them once at startup, not per-scenario)
initializer_instances = None
if context._initializer_names:
print(f"Running {len(context._initializer_names)} initializer(s)...")
if context._initializer_configs:
print(f"Running {len(context._initializer_configs)} initializer(s)...")
initializer_instances = []
for name in context._initializer_names:
initializer_class = context.initializer_registry.get_class(name)
initializer_instances.append(initializer_class())
for config in context._initializer_configs:
initializer_class = context.initializer_registry.get_class(config.name)
instance = initializer_class()
if config.args:
instance.params = {
k: [str(i) for i in v] if isinstance(v, list) else [str(v)] for k, v in config.args.items()
}
initializer_instances.append(instance)

# Re-initialize with initializers applied
await initialize_pyrit_async(
Expand All @@ -196,14 +201,14 @@ async def initialize_and_run_async(*, parsed_args: Namespace) -> int:
print(f"🚀 Starting PyRIT backend on http://{parsed_args.host}:{parsed_args.port}")
print(f" API Docs: http://{parsed_args.host}:{parsed_args.port}/docs")

config = uvicorn.Config(
uvicorn_config = uvicorn.Config(
"pyrit.backend.main:app",
host=parsed_args.host,
port=parsed_args.port,
log_level=parsed_args.log_level,
reload=parsed_args.reload,
)
server = uvicorn.Server(config)
server = uvicorn.Server(uvicorn_config)
await server.serve()

return 0
Expand Down
2 changes: 1 addition & 1 deletion pyrit/cli/pyrit_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def parse_args(args: Optional[list[str]] = None) -> Namespace:

parser.add_argument(
"--initializers",
type=str,
type=frontend_core._parse_initializer_arg,
nargs="+",
help=frontend_core.ARG_HELP["initializers"],
)
Expand Down
10 changes: 8 additions & 2 deletions pyrit/cli/pyrit_shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class PyRITShell(cmd.Cmd):
--no-animation Disable the animated startup banner

Run Command Options:
--initializers <name> ... Built-in initializers to run before the scenario
--initializers <name> ... Built-in initializers (supports name:key=val1,val2 syntax)
--initialization-scripts <...> Custom Python scripts to run before the scenario
--env-files <path> ... Environment files to load in order (overrides startup default)
--strategies, -s <s1> ... Strategy names to use
Expand Down Expand Up @@ -135,7 +135,7 @@ def do_run(self, line: str) -> None:
run <scenario_name> [options]

Options:
--initializers <name> ... Built-in initializers to run before the scenario
--initializers <name> ... Built-in initializers (supports name:key=val1,val2 syntax)
--initialization-scripts <...> Custom Python scripts to run before the scenario
--env-files <path> ... Environment files to load in order
--strategies, -s <s1> <s2> ... Strategy names to use
Expand All @@ -150,6 +150,8 @@ def do_run(self, line: str) -> None:
load_default_datasets
run garak.encoding --initializers custom_target \
load_default_datasets --strategies base64 rot13
run foundry --initializers target:tags=default,scorer \
dataset:mode=strict --strategies base64
run foundry --initializers openai_objective_target \
load_default_datasets --max-concurrency 10 --max-retries 3
run garak.encoding --initializers custom_target \
Expand Down Expand Up @@ -360,6 +362,10 @@ def do_help(self, arg: str) -> None:
print(f" {frontend_core.ARG_HELP['initializers']}")
print(" Every scenario requires at least one initializer")
print(" Example: run foundry --initializers openai_objective_target load_default_datasets")
print(" With params: run foundry --initializers target:tags=default,scorer")
print(
" Multiple with params: run foundry --initializers target:tags=default,scorer dataset:mode=strict"
)
print()
print(" --initialization-scripts <path> [<path> ...] (Alternative to --initializers)")
print(f" {frontend_core.ARG_HELP['initialization_scripts']}")
Expand Down
6 changes: 6 additions & 0 deletions pyrit/registry/class_registries/initializer_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ class InitializerMetadata(ClassRegistryEntry):
# Execution order priority (lower = earlier).
execution_order: int = field(kw_only=True)

# Supported parameters as tuples of (name, description, required, default).
supported_parameters: tuple[tuple[str, str, bool, Optional[list[str]]], ...] = field(kw_only=True, default=())


class InitializerRegistry(BaseClassRegistry["PyRITInitializer", InitializerMetadata]):
"""
Expand Down Expand Up @@ -223,6 +226,9 @@ def _build_metadata(self, name: str, entry: ClassEntry[PyRITInitializer]) -> Ini
display_name=instance.name,
required_env_vars=tuple(instance.required_env_vars),
execution_order=instance.execution_order,
supported_parameters=tuple(
(p.name, p.description, p.required, p.default) for p in instance.supported_parameters
),
)
except Exception as e:
logger.warning(f"Failed to get metadata for {name}: {e}")
Expand Down
10 changes: 8 additions & 2 deletions pyrit/setup/configuration_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,8 +325,14 @@ def _resolve_initializers(self) -> Sequence["PyRITInitializer"]:
f"Initializer '{config.name}' not found in registry.\nAvailable initializers: {available}"
)

# Instantiate with args if provided
instance = initializer_class(**config.args) if config.args else initializer_class()
# Instantiate and set params if provided
instance = initializer_class()
if config.args:
instance.params = {
k: [str(i) for i in v] if isinstance(v, list) else [str(v)] for k, v in config.args.items()
}
# Validate params early against supported_parameters to fail fast
instance._validate_params(params=instance.params)

resolved.append(instance)

Expand Down
Loading
Loading