Skip to content

fix(lint): tune pylintrc and fix violations across core modules #1793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ppradyoth wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(lint): tune pylintrc and fix violations across core modules #1793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 26 additions & 14 deletions garak/_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@

@dataclass
class GarakSubConfig:
pass
"""Base dataclass for garak configuration sub-objects."""


@dataclass
Expand Down Expand Up @@ -132,11 +132,10 @@ def _key_exists(d: dict, key: str) -> bool:
if not isinstance(d, dict) and not isinstance(d, list):
return False
if isinstance(d, list):
return any([_key_exists(item, key) for item in d])
return any(_key_exists(item, key) for item in d)
if isinstance(d, dict) and key in d.keys():
return True
else:
return any([_key_exists(val, key) for val in d.values()])
return any(_key_exists(val, key) for val in d.values())


def _set_settings(config_obj, settings_obj: dict):
Expand Down Expand Up @@ -187,7 +186,8 @@ def _load_config_files(settings_filenames) -> dict:
print(f"⚠️ {msg}")
else:
logging.info(
f"API key found in {settings_filename}. Checking readability..."
"API key found in %s. Checking readability...",
settings_filename,
)
res = os.stat(settings_filename)
if res.st_mode & stat.S_IROTH or res.st_mode & stat.S_IRGRP:
Expand Down Expand Up @@ -228,7 +228,8 @@ def _load_config_files(settings_filenames) -> dict:


def _store_config(settings_files) -> None:
global system, run, plugins, reporting, version
"""Load config files and apply settings to the global config objects."""
global system, run, plugins, reporting, version # pylint: disable=global-statement
settings = _load_config_files(settings_files)
system = _set_settings(system, settings["system"])
run = _set_settings(run, settings["run"])
Expand All @@ -246,19 +247,24 @@ def _store_config(settings_files) -> None:
REQUESTS_AGENT = ""


def _garak_user_agent(dummy=None):
def _garak_user_agent(_dummy=None):
"""Return the current garak requests user-agent string.

Accepts an ignored positional arg to match the ``requests`` UA callback signature.
"""
return str(REQUESTS_AGENT)


def set_all_http_lib_agents(agent_string):
"""Set the same user-agent string for all HTTP libraries (requests, httpx, aiohttp)."""
set_http_lib_agents(
{"requests": agent_string, "httpx": agent_string, "aiohttp": agent_string}
)


def set_http_lib_agents(agent_strings: dict):

global REQUESTS_AGENT
"""Set per-library user-agent strings from a dict keyed by library name."""
global REQUESTS_AGENT # pylint: disable=global-statement

if "requests" in agent_strings:
from requests import utils
Expand All @@ -276,6 +282,7 @@ def set_http_lib_agents(agent_strings: dict):


def get_http_lib_agents():
"""Return the current user-agent strings for requests, httpx, and aiohttp."""
from requests import utils
import httpx
import aiohttp
Expand All @@ -289,7 +296,8 @@ def get_http_lib_agents():


def load_base_config() -> None:
global loaded
"""Load garak.core.yaml — the minimal base configuration."""
global loaded # pylint: disable=global-statement
settings_files = [str(transient.package_dir / "resources" / "garak.core.yaml")]
logging.debug("Loading configs from: %s", ",".join(settings_files))
_store_config(settings_files=settings_files)
Expand All @@ -299,9 +307,10 @@ def load_base_config() -> None:
def load_config(
site_config_filename="garak.site.yaml", run_config_filename=None
) -> None:
"""Load site and run config files on top of the base config."""
# would be good to bubble up things from run_config, e.g. generator, probe(s), detector(s)
# and then not have cli be upset when these are not given as cli params
global loaded
global loaded # pylint: disable=global-statement

settings_files = [str(transient.package_dir / "resources" / "garak.core.yaml")]

Expand All @@ -318,7 +327,7 @@ def load_config(
message = "Multiple site config files found (garak.site.json, garak.site.yaml, garak.site.yml). Please use only one site config format."
logging.error(message)
raise ValueError(message)
elif has_json:
if has_json:
settings_files.append(site_config_json)
elif has_yaml:
settings_files.append(site_config_yaml)
Expand Down Expand Up @@ -378,7 +387,9 @@ def load_config(
if has_json and (has_yaml or has_yml):
yaml_ext = ".yaml" if has_yaml else ".yml"
logging.warning(
f"Both {run_config_filename}.json and {yaml_ext} found. Using .json"
"Both %s.json and %s found. Using .json",
run_config_filename,
yaml_ext,
)
if has_json:
settings_files.append(json_path)
Expand Down Expand Up @@ -410,6 +421,7 @@ def load_config(
def parse_plugin_spec(
spec: str, category: str, probe_tag_filter: str = ""
) -> tuple[List[str], List[str]]:
"""Expand a plugin spec string (e.g. 'all', 'dan', 'probes.dan.AntiDAN') into lists of known and unknown plugin names."""
from garak._plugins import enumerate_plugins

if spec is None or spec.lower() in ("", "auto", "none"):
Expand Down Expand Up @@ -453,7 +465,7 @@ def parse_plugin_spec(
plugin_class_name = plugin_name.split(".")[-1]
m = importlib.import_module(f"garak.{plugin_module_name}")
c = getattr(m, plugin_class_name)
if not any([tag.startswith(probe_tag_filter) for tag in c.tags]):
if not any(tag.startswith(probe_tag_filter) for tag in c.tags):
plugins_to_skip.append(
plugin_name
) # using list.remove doesn't update for-loop position
Expand Down
48 changes: 41 additions & 7 deletions garak/command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@


def hint(msg, logging=None):
"""Print a probabilistic hint message and optionally log it.

Uses a global HINT_CHANCE probability so hints don't appear on every run.
The logging parameter is passed explicitly to avoid import-order issues with
the thin garak logging setup.
"""
# sub-optimal, but because our logging setup is thin & uses the global
# default, placing a top-level import can break logging - so we can't
# assume `logging` is imported at this point.
Expand All @@ -22,6 +28,7 @@ def hint(msg, logging=None):


def deprecation_notice(deprecated_item: str, version: str, logging=None):
"""Print and optionally log a deprecation notice for the given item."""
msg = f"DEPRECATION: {deprecated_item} is deprecated since version {version}"
visible_msg = f"✋ {msg}"
if logging is not None:
Expand All @@ -30,6 +37,7 @@ def deprecation_notice(deprecated_item: str, version: str, logging=None):


def start_logging():
"""Initialise logging and return the configured log filename."""
from garak import _config

log_filename = _config.transient.log_filename
Expand All @@ -40,6 +48,7 @@ def start_logging():


def start_run():
"""Set up the run UUID, reporting directory, and open the report file."""
import logging
import os
import uuid
Expand All @@ -49,7 +58,7 @@ def start_run():

logging.info("run started at %s", _config.transient.starttime_iso)
# print("ASSIGN UUID", args)
if _config.system.lite and "probes" not in _config.transient.cli_args and _config.transient.cli_args.list_probes is None and not _config.transient.cli_args.list_detectors and not _config.transient.cli_args.list_generators and not _config.transient.cli_args.list_buffs and not _config.transient.cli_args.list_config and not _config.transient.cli_args.plugin_info and not _config.run.interactive: # type: ignore
if _config.system.lite and "probes" not in _config.transient.cli_args and _config.transient.cli_args.list_probes is None and not _config.transient.cli_args.list_detectors and not _config.transient.cli_args.list_generators and not _config.transient.cli_args.list_buffs and not _config.transient.cli_args.list_config and not _config.transient.cli_args.plugin_info and not _config.run.interactive: # type: ignore # pylint: disable=no-member # cli_args attrs set dynamically by argparse
hint(
"The current/default config is optimised for speed rather than thoroughness. Try e.g. --config full for a stronger test, or specify some probes.",
logging=logging,
Expand Down Expand Up @@ -122,6 +131,7 @@ def start_run():


def end_run():
"""Close the report file, write a completion entry, and build the HTML digest."""
import datetime
import logging

Expand All @@ -146,9 +156,11 @@ def end_run():

digest_filename = _config.transient.report_filename.replace(".jsonl", ".html")
print(f"📜 report html summary being written to {digest_filename}")
# pylint: disable=broad-exception-caught # report building must not crash the CLI run
try:
write_report_digest(_config.transient.report_filename, digest_filename)
except Exception as e:
# pylint: enable=broad-exception-caught
msg = "Didn't successfully build the report - JSON log preserved. " + repr(e)
logging.exception(e)
logging.info(msg)
Expand All @@ -163,6 +175,7 @@ def _tier_name(tier_value):
"""Convert a tier int value to its enum name string."""
try:
from garak.probes._tier import Tier

return Tier(int(tier_value)).name
except (ValueError, TypeError):
return ""
Expand All @@ -171,7 +184,7 @@ def _tier_name(tier_value):
def _truncate(text, max_len=80):
"""Truncate text to max_len, appending ellipsis if needed."""
if len(text) > max_len:
return text[:max_len - 1] + "…"
return text[: max_len - 1] + "…"
return text


Expand All @@ -180,7 +193,12 @@ def _truncate(text, max_len=80):
# "name" and "active" are always included and handled separately.
_PLUGIN_TABLE_COLUMNS = {
"probes": [
("tier", lambda info: _tier_name(info.get("tier")) if info.get("tier") is not None else ""),
(
"tier",
lambda info: (
_tier_name(info.get("tier")) if info.get("tier") is not None else ""
),
),
("description", lambda info: _truncate(info.get("description", ""))),
],
# Future plugin types can define their own extra columns here, e.g.:
Expand All @@ -190,7 +208,7 @@ def _truncate(text, max_len=80):
}


def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0):
def print_plugins(prefix: str, color, selected_plugins=None, verbose: int = 0):
"""
Print plugins for a category (probes/detectors/generators/buffs).

Expand All @@ -201,7 +219,7 @@ def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0):
verbose: Verbosity level. 0 = plain list, >=1 = markdown table with metadata.
"""
from colorama import Style
from garak._plugins import enumerate_plugins, plugin_info as get_plugin_info, PLUGIN_TYPES
from garak._plugins import enumerate_plugins, PLUGIN_TYPES

if prefix not in PLUGIN_TYPES:
raise ValueError(f"Requested prefix '{prefix}' is not a valid plugin type")
Expand All @@ -217,7 +235,10 @@ def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0):
print(f"No {prefix} match the provided filter")
return

short = [(p.replace(f"{prefix}.", ""), a, p) for p, a, *_ in [(pn, ac, pn) for pn, ac in rows]]
short = [
(p.replace(f"{prefix}.", ""), a, p)
for p, a, *_ in [(pn, ac, pn) for pn, ac in rows]
]
if selected_plugins is None:
module_names = {(m.split(".")[0], True, None) for m, a, _ in short}
short += module_names
Expand Down Expand Up @@ -270,7 +291,12 @@ def _print_plugins_table(sorted_items, prefix):
print(f"{prefix}:")
print(
markdown_table(table_data)
.set_params(row_sep="markdown", padding_width=1, padding_weight="centerleft", quote=False)
.set_params(
row_sep="markdown",
padding_width=1,
padding_weight="centerleft",
quote=False,
)
.get_markdown()
)

Expand All @@ -288,25 +314,29 @@ def print_probes(selected_probes=None, verbose=0):


def print_detectors(selected_detectors=None):
"""Print available detectors, optionally filtered to selected_detectors."""
from colorama import Fore

print_plugins("detectors", Fore.LIGHTBLUE_EX, selected_detectors)


def print_generators():
"""Print all available generators."""
from colorama import Fore

print_plugins("generators", Fore.LIGHTMAGENTA_EX)


def print_buffs():
"""Print all available buffs."""
from colorama import Fore

print_plugins("buffs", Fore.LIGHTGREEN_EX)


# describe plugin
def plugin_info(plugin_name):
"""Print all known metadata for the named plugin."""
from garak._plugins import plugin_info

info = plugin_info(plugin_name)
Expand All @@ -333,13 +363,15 @@ def plugin_info(plugin_name):

# do a run
def probewise_run(generator, probe_names, evaluator, buffs):
"""Run probes one-by-one through the probewise harness."""
import garak.harnesses.probewise

probewise_h = garak.harnesses.probewise.ProbewiseHarness()
probewise_h.run(generator, probe_names, evaluator, buffs)


def pxd_run(generator, probe_names, detector_names, evaluator, buffs):
"""Run probes through the probe-x-detector (PxD) harness."""
import garak.harnesses.pxd

pxd_h = garak.harnesses.pxd.PxD()
Expand All @@ -359,6 +391,7 @@ def _enumerate_obj_values(o):


def list_config():
"""Print all current garak config values to stdout."""
from garak import _config

print("_config:")
Expand All @@ -370,6 +403,7 @@ def list_config():


def write_report_digest(report_filename, html_report_filename):
"""Build and write the HTML digest for the given JSONL report file."""
from garak.analyze import report_digest

digest = report_digest.build_digest(report_filename)
Expand Down
Loading