diff --git a/.gitignore b/.gitignore index 630f838700..eb2bfd419f 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,17 @@ TEST-**.xml # Mac OS .DS_Store, which is a file that stores custom attributes of its containing folder .DS_Store *.env* + +# # Métricas +/metrics-before-radon +/metrics-before-pylint +/metrics-after-pylint +/metrics-before-codecarbon +/metrics-before-pytest + +extract_metrics_before_radon.py +extract_metrics_before_pylint.py +extract_metrics_after_pylint.py +extract_score_before_pylint.py +extract_metrics_before_codecarbon.py +extract_metrics_before_pytest.py \ No newline at end of file diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 5e43b0eef3..b517b72c18 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -25,42 +25,95 @@ class Event(Enum): @dataclass -class FilterContext: - """A dataclass containing the information that should be filtered, and output information of the filtering.""" +class FilterInput: # pylint: disable=too-many-instance-attributes + """Input data for filtering: event details and message content.""" - # Input context - event: Event # The type of event - author: User | Member | None # Who triggered the event - channel: TextChannel | VoiceChannel | StageChannel | Thread | DMChannel | None # The channel involved - content: str | Iterable # What actually needs filtering. The Iterable type depends on the filter list. - message: Message | None # The message involved - embeds: list[Embed] = field(default_factory=list) # Any embeds involved - attachments: list[discord.Attachment | FileAttachment] = field(default_factory=list) # Any attachments sent. + event: Event + author: User | Member | None + channel: TextChannel | VoiceChannel | StageChannel | Thread | DMChannel | None + content: str | Iterable + message: Message | None + embeds: list[Embed] = field(default_factory=list) + attachments: list[discord.Attachment | FileAttachment] = field(default_factory=list) before_message: Message | None = None message_cache: MessageCache | None = None - # Output context - dm_content: str = "" # The content to DM the invoker - dm_embed: str = "" # The embed description to DM the invoker - send_alert: bool = False # Whether to send an alert for the moderators - alert_content: str = "" # The content of the alert - alert_embeds: list[Embed] = field(default_factory=list) # Any embeds to add to the alert - action_descriptions: list[str] = field(default_factory=list) # What actions were taken - matches: list[str] = field(default_factory=list) # What exactly was found - notification_domain: str = "" # A domain to send the user for context - filter_info: dict[Filter, str] = field(default_factory=dict) # Additional info from a filter. - messages_deletion: bool = False # Whether the messages were deleted. Can't upload deletion log otherwise. - blocked_exts: set[str] = field(default_factory=set) # Any extensions blocked (used for snekbox) + + +@dataclass +class FilterOutput: # pylint: disable=too-many-instance-attributes + """Output data produced by filtering: alerts, actions, and results.""" + + dm_content: str = "" + dm_embed: str = "" + send_alert: bool = False + alert_content: str = "" + alert_embeds: list[Embed] = field(default_factory=list) + action_descriptions: list[str] = field(default_factory=list) + matches: list[str] = field(default_factory=list) + notification_domain: str = "" + filter_info: dict[Filter, str] = field(default_factory=dict) + messages_deletion: bool = False + blocked_exts: set[str] = field(default_factory=set) potential_phish: dict[FilterList, set[str]] = field(default_factory=dict) - # Additional actions to perform + + +_FILTER_CONTEXT_DIRECT_FIELDS = frozenset({ + 'input', 'output', 'additional_actions', 'related_messages', + 'related_channels', 'uploaded_attachments', 'upload_deletion_logs', +}) + + +@dataclass +class FilterContext: + """A dataclass containing the information that should be filtered, and output information of the filtering.""" + + input: FilterInput + output: FilterOutput additional_actions: list[Callable[[FilterContext], Coroutine]] = field(default_factory=list) - related_messages: set[Message] = field(default_factory=set) # Deletion will include these. + related_messages: set[Message] = field(default_factory=set) related_channels: set[TextChannel | Thread | DMChannel] = field(default_factory=set) - uploaded_attachments: dict[int, list[str]] = field(default_factory=dict) # Message ID to attachment URLs. - upload_deletion_logs: bool = True # Whether it's allowed to upload deletion logs. + uploaded_attachments: dict[int, list[str]] = field(default_factory=dict) + upload_deletion_logs: bool = True + + @property + def in_guild(self) -> bool: + """Whether the context is from a guild channel (not a DM).""" + return self.input.channel is None or self.input.channel.guild is not None + + def __getattr__(self, name): + try: + input_obj = object.__getattribute__(self, 'input') + if hasattr(input_obj, name): + return getattr(input_obj, name) + except AttributeError: + pass + try: + output_obj = object.__getattribute__(self, 'output') + if hasattr(output_obj, name): + return getattr(output_obj, name) + except AttributeError: + pass + raise AttributeError(f"'FilterContext' has no attribute '{name}'") - def __post_init__(self): - # If it's in the context of a DM channel, self.channel won't be None, but self.channel.guild will. - self.in_guild = self.channel is None or self.channel.guild is not None + def __setattr__(self, name, value): + if name in _FILTER_CONTEXT_DIRECT_FIELDS: + object.__setattr__(self, name, value) + return + try: + input_obj = object.__getattribute__(self, 'input') + if hasattr(input_obj, name): + setattr(input_obj, name, value) + return + except AttributeError: + pass + try: + output_obj = object.__getattribute__(self, 'output') + if hasattr(output_obj, name): + setattr(output_obj, name, value) + return + except AttributeError: + pass + object.__setattr__(self, name, value) @classmethod def from_message( @@ -68,17 +121,34 @@ def from_message( ) -> FilterContext: """Create a filtering context from the attributes of a message.""" return cls( - event, - message.author, - message.channel, - message.content, - message, - message.embeds, - message.attachments, - before, - cache + FilterInput( + event, + message.author, + message.channel, + message.content, + message, + message.embeds, + message.attachments, + before, + cache + ), + FilterOutput() ) def replace(self, **changes) -> FilterContext: """Return a new context object assigning new values to the specified fields.""" - return replace(self, **changes) + input_fields = FilterInput.__dataclass_fields__ + output_fields = FilterOutput.__dataclass_fields__ + input_changes = {} + output_changes = {} + context_changes = {} + for k, v in changes.items(): + if k in input_fields: + input_changes[k] = v + elif k in output_fields: + output_changes[k] = v + else: + context_changes[k] = v + new_input = replace(self.input, **input_changes) if input_changes else self.input + new_output = replace(self.output, **output_changes) if output_changes else self.output + return FilterContext(new_input, new_output, **context_changes) diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index ecb895e013..1a24c19031 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -13,7 +13,7 @@ from pydis_core.utils import scheduling from pydis_core.utils.logging import get_logger -from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filter_context import FilterContext, FilterInput, FilterOutput from bot.exts.filtering._filter_lists.filter_list import ListType, SubscribingAtomicList, UniquesListBase from bot.exts.filtering._filters.antispam import antispam_filter_types from bot.exts.filtering._filters.filter import Filter, UniqueFilter @@ -158,7 +158,7 @@ async def send_alert(self, antispam_list: AntispamList) -> None: return ctx, *other_contexts = self.contexts - new_ctx = FilterContext(ctx.event, ctx.author, ctx.channel, ctx.content, ctx.message) + new_ctx = FilterContext(FilterInput(ctx.event, ctx.author, ctx.channel, ctx.content, ctx.message), FilterOutput()) all_descriptions_counts = Counter(reduce( add, (other_ctx.action_descriptions for other_ctx in other_contexts), ctx.action_descriptions )) diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 3f201cfde4..1aa8a9a393 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -7,6 +7,16 @@ from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._settings import Defaults, create_settings from bot.exts.filtering._utils import FieldRequiring +from dataclasses import dataclass + +import arrow + + +@dataclass +class FilterTimestamps: + """Timestamps for when a filter was created and last updated.""" + created_at: arrow.Arrow + updated_at: arrow.Arrow class Filter(FieldRequiring): @@ -23,12 +33,14 @@ class Filter(FieldRequiring): # If a subclass uses extra fields, it should assign the pydantic model type to this variable. extra_fields_type = None - def __init__(self, filter_data: dict, defaults: Defaults | None = None): + def __init__(self, filter_data: dict, defaults: Defaults | None=None): self.id = filter_data["id"] self.content = filter_data["content"] self.description = filter_data["description"] - self.created_at = arrow.get(filter_data["created_at"]) - self.updated_at = arrow.get(filter_data["updated_at"]) + self.timestamps = FilterTimestamps( + created_at=arrow.get(filter_data["created_at"]), + updated_at=arrow.get(filter_data["updated_at"]) + ) self.actions, self.validations = create_settings(filter_data["settings"], defaults=defaults) if self.extra_fields_type: self.extra_fields = self.extra_fields_type.model_validate(filter_data["additional_settings"]) @@ -75,6 +87,15 @@ async def process_input(cls, content: str, description: str) -> tuple[str, str]: A BadArgument should be raised if the content can't be used. """ return content, description + + + @property + def created_at(self) -> arrow.Arrow: + return self.timestamps.created_at + + @property + def updated_at(self) -> arrow.Arrow: + return self.timestamps.updated_at def __str__(self) -> str: """A string representation of the filter.""" diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index bf19ed414d..08e9a57a9c 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from dataclasses import dataclass from typing import Any import discord @@ -109,6 +110,35 @@ async def on_submit(self, interaction: Interaction) -> None: await self.embed_view.apply_template(self.template.value, self.message, interaction) +@dataclass +class FilterTarget: + """The filter being edited and its context.""" + filter_list: FilterList + list_type: ListType + filter_type: type[Filter] + + +@dataclass +class FilterContent: + """Content and description of the filter being edited.""" + content: str | None + description: str | None + + +def build_type_per_setting_name( + filter_type: type[Filter], + loaded_settings: dict, + loaded_filter_settings: dict, +) -> dict: + """Build the type_per_setting_name dict from the loaded settings.""" + type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} + type_per_setting_name.update({ + f"{filter_type.name}/{name}": type_ + for name, (_, _, type_) in loaded_filter_settings.get(filter_type.name, {}).items() + }) + return type_per_setting_name + + class FilterEditView(EditBaseView): """A view used to edit a filter's settings before updating the database.""" @@ -117,54 +147,42 @@ class _REMOVE: def __init__( self, - filter_list: FilterList, - list_type: ListType, - filter_type: type[Filter], - content: str | None, - description: str | None, + filter_target: FilterTarget, + filter_content: FilterContent, settings_overrides: dict, filter_settings_overrides: dict, - loaded_settings: dict, - loaded_filter_settings: dict, + type_per_setting_name: dict, author: User, embed: Embed, confirm_callback: Callable ): super().__init__(author) - self.filter_list = filter_list - self.list_type = list_type - self.filter_type = filter_type - self.content = content - self.description = description + self.filter_target = filter_target + self.filter_content = filter_content self.settings_overrides = settings_overrides self.filter_settings_overrides = filter_settings_overrides - self.loaded_settings = loaded_settings - self.loaded_filter_settings = loaded_filter_settings + self.type_per_setting_name = type_per_setting_name self.embed = embed self.confirm_callback = confirm_callback all_settings_repr_dict = build_filter_repr_dict( - filter_list, list_type, filter_type, settings_overrides, filter_settings_overrides + filter_target.filter_list, filter_target.list_type, filter_target.filter_type, + settings_overrides, filter_settings_overrides ) populate_embed_from_dict(embed, all_settings_repr_dict) - self.type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} - self.type_per_setting_name.update({ - f"{filter_type.name}/{name}": type_ - for name, (_, _, type_) in loaded_filter_settings.get(filter_type.name, {}).items() - }) - add_select = CustomCallbackSelect( self._prompt_new_value, placeholder="Select a setting to edit", - options=[SelectOption(label=name) for name in sorted(self.type_per_setting_name)], + options=[SelectOption(label=name) for name in sorted(type_per_setting_name)], row=1 ) self.add_item(add_select) if settings_overrides or filter_settings_overrides: override_names = ( - list(settings_overrides) + [f"{filter_list.name}/{setting}" for setting in filter_settings_overrides] + list(settings_overrides) + + [f"{filter_target.filter_list.name}/{setting}" for setting in filter_settings_overrides] ) remove_select = CustomCallbackSelect( self._remove_override, @@ -200,21 +218,21 @@ async def enter_template(self, interaction: Interaction, button: discord.ui.Butt @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green, row=4) async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: """Confirm the content, description, and settings, and update the filters database.""" - if self.content is None: + if self.filter_content.content is None: await interaction.response.send_message( ":x: Cannot add a filter with no content.", ephemeral=True, reference=interaction.message ) - if self.description is None: - self.description = "" + if self.filter_content.description is None: + self.filter_content.description = "" await interaction.response.edit_message(view=None) # Make sure the interaction succeeds first. try: await self.confirm_callback( interaction.message, - self.filter_list, - self.list_type, - self.filter_type, - self.content, - self.description, + self.filter_target.filter_list, + self.filter_target.list_type, + self.filter_target.filter_type, + self.filter_content.content, + self.filter_content.description, self.settings_overrides, self.filter_settings_overrides ) @@ -262,7 +280,7 @@ async def update_embed( """ if content is not None or description is not None: if content is not None: - filter_type = self.filter_list.get_filter_type(content) + filter_type = self.filter_target.filter_list.get_filter_type(content) if not filter_type: if isinstance(interaction_or_msg, discord.Message): send_method = interaction_or_msg.channel.send @@ -270,16 +288,16 @@ async def update_embed( send_method = interaction_or_msg.response.send_message await send_method(f":x: Could not find a filter type appropriate for `{content}`.") return - self.content = content - self.filter_type = filter_type + self.filter_content.content = content + self.filter_target.filter_type = filter_type else: - content = self.content # If there's no content or description, use the existing values. + content = self.filter_content.content # If there's no content or description, use the existing values. if description is self._REMOVE: - self.description = None + self.filter_content.description = None elif description is not None: - self.description = description + self.filter_content.description = description else: - description = self.description + description = self.filter_content.description # Update the embed with the new content and/or description. self.embed.description = f"`{content}`" if content else "*No content*" @@ -293,10 +311,10 @@ async def update_embed( if "/" in setting_name: _filter_name, setting_name = setting_name.split("/", maxsplit=1) dict_to_edit = self.filter_settings_overrides - default_value = self.filter_type.extra_fields_type().model_dump()[setting_name] + default_value = self.filter_target.filter_type.extra_fields_type().model_dump()[setting_name] else: dict_to_edit = self.settings_overrides - default_value = self.filter_list[self.list_type].default(setting_name) + default_value = self.filter_target.filter_list[self.filter_target.list_type].default(setting_name) # Update the setting override value or remove it if setting_value is not self._REMOVE: if not repr_equals(setting_value, default_value): @@ -334,7 +352,7 @@ async def apply_template(self, template_id: str, embed_message: discord.Message, """Replace any non-overridden settings with overrides from the given filter.""" try: settings, filter_settings = template_settings( - template_id, self.filter_list, self.list_type, self.filter_type + template_id, self.filter_target.filter_list, self.filter_target.list_type, self.filter_target.filter_type ) except BadArgument as e: # The interaction object is necessary to send an ephemeral message. await interaction.response.send_message(f":x: {e}", ephemeral=True) @@ -359,15 +377,18 @@ async def _remove_override(self, interaction: Interaction, select: discord.ui.Se def copy(self) -> FilterEditView: """Create a copy of this view.""" return FilterEditView( - self.filter_list, - self.list_type, - self.filter_type, - self.content, - self.description, + FilterTarget( + self.filter_target.filter_list, + self.filter_target.list_type, + self.filter_target.filter_type, + ), + FilterContent( + self.filter_content.content, + self.filter_content.description, + ), self.settings_overrides, self.filter_settings_overrides, - self.loaded_settings, - self.loaded_filter_settings, + self.type_per_setting_name, self.author, self.embed, self.confirm_callback diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index c652098e17..0869152ddf 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -87,8 +87,8 @@ def __init__( self.embed = embed self.confirm_callback = confirm_callback - self.settings_repr_dict = {name: to_serializable(value) for name, value in settings.items()} - populate_embed_from_dict(embed, self.settings_repr_dict) + settings_repr_dict = {name: to_serializable(value) for name, value in settings.items()} + populate_embed_from_dict(embed, settings_repr_dict) self.type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} @@ -188,8 +188,8 @@ def __init__( self.embed = embed self.confirm_callback = confirm_callback - self.settings_repr_dict = build_filterlist_repr_dict(filter_list, list_type, new_settings) - populate_embed_from_dict(embed, self.settings_repr_dict) + settings_repr_dict = build_filterlist_repr_dict(filter_list, list_type, new_settings) + populate_embed_from_dict(embed, settings_repr_dict) self.type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} @@ -220,11 +220,11 @@ async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> N self.stop() def current_value(self, setting_name: str) -> Any: - """Get the current value stored for the setting or MISSING if none found.""" if setting_name in self.settings: return self.settings[setting_name] - if setting_name in self.settings_repr_dict: - return self.settings_repr_dict[setting_name] + settings_repr_dict = build_filterlist_repr_dict(self.filter_list, self.list_type, self.settings) + if setting_name in settings_repr_dict: + return settings_repr_dict[setting_name] return MISSING async def update_embed( diff --git a/bot/exts/filtering/_ui/search.py b/bot/exts/filtering/_ui/search.py index 352db00326..f407c7fcb8 100644 --- a/bot/exts/filtering/_ui/search.py +++ b/bot/exts/filtering/_ui/search.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from dataclasses import dataclass from typing import Any import discord @@ -20,6 +21,15 @@ ) +@dataclass +class FilterResources: + """Container for filter system resources passed to search views.""" + filter_lists: dict[str, FilterList] + filters: dict[str, type[Filter]] + settings: dict[str, tuple[str, SettingsEntry, type]] + filter_settings: dict[str, dict[str, tuple[str, SettingsEntry, type]]] + + def search_criteria_converter( filter_lists: dict, loaded_filters: dict, @@ -144,10 +154,7 @@ def __init__( filter_type: type[Filter] | None, settings: dict[str, Any], filter_settings: dict[str, Any], - loaded_filter_lists: dict[str, FilterList], - loaded_filters: dict[str, type[Filter]], - loaded_settings: dict[str, tuple[str, SettingsEntry, type]], - loaded_filter_settings: dict[str, dict[str, tuple[str, SettingsEntry, type]]], + filter_resources: FilterResources, author: discord.User | discord.Member, embed: discord.Embed, confirm_callback: Callable @@ -156,10 +163,7 @@ def __init__( self.filter_type = filter_type self.settings = settings self.filter_settings = filter_settings - self.loaded_filter_lists = loaded_filter_lists - self.loaded_filters = loaded_filters - self.loaded_settings = loaded_settings - self.loaded_filter_settings = loaded_filter_settings + self.filter_resources = filter_resources self.embed = embed self.confirm_callback = confirm_callback @@ -171,11 +175,11 @@ def __init__( settings_repr_dict = build_search_repr_dict(settings, filter_settings, filter_type) populate_embed_from_dict(embed, settings_repr_dict) - self.type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} + self.type_per_setting_name = {setting: info[2] for setting, info in filter_resources.settings.items()} if filter_type: self.type_per_setting_name.update({ f"{filter_type.name}/{name}": type_ - for name, (_, _, type_) in loaded_filter_settings.get(filter_type.name, {}).items() + for name, (_, _, type_) in filter_resources.filter_settings.get(filter_type.name, {}).items() }) add_select = CustomCallbackSelect( @@ -290,7 +294,7 @@ async def apply_template(self, template_id: str, embed_message: discord.Message, """Set any unset criteria with settings values from the given filter.""" try: settings, filter_settings, self.filter_type = template_settings( - template_id, self.loaded_filter_lists, self.filter_type + template_id, self.filter_resources.filter_lists, self.filter_type ) except BadArgument as e: # The interaction object is necessary to send an ephemeral message. await interaction.response.send_message(f":x: {e}", ephemeral=True) @@ -306,8 +310,8 @@ async def apply_template(self, template_id: str, embed_message: discord.Message, async def apply_filter_type(self, type_name: str, embed_message: discord.Message, interaction: Interaction) -> None: """Set a new filter type and reset any criteria for settings of the old filter type.""" - if type_name.lower() not in self.loaded_filters: - if type_name.lower()[:-1] not in self.loaded_filters: # In case the user entered the plural form. + if type_name.lower() not in self.filter_resources.filters: + if type_name.lower()[:-1] not in self.filter_resources.filters: # In case the user entered the plural form. await interaction.response.send_message(f":x: No such filter type {type_name!r}.", ephemeral=True) return type_name = type_name[:-1] @@ -316,7 +320,7 @@ async def apply_filter_type(self, type_name: str, embed_message: discord.Message if self.filter_type and type_name == self.filter_type.name: return - self.filter_type = self.loaded_filters[type_name] + self.filter_type = self.filter_resources.filters[type_name] self.filter_settings = {} self.embed.clear_fields() await embed_message.edit(embed=self.embed, view=self.copy()) @@ -328,10 +332,7 @@ def copy(self) -> SearchEditView: self.filter_type, self.settings, self.filter_settings, - self.loaded_filter_lists, - self.loaded_filters, - self.loaded_settings, - self.loaded_filter_settings, + self.filter_resources, self.author, self.embed, self.confirm_callback diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 210ae3fb05..80b244bcec 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -26,7 +26,7 @@ from bot.bot import Bot from bot.constants import BaseURLs, Channels, Guild, MODERATION_ROLES, Roles from bot.exts.backend.branding._repository import HEADERS, PARAMS -from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_context import Event, FilterContext, FilterInput, FilterOutput from bot.exts.filtering._filter_lists import FilterList, ListType, ListTypeConverter, filter_list_types from bot.exts.filtering._filter_lists.filter_list import AtomicList from bot.exts.filtering._filters.filter import Filter, UniqueFilter @@ -39,7 +39,7 @@ populate_embed_from_dict, ) from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter -from bot.exts.filtering._ui.search import SearchEditView, search_criteria_converter +from bot.exts.filtering._ui.search import FilterResources, SearchEditView, search_criteria_converter from bot.exts.filtering._ui.ui import ( AlertView, ArgumentCompletionView, @@ -55,6 +55,7 @@ from bot.utils.channel import is_mod_channel from bot.utils.lock import lock_arg from bot.utils.message_cache import MessageCache +from dataclasses import dataclass, field log = get_logger(__name__) @@ -75,6 +76,13 @@ async def _extract_text_file_content(att: discord.Attachment) -> str: return f"{att.filename}: {first_n_lines}" +@dataclass +class LoadedFilterData: + settings: dict = field(default_factory=dict) + filters: dict = field(default_factory=dict) + filter_settings: dict = field(default_factory=dict) + + class Filtering(Cog): """Filtering and alerting for content posted on the server.""" @@ -93,9 +101,7 @@ def __init__(self, bot: Bot): self.delete_scheduler = scheduling.Scheduler(self.__class__.__name__) self.webhook: discord.Webhook | None = None - self.loaded_settings = {} - self.loaded_filters = {} - self.loaded_filter_settings = {} + self.loaded_data = LoadedFilterData() self.message_cache = MessageCache(CACHE_SIZE, newest_first=True) @@ -159,7 +165,7 @@ def collect_loaded_types(self, example_list: AtomicList) -> None: """ # Get the filter types used by each filter list. for filter_list in self.filter_lists.values(): - self.loaded_filters.update({filter_type.name: filter_type for filter_type in filter_list.filter_types}) + self.loaded_data.filters.update({filter_type.name: filter_type for filter_type in filter_list.filter_types}) # Get the setting types used by each filter list. if self.filter_lists: @@ -174,24 +180,24 @@ def collect_loaded_types(self, example_list: AtomicList) -> None: if isinstance(setting_entry.description, str): # If it's a string, then the settings entry matches a single field in the DB, # and its name is the setting type's name attribute. - self.loaded_settings[setting_entry.name] = ( + self.loaded_data.settings[setting_entry.name] = ( setting_entry.description, setting_entry, type_hints[setting_entry.name] ) else: # Otherwise, the setting entry works with compound settings. - self.loaded_settings.update({ + self.loaded_data.settings.update({ subsetting: (description, setting_entry, type_hints[subsetting]) for subsetting, description in setting_entry.description.items() }) # Get the settings per filter as well. - for filter_name, filter_type in self.loaded_filters.items(): + for filter_name, filter_type in self.loaded_data.filters.items(): extra_fields_type = filter_type.extra_fields_type if not extra_fields_type: continue type_hints = get_type_hints(extra_fields_type) # A class var with a `_description` suffix is expected per field name. - self.loaded_filter_settings[filter_name] = { + self.loaded_data.filter_settings[filter_name] = { field_name: ( getattr(extra_fields_type, f"{field_name}_description", ""), extra_fields_type, @@ -285,13 +291,13 @@ async def on_message_edit(self, before: discord.Message, after: discord.Message) @Cog.listener() async def on_voice_state_update(self, member: discord.Member, *_) -> None: """Checks for bad words in usernames when users join, switch or leave a voice channel.""" - ctx = FilterContext(Event.NICKNAME, member, None, member.display_name, None) + ctx = FilterContext(FilterInput(Event.NICKNAME, member, None, member.display_name, None), FilterOutput()) await self._check_bad_display_name(ctx) @Cog.listener() async def on_thread_create(self, thread: Thread) -> None: """Check for bad words in new thread names.""" - ctx = FilterContext(Event.THREAD_NAME, thread.owner, thread, thread.name, None) + ctx = FilterContext(FilterInput(Event.THREAD_NAME, thread.owner, thread, thread.name, None), FilterOutput()) await self._check_bad_name(ctx) async def filter_snekbox_output( @@ -329,7 +335,7 @@ async def blocklist(self, ctx: Context) -> None: await ctx.send_help(ctx.command) @blocklist.command(name="list", aliases=("get",)) - async def bl_list(self, ctx: Context, list_name: str | None = None) -> None: + async def bl_list(self, ctx: Context, list_name: str | None=None) -> None: """List the contents of a specified blacklist.""" result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name, exclude="list_type") if not result: @@ -345,7 +351,7 @@ async def bl_add( list_name: str | None, content: str, *, - description_and_settings: str | None = None + description_and_settings: str | None=None ) -> None: """ Add a blocked filter to the specified filter list. @@ -372,7 +378,7 @@ async def allowlist(self, ctx: Context) -> None: await ctx.send_help(ctx.command) @allowlist.command(name="list", aliases=("get",)) - async def al_list(self, ctx: Context, list_name: str | None = None) -> None: + async def al_list(self, ctx: Context, list_name: str | None=None) -> None: """List the contents of a specified whitelist.""" result = await self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name, exclude="list_type") if not result: @@ -388,7 +394,7 @@ async def al_add( list_name: str | None, content: str, *, - description_and_settings: str | None = None + description_and_settings: str | None=None ) -> None: """ Add an allowed filter to the specified filter list. @@ -409,7 +415,7 @@ async def al_add( # region: filter commands @commands.group(aliases=("filters", "f"), invoke_without_command=True) - async def filter(self, ctx: Context, id_: int | None = None) -> None: + async def filter(self, ctx: Context, id_: int | None=None) -> None: """ Group for managing filters. @@ -447,8 +453,8 @@ async def filter(self, ctx: Context, id_: int | None = None) -> None: async def f_list( self, ctx: Context, - list_type: ListTypeConverter | None = None, - list_name: str | None = None, + list_type: ListTypeConverter | None=None, + list_name: str | None=None, ) -> None: """List the contents of a specified list of filters.""" result = await self._resolve_list_type_and_name(ctx, list_type, list_name) @@ -462,14 +468,14 @@ async def f_list( async def f_describe(self, ctx: Context, filter_name: str | None) -> None: """Show a description of the specified filter, or a list of possible values if no name is specified.""" if not filter_name: - filter_names = [f"» {f}" for f in self.loaded_filters] + filter_names = [f"» {f}" for f in self.loaded_data.filters] embed = Embed(colour=Colour.blue()) embed.set_author(name="List of filter names") await LinePaginator.paginate(filter_names, ctx, embed, max_lines=10, empty=False) else: - filter_type = self.loaded_filters.get(filter_name) + filter_type = self.loaded_data.filters.get(filter_name) if not filter_type: - filter_type = self.loaded_filters.get(filter_name[:-1]) # A plural form or a typo. + filter_type = self.loaded_data.filters.get(filter_name[:-1]) # A plural form or a typo. if not filter_type: await ctx.send(f":x: There's no filter type named {filter_name!r}.") return @@ -487,7 +493,7 @@ async def f_add( list_name: str | None, content: str, *, - description_and_settings: str | None = None + description_and_settings: str | None=None ) -> None: """ Add a filter to the specified filter list. @@ -516,7 +522,7 @@ async def f_edit( noui: Literal["noui"] | None, filter_id: int, *, - description_and_settings: str | None = None + description_and_settings: str | None=None ) -> None: """ Edit a filter specified by its ID. @@ -542,8 +548,8 @@ async def f_edit( description, new_settings, new_filter_settings = description_and_settings_converter( filter_list, list_type, filter_type, - self.loaded_settings, - self.loaded_filter_settings, + self.loaded_data.settings, + self.loaded_data.filter_settings, description_and_settings ) @@ -574,16 +580,17 @@ async def f_edit( f"run `{constants.Bot.prefix}filterlist describe {list_type.name} {filter_list.name}`." )) + filter_target = filters_ui.FilterTarget(filter_list, list_type, filter_type) + filter_content = filters_ui.FilterContent(content, description) + type_per_setting_name = filters_ui.build_type_per_setting_name( + filter_type, self.loaded_data.settings, self.loaded_data.filter_settings + ) view = filters_ui.FilterEditView( - filter_list, - list_type, - filter_type, - content, - description, + filter_target, + filter_content, settings, filter_settings, - self.loaded_settings, - self.loaded_filter_settings, + type_per_setting_name, ctx.author, embed, patch_func @@ -593,6 +600,7 @@ async def f_edit( @filter.command(name="delete", aliases=("d", "remove")) async def f_delete(self, ctx: Context, filter_id: int) -> None: """Delete the filter specified by its ID.""" + async def delete_list() -> None: """The actual removal routine.""" await bot.instance.api_client.delete(f"bot/filter/filters/{filter_id}") @@ -614,8 +622,8 @@ async def delete_list() -> None: async def setting(self, ctx: Context, setting_name: str | None) -> None: """Show a description of the specified setting, or a list of possible settings if no name is specified.""" if not setting_name: - settings_list = [f"» {setting_name}" for setting_name in self.loaded_settings] - for filter_name, filter_settings in self.loaded_filter_settings.items(): + settings_list = [f"» {setting_name}" for setting_name in self.loaded_data.settings] + for filter_name, filter_settings in self.loaded_data.filter_settings.items(): settings_list.extend(f"» {filter_name}/{setting}" for setting in filter_settings) embed = Embed(colour=Colour.blue()) embed.set_author(name="List of setting names") @@ -623,15 +631,15 @@ async def setting(self, ctx: Context, setting_name: str | None) -> None: else: # The setting is either in a SettingsEntry subclass, or a pydantic model. - setting_data = self.loaded_settings.get(setting_name) + setting_data = self.loaded_data.settings.get(setting_name) description = None if setting_data: description = setting_data[0] elif "/" in setting_name: # It's a filter specific setting. filter_name, filter_setting_name = setting_name.split("/", maxsplit=1) - if filter_name in self.loaded_filter_settings: - if filter_setting_name in self.loaded_filter_settings[filter_name]: - description = self.loaded_filter_settings[filter_name][filter_setting_name][0] + if filter_name in self.loaded_data.filter_settings: + if filter_setting_name in self.loaded_data.filter_settings[filter_name]: + description = self.loaded_data.filter_settings[filter_name][filter_setting_name][0] if description is None: await ctx.send(f":x: There's no setting type named {setting_name!r}.") return @@ -656,10 +664,10 @@ async def f_match( raise BadArgument("Please provide input.") if message: user = None if no_user else message.author - filter_ctx = FilterContext(Event.MESSAGE, user, message.channel, message.content, message, message.embeds) + filter_ctx = FilterContext(FilterInput(Event.MESSAGE, user, message.channel, message.content, message, message.embeds), FilterOutput()) else: python_general = ctx.guild.get_channel(Channels.python_general) - filter_ctx = FilterContext(Event.MESSAGE, None, python_general, string, None) + filter_ctx = FilterContext(FilterInput(Event.MESSAGE, None, python_general, string, None), FilterOutput()) _, _, triggers = await self._resolve_action(filter_ctx) lines = [] @@ -679,7 +687,7 @@ async def f_search( noui: Literal["noui"] | None, filter_type_name: str | None, *, - settings: str = "" + settings: str="" ) -> None: """ Find filters with the provided settings. The format is identical to that of the add and edit commands. @@ -690,9 +698,9 @@ async def f_search( filter_type = None if filter_type_name: filter_type_name = filter_type_name.lower() - filter_type = self.loaded_filters.get(filter_type_name) + filter_type = self.loaded_data.filters.get(filter_type_name) if not filter_type: - self.loaded_filters.get(filter_type_name[:-1]) # In case the user tried to specify the plural form. + self.loaded_data.filters.get(filter_type_name[:-1]) # In case the user tried to specify the plural form. # If settings were provided with no filter_type, discord.py will capture the first word as the filter type. if filter_type is None and filter_type_name is not None: if settings: @@ -703,9 +711,9 @@ async def f_search( settings, filter_settings, filter_type = search_criteria_converter( self.filter_lists, - self.loaded_filters, - self.loaded_settings, - self.loaded_filter_settings, + self.loaded_data.filters, + self.loaded_data.settings, + self.loaded_data.filter_settings, filter_type, settings ) @@ -715,14 +723,17 @@ async def f_search( return embed = Embed(colour=Colour.blue()) + filter_resources = FilterResources( + filter_lists=self.filter_lists, + filters=self.loaded_data.filters, + settings=self.loaded_data.settings, + filter_settings=self.loaded_data.filter_settings, + ) view = SearchEditView( filter_type, settings, filter_settings, - self.filter_lists, - self.loaded_filters, - self.loaded_settings, - self.loaded_filter_settings, + filter_resources, ctx.author, embed, self._search_filters @@ -731,7 +742,7 @@ async def f_search( @filter.command(root_aliases=("compfilter", "compf")) async def compadd( - self, ctx: Context, list_name: str | None, content: str, *, description: str | None = "Phishing" + self, ctx: Context, list_name: str | None, content: str, *, description: str | None="Phishing" ) -> None: """Add a filter to detect a compromised account. Will apply the equivalent of a compban if triggered.""" result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name, exclude="list_type") @@ -762,7 +773,7 @@ async def filterlist(self, ctx: Context) -> None: @filterlist.command(name="describe", aliases=("explain", "manual", "id")) async def fl_describe( - self, ctx: Context, list_type: ListTypeConverter | None = None, list_name: str | None = None + self, ctx: Context, list_type: ListTypeConverter | None=None, list_name: str | None=None ) -> None: """Show a description of the specified filter list, or a list of possible values if no values are provided.""" if not list_type and not list_name: @@ -812,13 +823,13 @@ async def fl_add(self, ctx: Context, list_type: ListTypeConverter, list_name: st embed = Embed(colour=Colour.blue()) embed.set_author(name=f"New Filter List - {list_description.title()}") - settings = {name: starting_value(value[2]) for name, value in self.loaded_settings.items()} + settings = {name: starting_value(value[2]) for name, value in self.loaded_data.settings.items()} view = FilterListAddView( list_name, list_type, settings, - self.loaded_settings, + self.loaded_data.settings, ctx.author, embed, self._post_filter_list @@ -831,8 +842,8 @@ async def fl_edit( self, ctx: Context, noui: Literal["noui"] | None, - list_type: ListTypeConverter | None = None, - list_name: str | None = None, + list_type: ListTypeConverter | None=None, + list_name: str | None=None, *, settings: str | None ) -> None: @@ -848,7 +859,7 @@ async def fl_edit( if result is None: return list_type, filter_list = result - settings = settings_converter(self.loaded_settings, settings) + settings = settings_converter(self.loaded_data.settings, settings) if noui: try: await self._patch_filter_list(ctx.message, filter_list, list_type, settings) @@ -864,7 +875,7 @@ async def fl_edit( filter_list, list_type, settings, - self.loaded_settings, + self.loaded_data.settings, ctx.author, embed, self._patch_filter_list @@ -874,9 +885,10 @@ async def fl_edit( @filterlist.command(name="delete", aliases=("remove",)) @has_any_role(Roles.admins) async def fl_delete( - self, ctx: Context, list_type: ListTypeConverter | None = None, list_name: str | None = None + self, ctx: Context, list_type: ListTypeConverter | None=None, list_name: str | None=None ) -> None: """Remove the filter list and all of its filters from the database.""" + async def delete_list() -> None: """The actual removal routine.""" list_data = await bot.instance.api_client.get(f"bot/filter/filter_lists/{list_id}") @@ -1044,7 +1056,7 @@ async def _check_bad_name(self, ctx: FilterContext) -> FilterContext: return new_ctx async def _resolve_list_type_and_name( - self, ctx: Context, list_type: ListType | None = None, list_name: str | None = None, *, exclude: str = "" + self, ctx: Context, list_type: ListType | None=None, list_name: str | None=None, *, exclude: str="" ) -> tuple[ListType, FilterList] | None: """Prompt the user to complete the list type or list name if one of them is missing.""" if list_name is None: @@ -1111,7 +1123,7 @@ async def _add_filter( list_type: ListType, filter_list: FilterList, content: str, - description_and_settings: str | None = None + description_and_settings: str | None=None ) -> None: """Add a filter to the database.""" # Validations. @@ -1127,8 +1139,8 @@ async def _add_filter( filter_list, list_type, filter_type, - self.loaded_settings, - self.loaded_filter_settings, + self.loaded_data.settings, + self.loaded_data.filter_settings, description_and_settings ) @@ -1155,16 +1167,17 @@ async def _add_filter( f"run `{constants.Bot.prefix}filterlist describe {list_type.name} {filter_list.name}`." )) + filter_target = filters_ui.FilterTarget(filter_list, list_type, filter_type) + filter_content = filters_ui.FilterContent(content, description) + type_per_setting_name = filters_ui.build_type_per_setting_name( + filter_type, self.loaded_data.settings, self.loaded_data.filter_settings + ) view = filters_ui.FilterEditView( - filter_list, - list_type, - filter_type, - content, - description, + filter_target, + filter_content, settings, filter_settings, - self.loaded_settings, - self.loaded_filter_settings, + type_per_setting_name, ctx.author, embed, self._post_new_filter @@ -1189,7 +1202,7 @@ def _identical_filters_message(content: str, filter_list: FilterList, list_type: @staticmethod async def _maybe_alert_auto_infraction( - filter_list: FilterList, list_type: ListType, filter_: Filter, old_filter: Filter | None = None + filter_list: FilterList, list_type: ListType, filter_: Filter, old_filter: Filter | None=None ) -> None: """If the filter is new and applies an auto-infraction, or was edited to apply a different one, log it.""" infraction_type = filter_.overrides[0].get("infraction_type") @@ -1439,7 +1452,7 @@ async def weekly_auto_infraction_report_task(self) -> None: async def send_weekly_auto_infraction_report( self, - channel: discord.TextChannel | discord.Thread | None = None, + channel: discord.TextChannel | discord.Thread | None=None, ) -> None: """ Send a list of auto-infractions added in the last 7 days to the specified channel. @@ -1474,7 +1487,7 @@ async def send_weekly_auto_infraction_report( # Nicely format the output so each filter list type is grouped lines = [f"**Auto-infraction filters added since {seven_days_ago.format('YYYY-MM-DD')}**"] for list_label, filters in found_filters.items(): - lines.append("\n".join([f"**{list_label.title()}**"]+[f"{filter_} ({infr})" for filter_, infr in filters])) + lines.append("\n".join([f"**{list_label.title()}**"] + [f"{filter_} ({infr})" for filter_, infr in filters])) if len(lines) == 1: lines.append("Nothing to show") diff --git a/bot/exts/info/doc/_cog.py b/bot/exts/info/doc/_cog.py index 4546fc14f3..4b9e8163e4 100644 --- a/bot/exts/info/doc/_cog.py +++ b/bot/exts/info/doc/_cog.py @@ -51,14 +51,13 @@ class DocCog(commands.Cog): """A set of commands for querying & displaying documentation.""" def __init__(self, bot: Bot): - # Contains URLs to documentation home pages. - # Used to calculate inventory diffs on refreshes and to display all currently stored inventories. - self.base_urls = {} self.bot = bot - self.doc_symbols: dict[str, DocItem] = {} # Maps symbol names to objects containing their metadata. - self.item_fetcher = _batch_parser.BatchParser() - # Maps a conflicting symbol name to a list of the new, disambiguated names created from conflicts with the name. - self.renamed_symbols = defaultdict(list) + self._inventory = SimpleNamespace( + base_urls={}, + doc_symbols={}, + renamed_symbols=defaultdict(list), + item_fetcher=_batch_parser.BatchParser(), + ) self.inventory_scheduler = Scheduler(self.__class__.__name__) @@ -81,7 +80,7 @@ def update_single(self, package_name: str, base_url: str, inventory: InventoryDi absolute paths that link to specific symbols * `package` is the content of a intersphinx inventory. """ - self.base_urls[package_name] = base_url + self._inventory.base_urls[package_name] = base_url for group, items in inventory.items(): for symbol_name, relative_doc_url in items: @@ -105,8 +104,8 @@ def update_single(self, package_name: str, base_url: str, inventory: InventoryDi sys.intern(relative_url_path), symbol_id, ) - self.doc_symbols[symbol_name] = doc_item - self.item_fetcher.add_item(doc_item) + self._inventory.doc_symbols[symbol_name] = doc_item + self._inventory.item_fetcher.add_item(doc_item) log.trace(f"Fetched inventory for {package_name}.") @@ -155,23 +154,23 @@ def ensure_unique_symbol_name(self, package_name: str, group_name: str, symbol_n If the existing symbol was renamed or there was no conflict, the returned name is equivalent to `symbol_name`. """ - if (item := self.doc_symbols.get(symbol_name)) is None: + if (item := self._inventory.doc_symbols.get(symbol_name)) is None: return symbol_name # There's no conflict so it's fine to simply use the given symbol name. def rename(prefix: str, *, rename_extant: bool = False) -> str: new_name = f"{prefix}.{symbol_name}" - if new_name in self.doc_symbols: + if new_name in self._inventory.doc_symbols: # If there's still a conflict, qualify the name further. if rename_extant: new_name = f"{item.package}.{item.group}.{symbol_name}" else: new_name = f"{package_name}.{group_name}.{symbol_name}" - self.renamed_symbols[symbol_name].append(new_name) + self._inventory.renamed_symbols[symbol_name].append(new_name) if rename_extant: # Instead of renaming the current symbol, rename the symbol with which it conflicts. - self.doc_symbols[new_name] = self.doc_symbols[symbol_name] + self._inventory.doc_symbols[new_name] = self._inventory.doc_symbols[symbol_name] return symbol_name return new_name @@ -201,10 +200,10 @@ async def refresh_inventories(self) -> None: log.debug("Refreshing documentation inventory...") self.inventory_scheduler.cancel_all() - self.base_urls.clear() - self.doc_symbols.clear() - self.renamed_symbols.clear() - await self.item_fetcher.clear() + self._inventory.base_urls.clear() + self._inventory.doc_symbols.clear() + self._inventory.renamed_symbols.clear() + await self._inventory.item_fetcher.clear() coros = [ self.update_or_reschedule_inventory( @@ -222,10 +221,10 @@ def get_symbol_item(self, symbol_name: str) -> tuple[str, DocItem | None]: If the doc item is not found directly from the passed in name and the name contains a space, the first word of the name will be attempted to be used to get the item. """ - doc_item = self.doc_symbols.get(symbol_name) + doc_item = self._inventory.doc_symbols.get(symbol_name) if doc_item is None and " " in symbol_name: symbol_name = symbol_name.split(maxsplit=1)[0] - doc_item = self.doc_symbols.get(symbol_name) + doc_item = self._inventory.doc_symbols.get(symbol_name) return symbol_name, doc_item @@ -241,7 +240,7 @@ async def get_symbol_markdown(self, doc_item: DocItem) -> str: if markdown is None: log.debug(f"Redis cache miss with {doc_item}.") try: - markdown = await self.item_fetcher.get_markdown(doc_item) + markdown = await self._inventory.item_fetcher.get_markdown(doc_item) except aiohttp.ClientError as e: log.warning(f"A network error has occurred when requesting parsing of {doc_item}.", exc_info=e) @@ -278,8 +277,8 @@ async def create_symbol_embed(self, symbol_name: str) -> discord.Embed | None: # Show all symbols with the same name that were renamed in the footer, # with a max of 200 chars. - if symbol_name in self.renamed_symbols: - renamed_symbols = ", ".join(self.renamed_symbols[symbol_name]) + if symbol_name in self._inventory.renamed_symbols: + renamed_symbols = ", ".join(self._inventory.renamed_symbols[symbol_name]) footer_text = textwrap.shorten("Similar names: " + renamed_symbols, 200, placeholder=" ...") else: footer_text = "" @@ -312,12 +311,12 @@ async def get_command(self, ctx: commands.Context, *, symbol_name: str | None) - """ if not symbol_name: inventory_embed = discord.Embed( - title=f"All inventories (`{len(self.base_urls)}` total)", + title=f"All inventories (`{len(self._inventory.base_urls)}` total)", colour=discord.Colour.blue() ) - lines = sorted(f"- [`{name}`]({url})" for name, url in self.base_urls.items()) - if self.base_urls: + lines = sorted(f"- [`{name}`]({url})" for name, url in self._inventory.base_urls.items()) + if self._inventory.base_urls: await LinePaginator.paginate(lines, ctx, inventory_embed, max_size=400, empty=False) else: @@ -418,10 +417,10 @@ async def delete_command(self, ctx: commands.Context, package_name: PackageName) @lock(NAMESPACE, COMMAND_LOCK_SINGLETON, raise_error=True) async def refresh_command(self, ctx: commands.Context) -> None: """Refresh inventories and show the difference.""" - old_inventories = set(self.base_urls) + old_inventories = set(self._inventory.base_urls) async with ctx.typing(): await self.refresh_inventories() - new_inventories = set(self.base_urls) + new_inventories = set(self._inventory.base_urls) if added := ", ".join(new_inventories - old_inventories): added = "+ " + added @@ -444,7 +443,7 @@ async def clear_cache_command( ) -> None: """Clear the persistent redis cache for `package`.""" if await doc_cache.delete(package_name): - await self.item_fetcher.stale_inventory_notifier.symbol_counter.delete(package_name) + await self._inventory.item_fetcher.stale_inventory_notifier.symbol_counter.delete(package_name) await ctx.send(f"Successfully cleared the cache for `{package_name}`.") else: await ctx.send("No keys matching the package found.") @@ -452,4 +451,4 @@ async def clear_cache_command( async def cog_unload(self) -> None: """Clear scheduled inventories, queued symbols and cleanup task on cog unload.""" self.inventory_scheduler.cancel_all() - await self.item_fetcher.clear() + await self._inventory.item_fetcher.clear() diff --git a/bot/exts/moderation/watchchannels/_watchchannel.py b/bot/exts/moderation/watchchannels/_watchchannel.py index 44c0be2a7e..7f5d8a43e5 100644 --- a/bot/exts/moderation/watchchannels/_watchchannel.py +++ b/bot/exts/moderation/watchchannels/_watchchannel.py @@ -38,6 +38,37 @@ class MessageHistory: message_count: int = 0 +@dataclass +class WatchChannelConfig: + """Configuration for a watch channel.""" + + bot: Bot + destination: int + webhook_id: int + api_endpoint: str + api_default_params: dict + logger: CustomLogger + disable_header: bool = False + + +@dataclass +class MessageQueueState: + """State for the message consumption queue.""" + + consume_task: asyncio.Task | None = None + message_queue: defaultdict | None = None + consumption_queue: dict | None = None + message_history: MessageHistory | None = None + + def __post_init__(self) -> None: + if self.message_queue is None: + self.message_queue = defaultdict(lambda: defaultdict(deque)) + if self.consumption_queue is None: + self.consumption_queue = {} + if self.message_history is None: + self.message_history = MessageHistory() + + class WatchChannel(metaclass=CogABCMeta): """ABC with functionality for relaying users' messages to a certain channel.""" @@ -53,33 +84,38 @@ def __init__( *, disable_header: bool = False ) -> None: - self.bot = bot - - self.destination = destination # E.g., Channels.big_brother - self.webhook_id = webhook_id # E.g., Webhooks.big_brother - self.api_endpoint = api_endpoint # E.g., 'bot/infractions' - self.api_default_params = api_default_params # E.g., {'active': 'true', 'type': 'watch'} - self.log = logger # Logger of the child cog for a correct name in the logs - - self._consume_task = None + self.config = WatchChannelConfig( + bot=bot, + destination=destination, + webhook_id=webhook_id, + api_endpoint=api_endpoint, + api_default_params=api_default_params, + logger=logger, + disable_header=disable_header, + ) + self.queue_state = MessageQueueState() self.watched_users = {} - self.message_queue = defaultdict(lambda: defaultdict(deque)) - self.consumption_queue = {} - self.retries = 5 - self.retry_delay = 10 self.channel = None self.webhook = None - self.message_history = MessageHistory() - self.disable_header = disable_header + + @property + def bot(self) -> Bot: + """Return the bot instance from config.""" + return self.config.bot + + @property + def log(self) -> CustomLogger: + """Return the logger from config.""" + return self.config.logger @property def consuming_messages(self) -> bool: """Checks if a consumption task is currently running.""" - if self._consume_task is None: + if self.queue_state.consume_task is None: return False - if self._consume_task.done(): - exc = self._consume_task.exception() + if self.queue_state.consume_task.done(): + exc = self.queue_state.consume_task.exception() if exc: self.log.exception( "The message queue consume task has failed with:", @@ -94,14 +130,14 @@ async def cog_load(self) -> None: await self.bot.wait_until_guild_available() try: - self.channel = await get_or_fetch_channel(self.bot, self.destination) + self.channel = await get_or_fetch_channel(self.bot, self.config.destination) except HTTPException: - self.log.exception(f"Failed to retrieve the text channel with id `{self.destination}`") + self.log.exception(f"Failed to retrieve the text channel with id `{self.config.destination}`") try: - self.webhook = await self.bot.fetch_webhook(self.webhook_id) + self.webhook = await self.bot.fetch_webhook(self.config.webhook_id) except discord.HTTPException: - self.log.exception(f"Failed to fetch webhook with id `{self.webhook_id}`") + self.log.exception(f"Failed to fetch webhook with id `{self.config.webhook_id}`") if self.channel is None or self.webhook is None: self.log.error("Failed to start the watch channel; unloading the cog.") @@ -149,7 +185,7 @@ async def fetch_user_cache(self) -> bool: This function returns `True` if the update succeeded. """ try: - data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params) + data = await self.bot.api_client.get(self.config.api_endpoint, params=self.config.api_default_params) except ResponseCodeError as err: self.log.exception("Failed to fetch the watched users from the API", exc_info=err) return False @@ -167,10 +203,10 @@ async def on_message(self, msg: Message) -> None: """Queues up messages sent by watched users.""" if msg.author.id in self.watched_users: if not self.consuming_messages: - self._consume_task = scheduling.create_task(self.consume_messages()) + self.queue_state.consume_task = scheduling.create_task(self.consume_messages()) self.log.trace(f"Received message: {msg.content} ({len(msg.attachments)} attachments)") - self.message_queue[msg.author.id][msg.channel.id].append(msg) + self.queue_state.message_queue[msg.author.id][msg.channel.id].append(msg) async def consume_messages(self, delay_consumption: bool = True) -> None: """Consumes the message queues to log watched users' messages.""" @@ -181,11 +217,11 @@ async def consume_messages(self, delay_consumption: bool = True) -> None: self.log.trace("Started consuming the message queue") # If the previous consumption Task failed, first consume the existing comsumption_queue - if not self.consumption_queue: - self.consumption_queue = self.message_queue.copy() - self.message_queue.clear() + if not self.queue_state.consumption_queue: + self.queue_state.consumption_queue = self.queue_state.message_queue.copy() + self.queue_state.message_queue.clear() - for user_id, channel_queues in self.consumption_queue.items(): + for user_id, channel_queues in self.queue_state.consumption_queue.items(): for channel_queue in channel_queues.values(): while channel_queue: msg = channel_queue.popleft() @@ -196,11 +232,11 @@ async def consume_messages(self, delay_consumption: bool = True) -> None: else: self.log.trace(f"Not consuming message {msg.id} as user {user_id} is no longer watched.") - self.consumption_queue.clear() + self.queue_state.consumption_queue.clear() - if self.message_queue: + if self.queue_state.message_queue: self.log.trace("Channel queue not empty: Continuing consuming queues") - self._consume_task = scheduling.create_task(self.consume_messages(delay_consumption=False)) + self.queue_state.consume_task = scheduling.create_task(self.consume_messages(delay_consumption=False)) else: self.log.trace("Done consuming messages.") @@ -226,11 +262,11 @@ async def relay_message(self, msg: Message, watch_info: dict) -> None: limit = BigBrotherConfig.header_message_limit if ( - msg.author.id != self.message_history.last_author - or msg.channel.id != self.message_history.last_channel - or self.message_history.message_count >= limit + msg.author.id != self.queue_state.message_history.last_author + or msg.channel.id != self.queue_state.message_history.last_channel + or self.queue_state.message_history.message_count >= limit ): - self.message_history = MessageHistory(last_author=msg.author.id, last_channel=msg.channel.id) + self.queue_state.message_history = MessageHistory(last_author=msg.author.id, last_channel=msg.channel.id) await self.send_header(msg, watch_info) @@ -269,11 +305,11 @@ async def relay_message(self, msg: Message, watch_info: dict) -> None: exc_info=exc ) - self.message_history.message_count += 1 + self.queue_state.message_history.message_count += 1 async def send_header(self, msg: Message, watch_info: dict) -> None: """Sends a header embed with information about the relayed messages to the watch channel.""" - if self.disable_header: + if self.config.disable_header: return guild = self.bot.get_guild(GuildConfig.id) @@ -372,7 +408,7 @@ def _remove_user(self, user_id: int) -> None: async def cog_unload(self) -> None: """Takes care of unloading the cog and canceling the consumption task.""" self.log.trace("Unloading the cog") - if self._consume_task and not self._consume_task.done(): + if self.queue_state.consume_task and not self.queue_state.consume_task.done(): def done_callback(task: asyncio.Task) -> None: """Send exception when consuming task have been cancelled.""" try: @@ -382,5 +418,5 @@ def done_callback(task: asyncio.Task) -> None: f"The consume task of {type(self).__name__} was canceled. Messages may be lost." ) - self._consume_task.add_done_callback(done_callback) - self._consume_task.cancel() + self.queue_state.consume_task.add_done_callback(done_callback) + self.queue_state.consume_task.cancel() diff --git a/bot/exts/moderation/watchchannels/bigbrother.py b/bot/exts/moderation/watchchannels/bigbrother.py index 7af8c7152b..bf9166c45e 100644 --- a/bot/exts/moderation/watchchannels/bigbrother.py +++ b/bot/exts/moderation/watchchannels/bigbrother.py @@ -106,7 +106,7 @@ async def apply_watch(self, ctx: Context, user: MemberOrUser, reason: str) -> No msg = f":white_check_mark: Messages sent by {user.mention} will now be relayed to Big Brother." history = await self.bot.api_client.get( - self.api_endpoint, + self.config.api_endpoint, params={ "user__id": str(user.id), "active": "false", @@ -133,10 +133,10 @@ async def apply_unwatch(self, ctx: Context, user: MemberOrUser, reason: str, sen `ctx`. """ active_watches = await self.bot.api_client.get( - self.api_endpoint, + self.config.api_endpoint, params=ChainMap( {"user__id": str(user.id)}, - self.api_default_params, + self.config.api_default_params, ) ) if active_watches: @@ -144,7 +144,7 @@ async def apply_unwatch(self, ctx: Context, user: MemberOrUser, reason: str, sen [infraction] = active_watches await self.bot.api_client.patch( - f"{self.api_endpoint}/{infraction['id']}", + f"{self.config.api_endpoint}/{infraction['id']}", json={"active": False} ) diff --git a/bot/exts/utils/internal.py b/bot/exts/utils/internal.py index ea27a5f503..d05443c26b 100644 --- a/bot/exts/utils/internal.py +++ b/bot/exts/utils/internal.py @@ -5,6 +5,7 @@ import textwrap import traceback from collections import Counter +from dataclasses import dataclass, field from io import StringIO from typing import Any @@ -21,6 +22,14 @@ log = get_logger(__name__) +@dataclass +class SocketStats: + """Container for websocket event statistics.""" + since: arrow.Arrow + event_total: int = 0 + events: Counter = field(default_factory=Counter) + + class Internal(Cog): """Administrator and Core Developer commands.""" @@ -30,9 +39,7 @@ def __init__(self, bot: Bot): self.ln = 0 self.stdout = StringIO() - self.socket_since = arrow.utcnow() - self.socket_event_total = 0 - self.socket_events = Counter() + self.socket_stats = SocketStats(since=arrow.utcnow()) if DEBUG_MODE: self.eval.add_check(is_owner().predicate) @@ -40,8 +47,8 @@ def __init__(self, bot: Bot): @Cog.listener() async def on_socket_event_type(self, event_type: str) -> None: """When a websocket event is received, increase our counters.""" - self.socket_event_total += 1 - self.socket_events[event_type] += 1 + self.socket_stats.event_total += 1 + self.socket_stats.events[event_type] += 1 def _format(self, inp: str, out: Any) -> tuple[str, discord.Embed | None]: """Format the eval output into a string & attempt to format it into an Embed.""" @@ -246,9 +253,9 @@ async def eval(self, ctx: Context, *, code: str) -> None: @has_any_role(Roles.admins, Roles.owners, Roles.core_developers) async def socketstats(self, ctx: Context) -> None: """Fetch information on the socket events received from Discord.""" - running_s = (arrow.utcnow() - self.socket_since).total_seconds() + running_s = (arrow.utcnow() - self.socket_stats.since).total_seconds() - per_s = self.socket_event_total / running_s + per_s = self.socket_stats.event_total / running_s stats_embed = discord.Embed( title="WebSocket statistics", @@ -256,7 +263,7 @@ async def socketstats(self, ctx: Context) -> None: color=discord.Color.og_blurple() ) - for event_type, count in self.socket_events.most_common(25): + for event_type, count in self.socket_stats.events.most_common(25): stats_embed.add_field(name=event_type, value=f"{count:,}", inline=True) await ctx.send(embed=stats_embed) diff --git a/tests/bot/exts/filtering/test_discord_token_filter.py b/tests/bot/exts/filtering/test_discord_token_filter.py index 1cb9e16fac..fbeffc10eb 100644 --- a/tests/bot/exts/filtering/test_discord_token_filter.py +++ b/tests/bot/exts/filtering/test_discord_token_filter.py @@ -5,7 +5,7 @@ import arrow -from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_context import Event, FilterContext, FilterInput, FilterOutput from bot.exts.filtering._filters.unique import discord_token from bot.exts.filtering._filters.unique.discord_token import DiscordTokenFilter, Token from tests.helpers import MockBot, MockMember, MockMessage, MockTextChannel, autospec @@ -32,7 +32,7 @@ def setUp(self): member = MockMember(id=123) channel = MockTextChannel(id=345) - self.ctx = FilterContext(Event.MESSAGE, member, channel, "", self.msg) + self.ctx = FilterContext(FilterInput(Event.MESSAGE, member, channel, "", self.msg), FilterOutput()) def test_extract_user_id_valid(self): """Should consider user IDs valid if they decode into an integer ID.""" diff --git a/tests/bot/exts/filtering/test_extension_filter.py b/tests/bot/exts/filtering/test_extension_filter.py index 67a503b306..e4f3761c51 100644 --- a/tests/bot/exts/filtering/test_extension_filter.py +++ b/tests/bot/exts/filtering/test_extension_filter.py @@ -4,7 +4,7 @@ import arrow from bot.constants import Channels -from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_context import Event, FilterContext, FilterInput, FilterOutput from bot.exts.filtering._filter_lists import extension from bot.exts.filtering._filter_lists.extension import ExtensionsList from bot.exts.filtering._filter_lists.filter_list import ListType @@ -39,7 +39,7 @@ def setUp(self): self.message = MockMessage() member = MockMember(id=123) channel = MockTextChannel(id=345) - self.ctx = FilterContext(Event.MESSAGE, member, channel, "", self.message) + self.ctx = FilterContext(FilterInput(Event.MESSAGE, member, channel, "", self.message), FilterOutput()) @patch("bot.instance", BOT) async def test_message_with_allowed_attachment(self): diff --git a/tests/bot/exts/filtering/test_settings_entries.py b/tests/bot/exts/filtering/test_settings_entries.py index f12b2caa55..72737d10e0 100644 --- a/tests/bot/exts/filtering/test_settings_entries.py +++ b/tests/bot/exts/filtering/test_settings_entries.py @@ -1,6 +1,6 @@ import unittest -from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_context import Event, FilterContext, FilterInput, FilterOutput from bot.exts.filtering._settings_types.actions.infraction_and_notification import ( Infraction, InfractionAndNotification, @@ -19,7 +19,7 @@ def setUp(self) -> None: member = MockMember(id=123) channel = MockTextChannel(id=345) message = MockMessage(author=member, channel=channel) - self.ctx = FilterContext(Event.MESSAGE, member, channel, "", message) + self.ctx = FilterContext(FilterInput(Event.MESSAGE, member, channel, "", message), FilterOutput()) def test_role_bypass_is_off_for_user_without_roles(self): """The role bypass should trigger when a user has no roles.""" diff --git a/tests/bot/exts/filtering/test_token_filter.py b/tests/bot/exts/filtering/test_token_filter.py index 03fa6b4b9e..7974d27458 100644 --- a/tests/bot/exts/filtering/test_token_filter.py +++ b/tests/bot/exts/filtering/test_token_filter.py @@ -2,7 +2,7 @@ import arrow -from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_context import Event, FilterContext, FilterInput, FilterOutput from bot.exts.filtering._filters.token import TokenFilter from tests.helpers import MockMember, MockMessage, MockTextChannel @@ -14,7 +14,7 @@ def setUp(self) -> None: member = MockMember(id=123) channel = MockTextChannel(id=345) message = MockMessage(author=member, channel=channel) - self.ctx = FilterContext(Event.MESSAGE, member, channel, "", message) + self.ctx = FilterContext(FilterInput(Event.MESSAGE, member, channel, "", message), FilterOutput()) async def test_token_filter_triggers(self): """The filter should evaluate to True only if its token is found in the context content."""