Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ check-format: ## Check code quality.
.PHONY: check-lint
check-lint: ## Check code quality.
ruff check ${python_src}
dmypy run -- ${python_src}
pyrefly check --search-path . ${python_src}

.PHONY: check-test
check-test: ## Check tests.
Expand Down
5 changes: 4 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ discolinks = "discolinks.cli:main"
[dependency-groups]
dev = [
"flask>=3.0.3",
"mypy>=1.0.0",
"pyrefly>=0.35.0",
"pytest>=8.2.2",
"ruff>=0.6.3",
"types-beautifulsoup4>=4.11.1",
Expand All @@ -51,3 +51,6 @@ check_untyped_defs = true
no_implicit_reexport = true
python_version = "3.9"
warn_unused_ignores = true

[tool.pyrefly]
python_version = "3.9.0"
15 changes: 7 additions & 8 deletions src/discolinks/analyzer.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
from dataclasses import dataclass
from typing import Mapping, Sequence

import attrs

from . import outcome
from .core import Url
from .url_store import UrlInfo


@attrs.frozen
@dataclass(frozen=True)
class LinkResult:
href: str
url: Url
Expand All @@ -17,7 +16,7 @@ def ok(self) -> bool:
return self.results.ok()


@attrs.frozen
@dataclass(frozen=True)
class Page:
links: Sequence[LinkResult]

Expand Down Expand Up @@ -55,10 +54,10 @@ def make_chain(
return chain


@attrs.define
@dataclass
class Stats:
ok: int = attrs.field(default=0)
failed: int = attrs.field(default=0)
ok: int = 0
failed: int = 0

@property
def total(self) -> int:
Expand All @@ -71,7 +70,7 @@ def add(self, ok: bool) -> None:
self.failed += 1


@attrs.frozen
@dataclass(frozen=True)
class Analysis:
stats: Stats
pages: Mapping[Url, Page]
Expand Down
21 changes: 13 additions & 8 deletions src/discolinks/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import click
import rich.console
import rich.markup
from rich.logging import RichHandler

from . import analyzer, export, text
Expand Down Expand Up @@ -91,22 +92,26 @@ async def main_async(
start_url: Url,
):
requester = Requester(excluder=excluder)
next_url: Optional[Url] = start_url
url = start_url

while next_url is not None:
url = next_url
while True:
result = await requester.get(url)
next_url = result.redirect_url()
new_urls = url_store.add_page(
url=url,
info=UrlInfo(result=result, links=get_links(url=url, result=result)),
)
if next_url is not None:
logger.info(f"Redirected to {next_url}")

if next_url not in new_urls:
logger.error("Detected circular redirects. Aborting.")
exit(1)
if next_url is None:
break

logger.info(f"Redirected to {next_url}")

if next_url not in new_urls:
logger.error("Detected circular redirects. Aborting.")
exit(1)

url = next_url

error_msg = result.error_msg()
if error_msg is not None:
Expand Down
7 changes: 3 additions & 4 deletions src/discolinks/core.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
from dataclasses import dataclass
from urllib.parse import urldefrag, urlparse

import attrs


@attrs.frozen
@dataclass(frozen=True)
class Url:
"""
Wrapper around URL strings.
Expand Down Expand Up @@ -31,7 +30,7 @@ def __str__(self) -> str:
return self.full


@attrs.frozen
@dataclass(frozen=True)
class Link:
href: str
url: Url
7 changes: 3 additions & 4 deletions src/discolinks/excluder.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import re
from dataclasses import dataclass
from typing import Sequence

import attrs

from .core import Url


@attrs.frozen
@dataclass(frozen=True)
class ExcluderRegexError(Exception):
regex: str
msg: str
Expand All @@ -19,7 +18,7 @@ def parse_regex(regex: str) -> re.Pattern:
raise ExcluderRegexError(regex=regex, msg=str(error)) from error


@attrs.frozen
@dataclass(frozen=True)
class Excluder:
"""
Preprocessed engine excluding certain URLs
Expand Down
7 changes: 3 additions & 4 deletions src/discolinks/export.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import json
from dataclasses import dataclass
from typing import Any, Mapping, Sequence

import attrs

from . import outcome
from .analyzer import Analysis, LinkResult, Page
from .core import Url


@attrs.frozen
@dataclass(frozen=True)
class Converter(outcome.Converter[Any]):
def convert_page(self, page: outcome.Page) -> Any:
return {
Expand All @@ -30,7 +29,7 @@ def convert_request_error(self, error: outcome.RequestError) -> Any:
"message": str(error.msg),
}

def convert_excluded(self, unknown: outcome.Excluded) -> Any:
def convert_excluded(self, excluded: outcome.Excluded) -> Any:
return {
"type": "excluded",
}
Expand Down
5 changes: 2 additions & 3 deletions src/discolinks/link_extractor.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
from dataclasses import dataclass
from typing import Optional, Sequence

import attrs

from . import html, outcome
from .core import Link, Url


@attrs.frozen
@dataclass(frozen=True)
class LinkExtractor(outcome.Converter[Optional[Sequence[Link]]]):
url: Url

Expand Down
18 changes: 9 additions & 9 deletions src/discolinks/monitor.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Iterator

import attrs
import rich.console
import rich.status

from . import outcome


@attrs.define
@dataclass
class Stats:
queued: int = attrs.field(default=0)
in_progress: int = attrs.field(default=0)
finished: int = attrs.field(default=0)
ok: int = attrs.field(default=0)
failed: int = attrs.field(default=0)
queued: int = 0
in_progress: int = 0
finished: int = 0
ok: int = 0
failed: int = 0

def on_task_started(self, queued: int) -> None:
self.queued = queued
Expand All @@ -30,7 +30,7 @@ def on_task_done(self, queued: int, result: outcome.Result) -> None:
self.failed += 1


@attrs.frozen
@dataclass(frozen=True)
class Monitor:
"""
Handle updates of the status bar during scraping.
Expand All @@ -40,7 +40,7 @@ class Monitor:

console: rich.console.Console
status: rich.status.Status
stats: Stats = attrs.field(init=False, factory=Stats)
stats: Stats = field(init=False, default_factory=Stats)

@classmethod
def start(cls, console: rich.console.Console) -> "Monitor":
Expand Down
17 changes: 8 additions & 9 deletions src/discolinks/outcome.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Generic, Optional, Sequence, TypeVar

import attrs

from .core import Url

Out = TypeVar("Out")


@attrs.frozen
@dataclass(frozen=True)
class Result(ABC):
@abstractmethod
def ok(self) -> bool:
Expand All @@ -31,7 +30,7 @@ def convert_with(self, converter: "Converter[Out]") -> Out:
pass


@attrs.frozen
@dataclass(frozen=True)
class Redirect(Result):
code: int
ref: str
Expand All @@ -53,7 +52,7 @@ def convert_with(self, converter: "Converter[Out]") -> Out:
return converter.convert_redirect(self)


@attrs.frozen
@dataclass(frozen=True)
class Page(Result):
code: int
body: str
Expand All @@ -74,7 +73,7 @@ def convert_with(self, converter: "Converter[Out]") -> Out:
return converter.convert_page(self)


@attrs.frozen
@dataclass(frozen=True)
class RequestError(Result):
msg: str

Expand All @@ -94,7 +93,7 @@ def convert_with(self, converter: "Converter[Out]") -> Out:
return converter.convert_request_error(self)


@attrs.frozen
@dataclass(frozen=True)
class Excluded(Result):
def ok(self) -> bool:
return True
Expand All @@ -112,7 +111,7 @@ def convert_with(self, converter: "Converter[Out]") -> Out:
return converter.convert_excluded(self)


@attrs.frozen
@dataclass(frozen=True)
class Unknown(Result):
def ok(self) -> bool:
# It's considered OK because the underlying error will be reported by another
Expand All @@ -132,7 +131,7 @@ def convert_with(self, converter: "Converter[Out]") -> Out:
return converter.convert_unknown(self)


@attrs.frozen
@dataclass(frozen=True)
class Results:
chain: Sequence[Result]

Expand Down
6 changes: 3 additions & 3 deletions src/discolinks/requester.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import logging
import ssl
from dataclasses import dataclass, field
from typing import Union

import attrs
import httpx

from . import outcome
Expand Down Expand Up @@ -33,10 +33,10 @@ def httpx_to_error(error: Union[httpx.RequestError, ssl.SSLError]) -> str:
return str(error)


@attrs.frozen
@dataclass(frozen=True)
class Requester:
excluder: Excluder
client: httpx.AsyncClient = attrs.field(init=False, factory=httpx.AsyncClient)
client: httpx.AsyncClient = field(init=False, default_factory=httpx.AsyncClient)

async def get(self, url: Url, use_head: bool = False) -> outcome.Result:
"""
Expand Down
5 changes: 3 additions & 2 deletions src/discolinks/text.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import attrs
from dataclasses import dataclass

from rich import print
from rich.markup import escape
from rich.tree import Tree

from . import analyzer, outcome


@attrs.frozen
@dataclass(frozen=True)
class Converter(outcome.Converter[str]):
def convert_page(self, page: outcome.Page) -> str:
return f"[red]{escape(str(page.code))}[/red]"
Expand Down
11 changes: 5 additions & 6 deletions src/discolinks/url_store.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
from dataclasses import dataclass, field
from typing import Mapping, Optional, Sequence

import attrs

from . import outcome
from .core import Link, Url


@attrs.frozen
@dataclass(frozen=True)
class UrlInfo:
result: outcome.Result
links: Optional[Sequence[Link]]
Expand All @@ -22,10 +21,10 @@ def link_urls(self) -> frozenset[Url]:
return frozenset(link.url for link in self.links)


@attrs.frozen
@dataclass(frozen=True)
class UrlStore:
url_infos: dict[Url, UrlInfo] = attrs.field(init=False, factory=dict)
seen_urls: set[Url] = attrs.field(init=False, factory=set)
url_infos: dict[Url, UrlInfo] = field(init=False, default_factory=dict)
seen_urls: set[Url] = field(init=False, default_factory=set)

def add_page(self, url: Url, info: UrlInfo) -> frozenset[Url]:
"""
Expand Down
Loading