diff --git a/.agents/skills/external/ibexa-documentation/SKILL.md b/.agents/skills/external/ibexa-documentation/SKILL.md new file mode 100644 index 0000000000..1948edcea4 --- /dev/null +++ b/.agents/skills/external/ibexa-documentation/SKILL.md @@ -0,0 +1,56 @@ +--- +name: ibexa-documentation +description: Look up features, concepts, configuration, extension points, and back office workflows in the locally installed Ibexa DXP documentation. Use whenever working on an Ibexa DXP project and you need to know how something in Ibexa works before writing code, configuration, or answers. +--- + +# Ibexa DXP documentation + +The official Ibexa DXP documentation is installed as Markdown files in `vendor/ibexa/documentation-developer/`. +It matches this project's Ibexa DXP release line (e.g. 5.0). +Always consult it before web searches or memory: it works offline, matches the installed version, and links directly to the code in `vendor/`. + +It contains two documentation sets: + +- `developer/` — APIs, configuration, extension points, tutorials. Mirrors https://doc.ibexa.co/en/latest/ +- `user/` — back office, editorial and commerce workflows. Mirrors https://doc.ibexa.co/projects/userguide/en/latest/ + +Use the documentation for what the code cannot tell you: + +- Concepts and the content model: content types, fields, locations, versions, languages, and how they relate. +- Release notes and changes between versions: `developer/release_notes/`. +- Update and migration instructions: `developer/update_and_migration/`. +- Project and feature setup, configuration, and best practices: `developer/infrastructure_and_maintenance/`. + +For method signatures and contracts, the code in `vendor/` is authoritative. +Use the documentation for the intent, concepts, and configuration around them. +When the documentation and the installed code don't match, the code is right. + +## Finding topics + +- Read the set's table of contents: `vendor/ibexa/documentation-developer/developer/llms.txt` (or `user/llms.txt`). Page titles are grouped by section, with relative links. +- Search the sets directly: `grep -ril "" vendor/ibexa/documentation-developer/`. + +## Conventions + +- Links between pages are relative and work offline. Follow them for related topics. +- PHP API references link to the class source in this project's `vendor/` directory. If the target file doesn't exist, that package isn't installed. + +## Keep documentation up to date + +The package is versioned as `MAJOR_DXP_VERSION.MINOR_DXP_VERSION.DATE_OF_TAGGING_YYYYMMDD` and must be used with the matching Ibexa DXP release line. +For example, `5.0.20260101` is for Ibexa DXP 5.0.x releases. +New version is released when the documentation content is updated. + +To check whether the documentation contains all the latest updates, run: + +``` bash +composer outdated --direct "ibexa/documentation-*" +``` + +To update it, run: + +```bash +composer update ibexa/documentation-developer --no-scripts +``` + +Always pass `--no-scripts`: the package contains only Markdown files, so the slow post-update scripts (cache clear, asset and frontend builds) are unnecessary. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..701ed09182 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +/* export-ignore +/.agents export-ignore=false +/developer export-ignore=false +/user export-ignore=false +/composer.json export-ignore=false +/LICENSE export-ignore=false +/README.md export-ignore=false diff --git a/.github/workflows/release_composer_package.yaml b/.github/workflows/release_composer_package.yaml new file mode 100644 index 0000000000..17804006c8 --- /dev/null +++ b/.github/workflows/release_composer_package.yaml @@ -0,0 +1,210 @@ +name: "Release Composer package" + +on: + schedule: + # Daily at 03:17 UTC. Fans out to every maintained version branch (see determine-branches). + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + version: + description: "Version branch to release" + required: true + default: "5.0" + type: string + devdoc_branch: + description: "Developer documentation branch to check out (defaults to version)" + required: false + default: "" + type: string + userdoc_branch: + description: "User documentation branch to check out (defaults to version)" + required: false + default: "" + type: string + +jobs: + determine-branches: + runs-on: ubuntu-latest + outputs: + branches: ${{ steps.set.outputs.branches }} + devdoc_branch: ${{ steps.set.outputs.devdoc_branch }} + userdoc_branch: ${{ steps.set.outputs.userdoc_branch }} + steps: + - id: set + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + # Manual runs release the branch chosen via the "version" input, + # not necessarily the branch the workflow was dispatched from. + branches='["${{ github.event.inputs.version }}"]' + # Left empty when not overridden; the release job then falls back + # to the version branch (matrix.branch) for the checkout ref. + devdoc_branch="${{ github.event.inputs.devdoc_branch }}" + userdoc_branch="${{ github.event.inputs.userdoc_branch }}" + else + # Scheduled runs always execute in the default branch's context, so they must + # explicitly check out each maintained version branch instead of relying on + # GITHUB_REF. Extend this list as new branches become active / drop EOL ones. + branches='["5.0"]' + devdoc_branch="" + userdoc_branch="" + fi + echo "branches=${branches}" >> "$GITHUB_OUTPUT" + echo "devdoc_branch=${devdoc_branch}" >> "$GITHUB_OUTPUT" + echo "userdoc_branch=${userdoc_branch}" >> "$GITHUB_OUTPUT" + + release: + needs: determine-branches + runs-on: ubuntu-latest + permissions: + # Needed to commit the generated Markdown and push a tag. + contents: write + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.determine-branches.outputs.branches) }} + + steps: + # Shallow checkout of the branch tip only. The generated Markdown is never + # pushed to the branch itself (see below), only tagged, so the diff target is + # the latest tag — fetched individually in "Find the last tag" instead of + # pulling the full history + every daily tag snapshot (which took minutes). + - uses: actions/checkout@v4 + with: + ref: ${{ needs.determine-branches.outputs.devdoc_branch || matrix.branch }} + + - name: Check out user documentation + uses: actions/checkout@v4 + with: + repository: ibexa/documentation-user + ref: ${{ needs.determine-branches.outputs.userdoc_branch || matrix.branch }} + path: user-docs + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.13" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # user-docs pins ibexa-llms-txt from the upstream git repo; drop that line — + # the editable install from this checkout (requirements.txt: -e .) already + # provides the plugin, and pip refuses two direct references to one package. + grep -v '^ibexa-llms-txt @' user-docs/requirements.txt > user-reqs-filtered.txt + pip install -r requirements.txt -r user-reqs-filtered.txt + + - name: Setup PHP + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 #2.37.2 + with: + php-version: "8.3" + coverage: none + + # TMP: Uncomment when merging to ibexa repository + # - name: Generate token + # id: generate_token + # uses: actions/create-github-app-token@v2 + # with: + # app-id: ${{ secrets.AUTOMATION_CLIENT_ID }} + # private-key: ${{ secrets.AUTOMATION_CLIENT_SECRET }} + # owner: ${{ github.repository_owner }} + + - name: Add composer keys for private packagist + run: | + composer config --global http-basic.updates.ibexa.co $SATIS_NETWORK_KEY $SATIS_NETWORK_TOKEN + composer config --global github-oauth.github.com $GITHUB_TOKEN + env: + SATIS_NETWORK_KEY: ${{ secrets.SATIS_NETWORK_KEY }} + SATIS_NETWORK_TOKEN: ${{ secrets.SATIS_NETWORK_TOKEN }} + GITHUB_TOKEN: ${{ secrets.MY_GITHUB_TOKEN }} #${{ steps.generate_token.outputs.token }} #TMP: Revert when merging to ibexa repository + + # Needed to resolve the PHP API classes referenced by the docs to + # their vendor/ source paths (dump_class_paths.php). + - uses: ramsey/composer-install@a8d0d959dab41457692a5e2041bd9b757a119e3f #v3.2.1 + with: + dependency-versions: highest + + - name: Build developer documentation + run: mkdocs build --strict + env: + # The cards() macro versions its links from this (as on RTD); + # without it they'd point at en/latest instead of this branch. + READTHEDOCS_VERSION_NAME: ${{ matrix.branch }} + + - name: Build user documentation + run: mkdocs build --strict + working-directory: user-docs + env: + READTHEDOCS_VERSION_NAME: ${{ matrix.branch }} + + - name: Resolve PHP API classes to vendor paths + run: php tools/llm_package/dump_class_paths.php site user-docs/site class_paths.json + + - name: Build package Markdown (developer/, user/) + run: python build_package_docs.py --version "${{ matrix.branch }}" + + - name: Copy package README + run: cp tools/llm_package/README.package.md README.md + + - name: Smoke-test package output + run: | + composer validate + + - name: Find the last tag for this branch + id: last_tag + env: + BRANCH: ${{ matrix.branch }} + run: | + # List tag names on the remote (no object download), then fetch only the + # latest one — the single commit the change check diffs against. + latest_tag=$(git ls-remote --tags origin "refs/tags/${BRANCH}.*" \ + | awk '{print $2}' | sed 's|^refs/tags/||' | grep -v '\^{}$' \ + | sort -t. -k1,1n -k2,2n -k3,3n -k4,4n | tail -1) + if [[ -n "$latest_tag" ]]; then + git fetch --depth=1 origin "refs/tags/${latest_tag}:refs/tags/${latest_tag}" + fi + echo "latest_tag=${latest_tag}" >> "$GITHUB_OUTPUT" + + - name: Check for content changes since the last tag + id: changes + env: + LATEST_TAG: ${{ steps.last_tag.outputs.latest_tag }} + run: | + # The generated Markdown is never committed/pushed to the branch itself (only + # tagged), so there's nothing on the branch to diff against — compare the freshly + # built content to what the last tag captured instead. -f because developer/ and + # user/ are gitignored to keep local builds out of the working tree. + git add -f developer user + git add README.md + if [[ -z "$LATEST_TAG" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + # .agents (agent skills, tracked on the branch) is part of the package too; + # include it so skill-only edits also produce a release. + elif git diff --cached --quiet "$LATEST_TAG" -- developer user README.md .agents; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit generated Markdown + if: steps.changes.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git commit -m "Update generated Markdown docs for ${{ matrix.branch }}" + + - name: Tag release + if: steps.changes.outputs.changed == 'true' + env: + BRANCH: ${{ matrix.branch }} + run: | + # Push only the tag, never the branch — the branch stays exactly as authored, + # the generated Markdown exists solely as the commit this tag points to. + tag="${BRANCH}.$(date -u +%Y%m%d)" + + if git rev-parse "$tag" >/dev/null 2>&1; then + echo "Tag $tag already exists, skipping." + exit 0 + fi + + git tag "$tag" + git push origin "$tag" diff --git a/.gitignore b/.gitignore index 1f2f58534f..f48666154a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,9 @@ lychee-report.md code_samples/_inline_php/ /build/ *.egg-info/ +/doc/ +/llms.txt +/developer/ +/user/ +/class_paths.json +/user-docs/ diff --git a/build_package_docs.py b/build_package_docs.py new file mode 100644 index 0000000000..8a000e9eb9 --- /dev/null +++ b/build_package_docs.py @@ -0,0 +1,368 @@ +""" +Build the Composer-package Markdown docs from the MkDocs build outputs. + +The package bundles two documentation sets, each produced by its own MkDocs build with the llmstxt plugin: + +- `developer/` — ibexa/documentation-developer +- `user/` — ibexa/documentation-user + +Each set gets its own `llms.txt` table of contents at its root. + +The following transformations are applied to the MkDocs build outputs before writing them into the package:: + +- Page links to developer and user documentation become relative links to the corresponding `//index.md`` files, including cross-set links between the two documentations: + + URLs matching a mkdocs-redirects entry are resolved through the redirect first. + URLs with no local page (other doc.ibexa.co projects, images, the separately hosted API reference HTML) are left untouched. + Links to versions other than the current branch's version (e.g. en/4.0/…) are also left untouched. + +- PHP API class links (`.../php_api_reference/classes/.html`) become relative links to the class source file in the vendor/ director: + + By using the FQCN -> path map produced by `tools/llm_package/dump_class_paths.php`. + The link text is upgraded to the FQCN so the class stays greppable even if the target package isn't installed in the user's project. + +Run after both documentation sites are built with MkDocs: + +``` bash +php tools/llm_package/dump_class_paths.php site user-docs/site class_paths.json +python build_package_docs.py --version 5.0 +``` +""" + +import argparse +import json +import posixpath +import re +import shutil +import sys +from pathlib import Path, PurePosixPath +from typing import NamedTuple + +import yaml + +# Number of path segments from the package root up to vendor/ +# (vendor/ibexa/documentation-developer/ -> vendor/). +_PACKAGE_DEPTH_IN_VENDOR = 2 + +# [text](url) or ![alt](url); group 1 distinguishes images. URLs never contain +# whitespace or parentheses in the generated Markdown. +_MD_LINK_RE = re.compile(r"(!?)\[([^\]]*)\]\(([^()\s]+)\)") + +_API_CLASS_URL_RE = re.compile(r"php_api_reference/classes/([A-Za-z0-9-]+)\.html$") + +# Same fence detection as llmstxt_preprocess.renumber_ordered_lists. +_FENCE_OPEN_RE = re.compile(r"^(\s*)(`{3,}|~{3,})") + +_LLMS_TXT_POINTER_RE = re.compile( + r"(?m)^> For the complete documentation index, see \[llms\.txt\]\([^)]*\)\.\n\n?" +) + + +class DocSet(NamedTuple): + """One documentation set shipped in the package.""" + + root: str # top-level package directory, e.g. 'developer' + base_urls: tuple # URL prefixes owned by this set, e.g. ('https://doc.ibexa.co/en/latest/',) + pages: frozenset # page paths relative to the set root, e.g. 'search/search/index.md' + redirects: dict # URL path -> URL path, from mkdocs-redirects + + +def load_redirect_maps(plugins_path): + """Read mkdocs-redirects ``redirect_maps`` from plugins.yml as URL paths. + + Entries map docs-relative source files to target files + ('guide/images.md' -> 'content_management/images/images.md'); with + use_directory_urls both sides publish as directory URLs, so the returned + dict maps 'guide/images/' -> 'content_management/images/images/'. + """ + with open(plugins_path, encoding="utf-8") as f: + data = yaml.safe_load(f) + + for plugin in data.get("plugins", []): + if isinstance(plugin, dict) and "redirects" in plugin: + redirect_maps = (plugin["redirects"] or {}).get("redirect_maps") or {} + return { + _md_to_url_path(src): _md_to_url_path(target) + for src, target in redirect_maps.items() + } + return {} + + +def _md_to_url_path(md_path): + """'guide/images.md' -> 'guide/images/' (mkdocs directory URL).""" + path = md_path[: -len(".md")] if md_path.endswith(".md") else md_path + if path.endswith("/index"): + path = path[: -len("index")] + return path.rstrip("/") + "/" if path else "" + + +def _page_file(url_path, pages, redirects): + """Map a site-relative URL path to its Markdown file, or None. + + ``url_path`` is the part after the set's base URL without the anchor. Only + page URLs (directory URLs or explicit .md paths) are considered; anything + else (images, API reference HTML, files) returns None. + """ + if url_path.endswith(".md"): + candidate = url_path + elif url_path == "" or url_path.endswith("/"): + candidate = url_path + "index.md" + else: + return None + + if candidate in pages: + return candidate + + redirect_target = redirects.get(url_path) + if redirect_target is not None: + candidate = redirect_target + "index.md" + if candidate in pages: + return candidate + return None + + +def _split_anchor(url): + if "#" in url: + base, anchor = url.split("#", 1) + return base, "#" + anchor + return url, "" + + +def _rewrite_api_link(text, url, anchor, page_path, class_paths): + """Rewrite one PHP API class link; returns the full markdown link.""" + slug = _API_CLASS_URL_RE.search(url).group(1) + fqcn = slug.replace("-", "\\") + short_name = fqcn.rsplit("\\", 1)[-1] + + # Upgrade bare class-name link text to the greppable FQCN; keep richer + # texts (e.g. `publishVersion()`) as authored. + if text.strip("`").strip() in (short_name, fqcn): + text = f"`{fqcn}`" + + vendor_path = class_paths.get(fqcn) + if vendor_path is None: + return f"[{text}]({url}{anchor})" + + # Steps up from the page's directory to vendor/: the page lives at + # vendor/ibexa/documentation-developer/. + ups = len(PurePosixPath(page_path).parent.parts) + _PACKAGE_DEPTH_IN_VENDOR + # HTML anchors (#method_…) have no equivalent in the source file; drop them. + return f"[{text}]({'../' * ups}{vendor_path})" + + +def rewrite_links(line, page_path, docsets, class_paths): + """Rewrite all links on one (non-code) Markdown line. + + ``page_path`` is the page's package-relative path (e.g. + 'developer/search/search/index.md'). ``docsets`` are all sets in the + package — links to any of their base URLs are localized, so cross-set + links (developer docs → user docs and back) become relative too. Links to + other doc versions or other doc.ibexa.co projects stay absolute. + """ + + def _replace(match): + bang, text, url = match.groups() + if bang: + return match.group(0) + + bare_url, anchor = _split_anchor(url) + owner = next( + ( + (docset, base) + for docset in docsets + for base in docset.base_urls + if bare_url.startswith(base) + ), + None, + ) + if owner is None: + return match.group(0) + docset, base = owner + + if _API_CLASS_URL_RE.search(bare_url): + return _rewrite_api_link(text, bare_url, anchor, page_path, class_paths) + + target = _page_file(bare_url[len(base):], docset.pages, docset.redirects) + if target is None: + return match.group(0) + relative = posixpath.relpath(f"{docset.root}/{target}", posixpath.dirname(page_path)) + return f"[{text}]({relative}{anchor})" + + return _MD_LINK_RE.sub(_replace, line) + + +def strip_llms_txt_pointer(content): + """Drop the llmstxt plugin's "see llms.txt" pointer line (see _LLMS_TXT_POINTER_RE).""" + return _LLMS_TXT_POINTER_RE.sub("", content) + + +def rewrite_page(content, page_path, docsets, class_paths): + """Rewrite a page's links, leaving fenced code blocks untouched.""" + lines = content.split("\n") + result = [] + fence = None # (fence_char, fence_length) while inside a fenced code block + + for line in lines: + if fence is not None: + stripped = line.strip() + if stripped and set(stripped) == {fence[0]} and len(stripped) >= fence[1]: + fence = None + result.append(line) + continue + + fence_match = _FENCE_OPEN_RE.match(line) + if fence_match: + marker = fence_match.group(2) + fence = (marker[0], len(marker)) + result.append(line) + continue + + result.append(rewrite_links(line, page_path, docsets, class_paths)) + + return "\n".join(result) + + +def rewrite_llms_txt(content, docset): + """Point a set's llms.txt links at its packaged pages. + + llms.txt sits at the set's root, so targets are simply the page paths + relative to that root. + """ + + def _replace(match): + bang, text, url = match.groups() + if bang: + return match.group(0) + bare_url, anchor = _split_anchor(url) + for base in docset.base_urls: + if bare_url.startswith(base): + target = _page_file(bare_url[len(base):], docset.pages, docset.redirects) + if target is not None: + return f"[{text}]({target}{anchor})" + break + return match.group(0) + + return _MD_LINK_RE.sub(_replace, content) + + +def check_relative_doc_links(pages): + """Verify every relative .md link in the rewritten pages resolves. + + ``pages`` maps package-relative page paths -> content. Returns a list of + error strings. Vendor class links (.php) can't be checked against the docs + tree and are skipped. + """ + errors = [] + for page_path, content in pages.items(): + for match in _MD_LINK_RE.finditer(content): + url, _ = _split_anchor(match.group(3)) + if "://" in url or url.startswith("#") or not url.endswith(".md"): + continue + resolved = posixpath.normpath(posixpath.join(posixpath.dirname(page_path), url)) + if resolved.startswith("..") or resolved not in pages: + errors.append(f"{page_path}: broken relative link {match.group(3)}") + return errors + + +def _version_bases(url_prefix, version): + """Base URLs owned by a doc set: en/latest plus the branch's own version.""" + bases = [f"{url_prefix}en/latest/"] + if version: + bases.append(f"{url_prefix}en/{version}/") + return tuple(bases) + + +def _load_docset(root, site_dir, plugins_path, base_urls): + site = Path(site_dir) + if not site.is_dir(): + sys.exit(f"Site directory not found: {site_dir} (run mkdocs build first)") + pages = frozenset(path.relative_to(site).as_posix() for path in site.rglob("*.md")) + return DocSet(root, base_urls, pages, load_redirect_maps(plugins_path)), site + + +def build(dev_site, dev_plugins, user_site, user_plugins, class_map_path, version=None): + developer, dev_dir = _load_docset( + "developer", dev_site, dev_plugins, _version_bases("https://doc.ibexa.co/", version) + ) + user, user_dir = _load_docset( + "user", + user_site, + user_plugins, + _version_bases("https://doc.ibexa.co/projects/userguide/", version), + ) + docsets = ((developer, dev_dir), (user, user_dir)) + class_paths = json.loads(Path(class_map_path).read_text(encoding="utf-8")) + + rewritten = {} + llms = {} + for docset, site in docsets: + for page_rel in sorted(docset.pages): + page_path = f"{docset.root}/{page_rel}" + content = strip_llms_txt_pointer((site / page_rel).read_text(encoding="utf-8")) + rewritten[page_path] = rewrite_page(content, page_path, (developer, user), class_paths) + llms[docset.root] = rewrite_llms_txt( + (site / "llms.txt").read_text(encoding="utf-8"), docset + ) + + errors = check_relative_doc_links(rewritten) + if errors: + for error in errors: + print(error, file=sys.stderr) + sys.exit(f"{len(errors)} broken relative links, not writing output") + + for docset, _ in docsets: + out = Path(docset.root) + if out.exists(): + shutil.rmtree(out) + out.mkdir() + (out / "llms.txt").write_text(llms[docset.root], encoding="utf-8") + + for page_path, content in rewritten.items(): + target = Path(page_path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + all_bases = developer.base_urls + user.base_urls + remaining = sum(content.count(base) for content in rewritten.values() for base in all_bases) + print(f"Wrote {len(rewritten)} pages to developer/ and user/ (each with its llms.txt)") + print(f"{remaining} links intentionally left absolute (no local page)") + + +def _require_local_path(path_str, arg_name): + """Reject a --site/--plugins/--user-site/--user-plugins/--class-map value + that resolves outside the current directory (e.g. an absolute path or a + ../ escape), so a wrong or agent-generated argument can't make the script + read files from elsewhere on disk. + """ + resolved = Path(path_str).resolve() + cwd = Path.cwd().resolve() + if resolved != cwd and cwd not in resolved.parents: + sys.exit(f"--{arg_name} must stay within the current directory: {path_str}") + return path_str + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n", 2)[1]) + parser.add_argument("--site", default="site", help="Developer docs MkDocs build output") + parser.add_argument("--plugins", default="plugins.yml", + help="Developer docs plugins.yml with redirect_maps") + parser.add_argument("--user-site", default="user-docs/site", + help="User docs MkDocs build output") + parser.add_argument("--user-plugins", default="user-docs/plugins.yml", + help="User docs plugins.yml with redirect_maps") + parser.add_argument("--class-map", default="class_paths.json", + help="FQCN -> vendor path map from dump_class_paths.php") + parser.add_argument("--version", default=None, + help="This branch's documentation version (e.g. 5.0): links pinned to " + "it (en/5.0/…) are rewritten like en/latest ones") + args = parser.parse_args() + + for arg_name in ("site", "plugins", "user-site", "user-plugins", "class-map"): + value = getattr(args, arg_name.replace("-", "_")) + _require_local_path(value, arg_name) + + build(args.site, args.plugins, args.user_site, args.user_plugins, args.class_map, args.version) + + +if __name__ == "__main__": + main() diff --git a/composer.json b/composer.json index 1135d00aa9..0db07f05c8 100644 --- a/composer.json +++ b/composer.json @@ -1,7 +1,8 @@ { "name": "ibexa/documentation-developer", - "description": "This repository is the source for the developer documentation for eZ Platform, an open source CMS based on the Symfony Full Stack Framework in PHP.", + "description": "Ibexa DXP developer documentation as Markdown files, for offline reading and AI coding assistants working inside your project.", "type": "library", + "keywords": ["ibexa", "ibexa-dxp", "documentation", "markdown", "llm", "ai", "agents"], "license": "GNU General Public License v2.0", "autoload-dev": { "psr-4": { @@ -99,7 +100,9 @@ "ibexa/ckeditor-premium": "~5.0.x-dev", "ibexa/measurement": "~5.0.x-dev", "ibexa/connector-actito": "~5.0.x-dev", - "ibexa/fastly": "~5.0.x-dev" + "ibexa/fastly": "~5.0.x-dev", + "ibexa/connect": "~5.0.x-dev", + "ibexa/connector-qualifio": "~5.0.x-dev" }, "scripts": { "fix-cs": [ diff --git a/requirements.txt b/requirements.txt index 8edaf77560..8b40b935a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ mkdocs-macros-plugin==1.3.7 mkdocs-redirects==1.2.2 mkdocs-autolinks-plugin==0.7.1 Jinja2==3.1.6 -mkdocs-llmstxt +mkdocs-llmstxt==0.5.0 -e . diff --git a/tests/python/test_build_package_docs.py b/tests/python/test_build_package_docs.py new file mode 100644 index 0000000000..7977b251da --- /dev/null +++ b/tests/python/test_build_package_docs.py @@ -0,0 +1,312 @@ +import pytest + +from build_package_docs import ( + DocSet, + _require_local_path, + check_relative_doc_links, + rewrite_llms_txt, + rewrite_page, + strip_llms_txt_pointer, +) + +DEVELOPER = DocSet( + root="developer", + base_urls=("https://doc.ibexa.co/en/latest/", "https://doc.ibexa.co/en/5.0/"), + pages=frozenset( + { + "index.md", + "search/search/index.md", + "search/search_api/index.md", + "administration/back_office/back_office_menus/add_menu_item/index.md", + "administration/back_office/back_office_menus/back_office_menus/index.md", + "content_management/images/images/index.md", + } + ), + redirects={"guide/images/": "content_management/images/images/"}, +) + +USER = DocSet( + root="user", + base_urls=( + "https://doc.ibexa.co/projects/userguide/en/latest/", + "https://doc.ibexa.co/projects/userguide/en/5.0/", + ), + pages=frozenset( + { + "index.md", + "image_management/edit_images/index.md", + "getting_started/get_started/index.md", + } + ), + redirects={"getting_started/": "getting_started/get_started/"}, +) + +DOCSETS = (DEVELOPER, USER) + +CLASS_PATHS = { + "Ibexa\\Contracts\\AdminUi\\Tab\\AbstractTab": "ibexa/admin-ui/src/contracts/Tab/AbstractTab.php", + "Ibexa\\Contracts\\Core\\Repository\\SearchService": "ibexa/core/src/contracts/Repository/SearchService.php", +} + + +def rewrite(content, page_path="developer/search/search/index.md"): + return rewrite_page(content, page_path, DOCSETS, CLASS_PATHS) + + +class TestInternalLinks: + def test_page_link_becomes_relative(self): + assert ( + rewrite("See [Search API](https://doc.ibexa.co/en/latest/search/search_api/).") + == "See [Search API](../search_api/index.md)." + ) + + def test_anchor_is_kept(self): + content = "[events](https://doc.ibexa.co/en/latest/administration/back_office/back_office_menus/back_office_menus/#menu-events)" + assert rewrite(content) == ( + "[events](../../administration/back_office/back_office_menus/back_office_menus/index.md#menu-events)" + ) + + def test_site_root_link(self): + assert ( + rewrite("[home](https://doc.ibexa.co/en/latest/)") + == "[home](../../index.md)" + ) + + def test_link_from_set_root_page(self): + assert ( + rewrite("[search](https://doc.ibexa.co/en/latest/search/search/)", + page_path="developer/index.md") + == "[search](search/search/index.md)" + ) + + def test_redirected_url_resolves_through_redirect_map(self): + assert ( + rewrite("[images](https://doc.ibexa.co/en/latest/guide/images/)") + == "[images](../../content_management/images/images/index.md)" + ) + + def test_own_version_is_rewritten_like_latest(self): + assert ( + rewrite("[Search API](https://doc.ibexa.co/en/5.0/search/search_api/)") + == "[Search API](../search_api/index.md)" + ) + + def test_unknown_page_stays_absolute(self): + content = "[gone](https://doc.ibexa.co/en/latest/no/such/page/)" + assert rewrite(content) == content + + def test_other_versions_stay_absolute(self): + content = "[4.6 docs](https://doc.ibexa.co/en/4.6/search/search/)" + assert rewrite(content) == content + + def test_external_links_stay_absolute(self): + content = "[Symfony](https://symfony.com/doc/current/index.html)" + assert rewrite(content) == content + + def test_images_are_not_rewritten(self): + content = "![Admin panel](https://doc.ibexa.co/en/latest/search/search/)" + assert rewrite(content) == content + + def test_links_in_fenced_code_blocks_are_untouched(self): + content = "\n".join( + [ + "```markdown", + "[Search](https://doc.ibexa.co/en/latest/search/search_api/)", + "```", + "[Search](https://doc.ibexa.co/en/latest/search/search_api/)", + ] + ) + assert rewrite(content) == "\n".join( + [ + "```markdown", + "[Search](https://doc.ibexa.co/en/latest/search/search_api/)", + "```", + "[Search](../search_api/index.md)", + ] + ) + + +class TestCrossSetLinks: + def test_developer_page_links_to_user_doc(self): + content = "[edit images](https://doc.ibexa.co/projects/userguide/en/latest/image_management/edit_images/)" + assert rewrite(content) == "[edit images](../../../user/image_management/edit_images/index.md)" + + def test_user_page_links_to_developer_doc(self): + content = "[Search API](https://doc.ibexa.co/en/latest/search/search_api/)" + assert ( + rewrite(content, page_path="user/getting_started/get_started/index.md") + == "[Search API](../../../developer/search/search_api/index.md)" + ) + + def test_user_internal_link_with_version_and_redirect(self): + content = "[start](https://doc.ibexa.co/projects/userguide/en/5.0/getting_started/)" + assert ( + rewrite(content, page_path="user/image_management/edit_images/index.md") + == "[start](../../getting_started/get_started/index.md)" + ) + + def test_developer_link_to_user_doc_with_matching_version(self): + content = "[edit images](https://doc.ibexa.co/projects/userguide/en/5.0/image_management/edit_images/)" + assert rewrite(content) == "[edit images](../../../user/image_management/edit_images/index.md)" + + def test_developer_link_to_user_doc_with_other_version_stays_absolute(self): + content = "[old](https://doc.ibexa.co/projects/userguide/en/4.6/image_management/edit_images/)" + assert rewrite(content) == content + + def test_user_link_to_developer_doc_with_matching_version(self): + content = "[Search API](https://doc.ibexa.co/en/5.0/search/search_api/)" + assert ( + rewrite(content, page_path="user/index.md") + == "[Search API](../developer/search/search_api/index.md)" + ) + + def test_user_link_to_developer_doc_with_other_version_stays_absolute(self): + content = "[old](https://doc.ibexa.co/en/4.6/search/search_api/)" + assert rewrite(content, page_path="user/index.md") == content + + def test_unknown_user_page_stays_absolute(self): + content = "[gone](https://doc.ibexa.co/projects/userguide/en/latest/no/such/)" + assert rewrite(content) == content + + def test_other_projects_stay_absolute(self): + content = "[connect](https://doc.ibexa.co/projects/connect/en/latest/)" + assert rewrite(content) == content + + +class TestApiLinks: + URL = "https://doc.ibexa.co/en/latest/api/php_api/php_api_reference/classes/Ibexa-Contracts-AdminUi-Tab-AbstractTab.html" + + def test_resolved_class_links_to_vendor_source_with_fqcn_text(self): + # From developer/search/search/index.md: 3 dirs inside the package + 2 up to vendor/. + assert rewrite(f"[`AbstractTab`]({self.URL})") == ( + "[`Ibexa\\Contracts\\AdminUi\\Tab\\AbstractTab`]" + "(../../../../../ibexa/admin-ui/src/contracts/Tab/AbstractTab.php)" + ) + + def test_depth_follows_page_location(self): + page = "developer/administration/back_office/back_office_menus/add_menu_item/index.md" + result = rewrite(f"[`AbstractTab`]({self.URL})", page_path=page) + assert result.endswith("(../../../../../../../ibexa/admin-ui/src/contracts/Tab/AbstractTab.php)") + + def test_method_anchor_is_dropped_and_text_kept(self): + url = "https://doc.ibexa.co/en/latest/api/php_api/php_api_reference/classes/Ibexa-Contracts-Core-Repository-SearchService.html" + assert rewrite(f"[`findContent()`]({url}#method_findContent)") == ( + "[`findContent()`](../../../../../ibexa/core/src/contracts/Repository/SearchService.php)" + ) + + def test_unresolved_class_keeps_url_but_gets_fqcn_text(self): + url = "https://doc.ibexa.co/en/latest/api/php_api/php_api_reference/classes/Ibexa-Contracts-Connect-ConnectClientInterface.html" + assert rewrite(f"[`ConnectClientInterface`]({url})") == ( + f"[`Ibexa\\Contracts\\Connect\\ConnectClientInterface`]({url})" + ) + + def test_other_version_api_links_stay_absolute(self): + content = "[`X`](https://doc.ibexa.co/en/4.6/api/php_api/php_api_reference/classes/Ibexa-Contracts-AdminUi-Tab-AbstractTab.html)" + assert rewrite(content) == content + + +class TestLlmsTxt: + def test_page_links_are_relative_to_set_root(self): + content = "- [Search](https://doc.ibexa.co/en/latest/search/search/index.md)" + assert rewrite_llms_txt(content, DEVELOPER) == "- [Search](search/search/index.md)" + + def test_set_root_page(self): + content = "- [Home](https://doc.ibexa.co/en/latest/index.md)" + assert rewrite_llms_txt(content, DEVELOPER) == "- [Home](index.md)" + + def test_user_set_uses_its_own_base(self): + content = "- [Edit images](https://doc.ibexa.co/projects/userguide/en/latest/image_management/edit_images/index.md)" + assert rewrite_llms_txt(content, USER) == "- [Edit images](image_management/edit_images/index.md)" + + def test_unknown_page_stays_absolute(self): + content = "- [Gone](https://doc.ibexa.co/en/latest/no/such/index.md)" + assert rewrite_llms_txt(content, DEVELOPER) == content + + +class TestLlmsTxtPointer: + """Mirrors the shapes inject_page_metadata() (tools/llms_txt/llmstxt_preprocess.py) produces.""" + + def test_pointer_is_removed_without_leaving_a_double_blank_line(self): + content = ( + "# Getting started\n" + "\n" + "> For the complete documentation index, see [llms.txt](https://doc.ibexa.co/en/5.0/llms.txt).\n" + "\n" + "Body text here.\n" + ) + assert strip_llms_txt_pointer(content) == "# Getting started\n\nBody text here.\n" + + def test_description_and_editions_lines_are_kept(self): + content = ( + "# Heading\n" + "\n" + "> For the complete documentation index, see [llms.txt](https://doc.ibexa.co/en/5.0/llms.txt).\n" + "\n" + "Some page description.\n" + "\n" + "Editions: Content, Experience\n" + ) + assert strip_llms_txt_pointer(content) == ( + "# Heading\n\nSome page description.\n\nEditions: Content, Experience\n" + ) + + def test_nested_project_url_is_also_stripped(self): + content = ( + "# Edit images\n" + "\n" + "> For the complete documentation index, see " + "[llms.txt](https://doc.ibexa.co/projects/userguide/en/5.0/llms.txt).\n" + "\n" + "Body.\n" + ) + assert strip_llms_txt_pointer(content) == "# Edit images\n\nBody.\n" + + def test_pointer_without_a_heading_is_stripped_cleanly(self): + content = ( + "> For the complete documentation index, see [llms.txt](https://doc.ibexa.co/en/5.0/llms.txt).\n" + "\n" + "Just body text.\n" + ) + assert strip_llms_txt_pointer(content) == "Just body text.\n" + + +class TestRequireLocalPath: + def test_relative_path_under_cwd_is_accepted(self): + assert _require_local_path("site", "site") == "site" + + def test_nested_relative_path_is_accepted(self): + assert _require_local_path("user-docs/site", "user-site") == "user-docs/site" + + def test_absolute_path_outside_cwd_is_rejected(self): + with pytest.raises(SystemExit): + _require_local_path("/etc/passwd", "site") + + def test_parent_escape_is_rejected(self): + with pytest.raises(SystemExit): + _require_local_path("../../../../etc/passwd", "class-map") + + +class TestSelfCheck: + def test_valid_links_pass(self): + pages = { + "developer/a/index.md": "[ok](../b/index.md) [cross](../../user/c/index.md)", + "developer/b/index.md": "[ok](../a/index.md#anchor)", + "user/c/index.md": "[cross](../../developer/a/index.md)", + } + assert check_relative_doc_links(pages) == [] + + def test_broken_link_is_reported(self): + pages = {"developer/a/index.md": "[broken](../missing/index.md)"} + errors = check_relative_doc_links(pages) + assert len(errors) == 1 + assert "developer/a/index.md" in errors[0] + + def test_link_escaping_package_tree_is_reported(self): + pages = {"developer/a/index.md": "[escape](../../../outside.md)"} + assert len(check_relative_doc_links(pages)) == 1 + + def test_vendor_and_external_links_are_skipped(self): + pages = { + "developer/a/index.md": "[php](../../../ibexa/core/src/S.php) [web](https://example.com/x.md)" + } + assert check_relative_doc_links(pages) == [] diff --git a/tools/llm_package/README.package.md b/tools/llm_package/README.package.md new file mode 100644 index 0000000000..434f4a687c --- /dev/null +++ b/tools/llm_package/README.package.md @@ -0,0 +1,31 @@ +# Ibexa DXP documentation + +Official Ibexa DXP documentation as plain Markdown files. + +## Installation + +Run the following command: + +```bash +composer require --dev ibexa/documentation-developer:~5.0 --no-scripts +``` + +## Content + +This package contains two documentation sets: + +- `developer` contains content of the [developer documentation](https://doc.ibexa.co/en/latest/). +- `user` contains content of the [user documentation](https://doc.ibexa.co/projects/userguide). + +Each set has its own `llms.txt` at its root (`developer/llms.txt`, `user/llms.txt`) with a table of contents with relative links. + +## Use with AI Agents + +To use the documentation with AI agents, you can: + +- use the `ibexa-documentation` Agent skill provided by this package (recommended) +- instruct the agent manually, by mentioning in the prompt: + +``` text +Ibexa DXP documentation is installed locally in vendor/ibexa/documentation-developer/ +``` diff --git a/tools/llm_package/dump_class_paths.php b/tools/llm_package/dump_class_paths.php new file mode 100644 index 0000000000..0d7e04f467 --- /dev/null +++ b/tools/llm_package/dump_class_paths.php @@ -0,0 +1,79 @@ +...] [] + * + * Scans the MkDocs build output(s) for `php_api_reference/classes/.html` + * links, converts each slug to a FQCN (dashes become namespace separators; + * PHP class names cannot contain dashes, so this is unambiguous) and resolves + * it through this repository's Composer autoloader. The resulting JSON maps + * FQCNs to paths relative to vendor/, e.g. + * "Ibexa\\Contracts\\AdminUi\\Tab\\AbstractTab": "ibexa/admin-ui/src/contracts/Tab/AbstractTab.php". + * + * Classes that don't resolve (their package is missing from composer.json) + * are listed on stderr and omitted from the map; build_package_docs.py keeps + * the absolute URL for them. + */ + +declare(strict_types=1); + +$args = array_slice($argv, 1); +$outputPath = count($args) >= 2 ? array_pop($args) : 'class_paths.json'; +$siteDirs = $args !== [] ? $args : ['site']; + +$root = dirname(__DIR__, 2); +$loader = require $root . '/vendor/autoload.php'; +$vendorDir = realpath($root . '/vendor'); + +$slugs = []; +foreach ($siteDirs as $siteDir) { + if (!is_dir($siteDir)) { + fwrite(STDERR, "Site directory not found: $siteDir (run mkdocs build first)\n"); + exit(1); + } + + $iterator = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($siteDir, FilesystemIterator::SKIP_DOTS) + ); + foreach ($iterator as $file) { + if ($file->getExtension() !== 'md') { + continue; + } + $content = file_get_contents($file->getPathname()); + if (preg_match_all('~php_api_reference/classes/([A-Za-z0-9-]+)\.html~', $content, $matches)) { + foreach ($matches[1] as $slug) { + $slugs[$slug] = true; + } + } + } +} +ksort($slugs); + +$map = []; +$unresolved = []; +foreach (array_keys($slugs) as $slug) { + $fqcn = str_replace('-', '\\', $slug); + $path = $loader->findFile($fqcn); + $realPath = $path ? realpath($path) : false; + if ($realPath === false || !str_starts_with($realPath, $vendorDir . DIRECTORY_SEPARATOR)) { + $unresolved[] = $fqcn; + continue; + } + $map[$fqcn] = str_replace(DIRECTORY_SEPARATOR, '/', substr($realPath, strlen($vendorDir) + 1)); +} + +file_put_contents( + $outputPath, + json_encode($map, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n" +); + +printf("Resolved %d/%d API-referenced classes into %s\n", count($map), count($slugs), $outputPath); +if ($unresolved !== []) { + fwrite(STDERR, "Unresolved classes (their links keep absolute URLs):\n"); + foreach ($unresolved as $fqcn) { + fwrite(STDERR, " $fqcn\n"); + } +}