format-pdf.ts

/*
 * format-pdf.ts
 *
 * Copyright (C) 2020-2022 Posit Software, PBC
 */

import { basename, extname, join } from "../../deno_ral/path.ts";

import { mergeConfigs } from "../../core/config.ts";
import { texSafeFilename } from "../../core/tex.ts";

import {
  kBibliography,
  kCapBottom,
  kCapLoc,
  kCapTop,
  kCitationLocation,
  kCiteMethod,
  kClassOption,
  kDefaultImageExtension,
  kDocumentClass,
  kEcho,
  kFigCapLoc,
  kFigDpi,
  kFigFormat,
  kFigHeight,
  kFigWidth,
  kHeaderIncludes,
  kKeepTex,
  kLang,
  kNumberSections,
  kPaperSize,
  kPdfEngine,
  kPdfStandard,
  kPdfStandardApplied,
  kReferenceLocation,
  kShiftHeadingLevelBy,
  kTblCapLoc,
  kTopLevelDivision,
  kWarning,
  pdfStandardEnv,
} from "../../config/constants.ts";
import { warning } from "../../deno_ral/log.ts";
import { asArray } from "../../core/array.ts";
import { Format, FormatExtras, PandocFlags } from "../../config/types.ts";

import { createFormat } from "../formats-shared.ts";

import { RenderedFile, RenderServices } from "../../command/render/types.ts";
import { ProjectConfig, ProjectContext } from "../../project/types.ts";
import { BookExtension } from "../../project/types/book/book-shared.ts";

import { readLines } from "io/read-lines";
import { TempContext } from "../../core/temp.ts";
import { isLatexPdfEngine, pdfEngine } from "../../config/pdf.ts";
import { formatResourcePath } from "../../core/resources.ts";
import { kTemplatePartials } from "../../command/render/template.ts";
import { copyTo } from "../../core/copy.ts";
import { kCodeAnnotations } from "../html/format-html-shared.ts";
import { safeModeFromFile } from "../../deno_ral/fs.ts";
import { hasLevelOneHeadings as hasL1Headings } from "../../core/lib/markdown-analysis/level-one-headings.ts";

export function pdfFormat(): Format {
  return mergeConfigs(
    createPdfFormat("PDF"),
    {
      extensions: {
        book: pdfBookExtension,
      },
    },
  );
}

export function beamerFormat(): Format {
  return createFormat(
    "Beamer",
    "pdf",
    createPdfFormat("Beamer", false, false),
    {
      execute: {
        [kFigWidth]: 10,
        [kFigHeight]: 7,
        [kEcho]: false,
        [kWarning]: false,
      },
      classoption: ["notheorems"],
    },
  );
}

export function latexFormat(displayName: string): Format {
  return createFormat(
    displayName,
    "tex",
    mergeConfigs(
      createPdfFormat(displayName),
      {
        extensions: {
          book: {
            onSingleFilePreRender: (
              format: Format,
              _config?: ProjectConfig,
            ) => {
              // If we're targeting LaTeX output, be sure to keep
              // the supporting files around (since we're not building
              // them into a PDF)
              format.render[kKeepTex] = true;
              return format;
            },
            formatOutputDirectory: () => {
              return "book-latex";
            },
          },
        },
      },
    ),
  );
}

function createPdfFormat(
  displayName: string,
  autoShiftHeadings = true,
  koma = true,
): Format {
  return createFormat(
    displayName,
    "pdf",
    {
      execute: {
        [kFigWidth]: 5.5,
        [kFigHeight]: 3.5,
        [kFigFormat]: "pdf",
        [kFigDpi]: 300,
      },
      pandoc: {
        [kPdfEngine]: "lualatex",
        standalone: true,
        variables: {
          graphics: true,
          tables: true,
        },
        [kDefaultImageExtension]: "pdf",
      },
      metadata: {
        ["block-headings"]: true,
      },
      formatExtras: async (
        _input: string,
        markdown: string,
        flags: PandocFlags,
        format: Format,
        _libDir: string,
        services: RenderServices,
      ) => {
        const extras: FormatExtras = {};

        // only apply extras if this is latex (as opposed to context)
        const engine = pdfEngine(format.pandoc, format.render, flags);
        if (!isLatexPdfEngine(engine)) {
          return extras;
        }

        // Post processed for dealing with latex output
        extras.postprocessors = [
          pdfLatexPostProcessor(flags, format, services.temp),
        ];

        // user may have overridden koma, check for that here
        const documentclass = format.metadata[kDocumentClass] as
          | string
          | undefined;

        const usingCustomTemplates = format.pandoc.template !== undefined ||
          format.metadata[kTemplatePartials] !== undefined;

        if (
          usingCustomTemplates ||
          (documentclass &&
            ![
              "srcbook",
              "scrreprt",
              "scrreport",
              "scrartcl",
              "scrarticle",
            ].includes(
              documentclass,
            ))
        ) {
          koma = false;
        }

        // default to KOMA article class. we do this here rather than
        // above so that projectExtras can override us
        if (koma) {
          // determine caption options
          const captionOptions = [];
          const tblCaploc = tblCapLocation(format);
          captionOptions.push(
            tblCaploc === kCapTop ? "tableheading" : "tablesignature",
          );
          if (figCapLocation(format) === kCapTop) {
            captionOptions.push("figureheading");
          }

          // establish default class options
          const defaultClassOptions = ["DIV=11"];
          if (format.metadata[kLang] !== "de") {
            defaultClassOptions.push("numbers=noendperiod");
          }

          // determine class options (filter by options already set by the user)
          const userClassOptions = format.metadata[kClassOption] as
            | string[]
            | undefined;
          const classOptions = defaultClassOptions.filter((option) => {
            if (Array.isArray(userClassOptions)) {
              const name = option.split("=")[0];
              return !userClassOptions.some((userOption) =>
                String(userOption).startsWith(name + "=")
              );
            } else {
              return true;
            }
          });

          const headerIncludes = [];
          headerIncludes.push(
            "\\KOMAoption{captions}{" + captionOptions.join(",") + "}",
          );

          extras.metadata = {
            [kDocumentClass]: "scrartcl",
            [kClassOption]: classOptions,
            [kPaperSize]: "letter",
            [kHeaderIncludes]: headerIncludes,
          };
        }

        // Provide a custom template for this format
        // Partials can be the one from Quarto division
        const partialNamesQuarto: string[] = [
          "babel-lang",
          "before-bib",
          "biblio",
          "biblio-config",
          "citations",
          "doc-class",
          "graphics",
          "after-body",
          "before-body",
          "pandoc",
          "tables",
          "tightlist",
          "before-title",
          "title",
          "toc",
        ];
        // or the one from Pandoc division (since Pandoc 3.6.3)
        const partialNamesPandoc: string[] = [
          "after-header-includes",
          "common",
          "document-metadata",
          "font-settings",
          "fonts",
          "hypersetup",
          "passoptions",
        ];

        const createTemplateContext = function (
          to: string,
          partialNamesQuarto: string[],
          partialNamesPandoc: string[],
        ) {
          return {
            template: formatResourcePath(to, "pandoc/template.tex"),
            partials: [
              ...partialNamesQuarto.map((name) => {
                return formatResourcePath(to, `pandoc/${name}.tex`);
              }),
              ...partialNamesPandoc.map((name) => {
                return formatResourcePath(to, `pandoc/${name}.latex`);
              }),
            ],
          };
        };
        // Beamer doesn't use document-metadata partial (its template doesn't include it)
        const beamerPartialNamesPandoc = partialNamesPandoc.filter(
          (name) => name !== "document-metadata",
        );
        extras.templateContext = createTemplateContext(
          displayName === "Beamer" ? "beamer" : "pdf",
          partialNamesQuarto,
          displayName === "Beamer"
            ? beamerPartialNamesPandoc
            : partialNamesPandoc,
        );

        // Don't shift the headings if we see any H1s (we can't shift up any longer)
        const hasLevelOneHeadings = await hasL1Headings(markdown);

        // pdfs with no other heading level oriented options get their heading level shifted by -1
        if (
          !hasLevelOneHeadings &&
          autoShiftHeadings &&
          (flags?.[kNumberSections] === true ||
            format.pandoc[kNumberSections] === true) &&
          flags?.[kTopLevelDivision] === undefined &&
          format.pandoc?.[kTopLevelDivision] === undefined &&
          flags?.[kShiftHeadingLevelBy] === undefined &&
          format.pandoc?.[kShiftHeadingLevelBy] === undefined
        ) {
          extras.pandoc = {
            [kShiftHeadingLevelBy]: -1,
          };
        }

        // pdfs with document class scrbook get number sections turned on
        // https://github.com/quarto-dev/quarto-cli/issues/2369
        extras.pandoc = extras.pandoc || {};
        if (
          documentclass === "scrbook" &&
          format.pandoc[kNumberSections] !== false &&
          flags[kNumberSections] !== false
        ) {
          extras.pandoc[kNumberSections] = true;
        }

        // Handle pdf-standard option for PDF/A, PDF/UA, PDF/X conformance
        const pdfStandard = asArray(
          format.render?.[kPdfStandard] ?? format.metadata?.[kPdfStandard] ??
            pdfStandardEnv(),
        );
        if (pdfStandard.length > 0) {
          const { version, standards, needsTagging } =
            normalizePdfStandardForLatex(pdfStandard);
          // Set pdfstandard as a map if there are standards or a version
          if (standards.length > 0 || version) {
            extras.pandoc.variables = extras.pandoc.variables || {};
            const pdfstandardMap: Record<string, unknown> = {};
            if (standards.length > 0) {
              pdfstandardMap.standards = standards;
            }
            if (version) {
              pdfstandardMap.version = version;
            }
            if (needsTagging) {
              pdfstandardMap.tagging = true;
            }
            extras.pandoc.variables["pdfstandard"] = pdfstandardMap;
          }
          // Store applied standards in metadata for verapdf validation
          // (only standards that LaTeX actually supports, not the original list)
          if (standards.length > 0) {
            extras.metadata = extras.metadata || {};
            extras.metadata[kPdfStandardApplied] = standards;
          }
        }

        return extras;
      },
    },
  );
}

const pdfBookExtension: BookExtension = {
  selfContainedOutput: true,
  onSingleFilePostRender: (
    project: ProjectContext,
    renderedFile: RenderedFile,
  ) => {
    // if we have keep-tex then rename the input tex file to match the final output
    // file (but make sure it has a tex-friendly filename)
    if (renderedFile.format.render[kKeepTex]) {
      const finalOutputFile = renderedFile.file!;
      const texOutputFile =
        texSafeFilename(basename(finalOutputFile, extname(finalOutputFile))) +
        ".tex";
      Deno.renameSync(
        join(project.dir, "index.tex"),
        join(project.dir, texOutputFile),
      );
    }
  },
};
type LineProcessor = (line: string) => string | undefined;

function pdfLatexPostProcessor(
  flags: PandocFlags,
  format: Format,
  temp: TempContext,
) {
  return async (output: string) => {
    const lineProcessors: LineProcessor[] = [
      sidecaptionLineProcessor(),
      calloutFloatHoldLineProcessor(),
      tableColumnMarginLineProcessor(),
      guidsProcessor(),
    ];

    if (format.pandoc[kCiteMethod] === "biblatex") {
      lineProcessors.push(bibLatexBibligraphyRefsDivProcessor());
    } else if (format.pandoc[kCiteMethod] === "natbib") {
      lineProcessors.push(
        natbibBibligraphyRefsDivProcessor(
          format.metadata[kBibliography] as string[] | undefined,
        ),
      );
    }

    const marginCites = format.metadata[kCitationLocation] === "margin";
    const renderedCites = {};
    if (marginCites) {
      // Based upon the cite method, post process the file to
      // process unresolved citations
      if (format.pandoc[kCiteMethod] === "biblatex") {
        lineProcessors.push(suppressBibLatexBibliographyLineProcessor());
        lineProcessors.push(bibLatexCiteLineProcessor());
      } else if (format.pandoc[kCiteMethod] === "natbib") {
        lineProcessors.push(suppressNatbibBibliographyLineProcessor());
        lineProcessors.push(natbibCiteLineProcessor());
      } else {
        // If this is using the pandoc default citeproc, we need to
        // do a more complex processing, since it is generating raw latex
        // for the citations (not running a tool in the pdf chain to
        // generate the bibliography). As a result, we first read the
        // rendered bibliography, indexing the entring and removing it
        // from the latex, then we run a second pass where we use that index
        // to replace cites with the rendered versions.
        lineProcessors.push(
          indexAndSuppressPandocBibliography(renderedCites),
          cleanReferencesChapter(),
        );
      }
    }

    // Move longtable captions below if requested
    if (tblCapLocation(format) === kCapBottom) {
      lineProcessors.push(longtableBottomCaptionProcessor());
    }

    // If enabled, switch to sidenote footnotes
    if (marginRefs(flags, format)) {
      // Replace notes with side notes
      lineProcessors.push(sideNoteLineProcessor());
    }
    lineProcessors.push(captionFootnoteLineProcessor());

    if (
      format.metadata[kCodeAnnotations] as boolean !== false &&
      format.metadata[kCodeAnnotations] as string !== "none"
    ) {
      lineProcessors.push(codeAnnotationPostProcessor());
      lineProcessors.push(codeListAnnotationPostProcessor());
    }

    lineProcessors.push(tableSidenoteProcessor());

    // This is pass 1
    await processLines(output, lineProcessors, temp);

    // This is pass 2; we need these to happen after the first pass
    const pass2Processors: LineProcessor[] = [
      longTableSidenoteProcessor(),
    ];
    if (Object.keys(renderedCites).length > 0) {
      pass2Processors.push(placePandocBibliographyEntries(renderedCites));
    }
    await processLines(output, pass2Processors, temp);
  };
}

function tblCapLocation(format: Format) {
  return format.metadata[kTblCapLoc] || format.metadata[kCapLoc] || kCapTop;
}

function figCapLocation(format: Format) {
  return format.metadata[kFigCapLoc] || format.metadata[kCapLoc] || kCapBottom;
}

function marginRefs(flags: PandocFlags, format: Format) {
  return format.pandoc[kReferenceLocation] === "margin" ||
    flags[kReferenceLocation] === "margin";
}

// Processes the lines of an input file, processing each line
// and replacing the input file with the processed output file
async function processLines(
  inputFile: string,
  lineProcessors: LineProcessor[],
  temp: TempContext,
) {
  // The temp file we generate into
  const outputFile = temp.createFile({ suffix: ".tex" });
  const file = await Deno.open(inputFile);
  // Preserve the existing permissions as we'll replace
  const mode = safeModeFromFile(inputFile);
  try {
    for await (const line of readLines(file)) {
      let processedLine: string | undefined = line;
      // Give each processor a shot at the line
      for (const processor of lineProcessors) {
        if (processedLine !== undefined) {
          processedLine = processor(processedLine);
        }
      }

      // skip lines that a processor has 'eaten'
      if (processedLine !== undefined) {
        Deno.writeTextFileSync(outputFile, processedLine + "\n", {
          append: true,
          mode,
        });
      }
    }
  } finally {
    file.close();

    // Always overwrite the input file with an incompletely processed file
    // which should make debugging the error easier (I hope)
    copyTo(outputFile, inputFile);
  }
}

const kBeginScanRegex = /^%quartopost-sidecaption-206BE349/;
const kEndScanRegex = /^%\/quartopost-sidecaption-206BE349/;

const sidecaptionLineProcessor = () => {
  let state: "scanning" | "replacing" = "scanning";
  return (line: string): string | undefined => {
    switch (state) {
      case "scanning":
        if (line.match(kBeginScanRegex)) {
          state = "replacing";
          return kbeginLongTablesideCap;
        } else {
          return line;
        }

      case "replacing":
        if (line.match(kEndScanRegex)) {
          state = "scanning";
          return kEndLongTableSideCap;
        } else {
          return line;
        }
    }
  };
};

// Reads the first command encountered as a balanced command
// (e.g. \caption{...} or \footnote{...}) and returns
// the complete command
//
// This expects the latex string to start with the command
const readBalancedCommand = (latex: string) => {
  let braceCount = 0;
  let entered = false;
  const chars: string[] = [];
  for (let i = 0; i < latex.length; i++) {
    const char = latex.charAt(i);
    if (char === "{") {
      braceCount++;
      entered = true;
    } else if (char === "}") {
      braceCount--;
    }

    chars.push(char);
    if (entered && braceCount === 0) {
      break;
    }
  }
  return chars.join("");
};

// Process element caption footnotes on a latex string
// This expects a latex elements with a `\caption{}`
//
// It will extract footnotes from the caption and replace
// them with a footnote mark and position the footnote
// below the latex element (e.g. it will remove the footnote
// from the element and then return the footnote below
// the element)
const processElementCaptionFootnotes = (latexFigure: string) => {
  const footnoteMark = "\\footnote{";
  const captionMark = "\\caption{";

  // Contents holds the final contents that will be returned
  // after being joined. This function will append to contents
  // to build up the final output
  const contents: string[] = [];

  // Read up to the caption itself
  const captionIndex = latexFigure.indexOf(captionMark);
  if (captionIndex > -1) {
    // Slice off the figure up to the caption
    contents.push(latexFigure.substring(0, captionIndex));
    const captionStartStr = latexFigure.slice(captionIndex);

    // Read the caption
    const captionLatex = readBalancedCommand(captionStartStr);
    const figureSuffix = captionStartStr.slice(captionLatex.length);

    // Slice off the command prefix and suffix
    let captionContents = captionLatex.slice(
      captionMark.length,
      captionLatex.length - 1,
    );

    // Deal with footnotes in the caption
    let footNoteIndex = captionContents.indexOf(footnoteMark);
    if (footNoteIndex > -1) {
      // Caption text will not have any footnotes in it
      const captionText: string[] = [];
      // Caption with note will have footnotemarks in it
      const captionWithNote: string[] = [];
      // The footnotes that we found along the way
      const footNotes: string[] = [];
      while (footNoteIndex > -1) {
        // capture any prefix
        const prefix = captionContents.substring(0, footNoteIndex);
        captionContents = captionContents.slice(footNoteIndex);

        // push the prefix onto the captions
        captionText.push(prefix);
        captionWithNote.push(prefix);

        // process the footnote
        const footnoteLatex = readBalancedCommand(captionContents);
        captionContents = captionContents.slice(footnoteLatex.length);
        footNoteIndex = captionContents.indexOf(footnoteMark);

        // Capture the footnote and place a footnote mark in the caption
        captionWithNote.push("\\footnotemark{}");
        footNotes.push(
          footnoteLatex.slice(footnoteMark.length, footnoteLatex.length - 1),
        );
      }
      // Push any leftovers onto the caption contents
      captionText.push(captionContents);
      captionWithNote.push(captionContents);

      // push the caption onto the contents
      contents.push(
        `\\caption[${captionText.join("")}]{${captionWithNote.join("")}}`,
      );

      // push the suffix onto the contents
      contents.push(figureSuffix);

      // push the footnotes on the contents
      contents.push("\n");

      // Add a proper footnote counter offset, if necessary
      if (footNotes.length > 1) {
        contents.push(`\\addtocounter{footnote}{-${footNotes.length - 1}}`);
      }

      for (let i = 0; i < footNotes.length; i++) {
        contents.push(`\\footnotetext{${footNotes[i]}}`);
        if (footNotes.length > 1 && i < footNotes.length - 1) {
          contents.push(`\\addtocounter{footnote}{1}`);
        }
      }
      return contents.join("");
    } else {
      // No footnotes in the caption, just leave it alone
      return latexFigure;
    }
  } else {
    // No caption means just let it go
    return latexFigure;
  }
};

const kMatchLongTableSize = /^(.*)p{\(\\columnwidth - (\d+\\tabcolsep\).*$)/;

const kStartLongTable = /^\\begin{longtable}/;
const kEndLongTable = /^\\end{longtable}/;

const guidsProcessor = () => {
  let state: "looking-for-definition-start" | "looking-for-definition-end" =
    "looking-for-definition-start";
  const guidDefinitions: [string, string][] = [];
  let guidBeingProcessed: string | undefined;
  let guidContents: string[] = [];
  return (line: string): string | undefined => {
    switch (state) {
      case "looking-for-definition-start": {
        if (line.startsWith("%quarto-define-uuid: ")) {
          state = "looking-for-definition-end";
          line = line.replace(/^%quarto-define-uuid:\s*/, "");
          guidBeingProcessed = line.trim();
          return undefined;
        }
        for (const [key, value] of guidDefinitions) {
          line = line.replaceAll(key, value);
        }
        return line;
      }
      case "looking-for-definition-end": {
        if (line === "%quarto-end-define-uuid") {
          state = "looking-for-definition-start";
          if (guidBeingProcessed === undefined) {
            throw new Error("guidBeingProcessed is undefined");
          }
          guidDefinitions.push([
            guidBeingProcessed,
            guidContents.join("").trim(),
          ]);
          guidContents = [];
          guidBeingProcessed = undefined;
          return undefined;
        } else {
          guidContents.push(line);
          return undefined;
        }
      }
    }
  };
};

const tableColumnMarginLineProcessor = () => {
  let state: "looking-for-boundaries" | "looking-for-tables" | "processing" =
    "looking-for-boundaries";
  return (line: string): string | undefined => {
    switch (state) {
      case "looking-for-boundaries": {
        if (line === "% quarto-tables-in-margin-AB1927C9:begin") {
          state = "looking-for-tables";
          return undefined;
        }
        return line;
      }
      case "looking-for-tables": {
        if (line.match(kStartLongTable)) {
          state = "processing";
          return line;
        } else if (line === "% quarto-tables-in-margin-AB1927C9:end") {
          state = "looking-for-boundaries";
          return undefined;
        }
        return line;
      }
      case "processing": {
        if (line.match(kEndLongTable)) {
          state = "looking-for-tables";
          return line;
        } else {
          const match = line.match(kMatchLongTableSize);
          if (match) {
            return `${
              match[1]
            }p{(\\marginparwidth + \\marginparsep + \\columnwidth - ${
              match[2]
            }`;
          } else {
            return line;
          }
        }
      }
      default: {
        return line;
      }
    }
  };
};

const captionFootnoteLineProcessor = () => {
  let state: "scanning" | "capturing" = "scanning";
  let capturedLines: string[] = [];
  return (line: string): string | undefined => {
    switch (state) {
      case "scanning":
        if (line.match(/^\\begin{figure}.*$/)) {
          state = "capturing";
          capturedLines = [line];
          return undefined;
        } else {
          return line;
        }
      case "capturing":
        capturedLines.push(line);
        if (line.match(/^\\end{figure}%*$/)) {
          state = "scanning";

          // read the whole figure and clear any capture state
          const lines = capturedLines.join("\n");
          capturedLines = [];

          // Process the captions and relocate footnotes
          return processElementCaptionFootnotes(lines);
        } else {
          return undefined;
        }
    }
  };
};

const processSideNotes = (endMarker: string) => {
  return (latexLongTable: string) => {
    const sideNoteMarker = "\\sidenote{\\footnotesize ";
    let strProcessing = latexLongTable;
    const strOutput: string[] = [];
    const sidenotes: string[] = [];

    let sidenotePos = strProcessing.indexOf(sideNoteMarker);
    while (sidenotePos > -1) {
      strOutput.push(strProcessing.substring(0, sidenotePos));

      const remainingStr = strProcessing.substring(
        sidenotePos + sideNoteMarker.length,
      );
      let escaped = false;
      let sideNoteEnd = -1;
      for (let i = 0; i < remainingStr.length; i++) {
        const ch = remainingStr[i];
        if (ch === "\\") {
          escaped = true;
        } else {
          if (!escaped && ch === "}") {
            sideNoteEnd = i;
            break;
          } else {
            escaped = false;
          }
        }
      }

      if (sideNoteEnd > -1) {
        strOutput.push("\\sidenotemark{}");
        const contents = remainingStr.substring(0, sideNoteEnd);
        sidenotes.push(contents);
        strProcessing = remainingStr.substring(sideNoteEnd + 1);
        sidenotePos = strProcessing.indexOf(sideNoteMarker);
      } else {
        strOutput.push(remainingStr);
      }
    }

    // Ensure that we inject sidenotes after the longtable
    const endTable = endMarker;
    const endPos = strProcessing.indexOf(endTable);
    const prefix = strProcessing.substring(0, endPos + endTable.length);
    const suffix = strProcessing.substring(
      endPos + endTable.length,
      strProcessing.length,
    );

    strOutput.push(prefix);
    for (const note of sidenotes) {
      strOutput.push(`\\sidenotetext{${note}}\n`);
    }
    if (suffix) {
      strOutput.push(suffix);
    }

    return strOutput.join("");
  };
};

const processLongTableSidenotes = processSideNotes("\\end{longtable}");
const processTableSidenotes = processSideNotes("\\end{table}");

const sideNoteProcessor = (
  beginRegex: RegExp,
  endRegex: RegExp,
  callback: (str: string) => string,
) => {
  return () => {
    let state: "scanning" | "capturing" = "scanning";
    let capturedLines: string[] = [];
    return (line: string): string | undefined => {
      switch (state) {
        case "scanning":
          if (line.match(beginRegex)) {
            state = "capturing";
            capturedLines = [line];
            return undefined;
          } else {
            return line;
          }
        case "capturing":
          capturedLines.push(line);
          if (line.match(endRegex)) {
            state = "scanning";

            // read the whole figure and clear any capture state
            const lines = capturedLines.join("\n");
            capturedLines = [];

            // Process the captions and relocate footnotes
            return callback(lines);
          } else {
            return undefined;
          }
      }
    };
  };
};
const longTableSidenoteProcessor = sideNoteProcessor(
  /^\\begin{longtable}.*$/,
  /^\\end{longtable}%*$/,
  processLongTableSidenotes,
);

const tableSidenoteProcessor = sideNoteProcessor(
  /^\\begin{table}.*$/,
  /^\\end{table}%*$/,
  processTableSidenotes,
);

const calloutFloatHoldLineProcessor = () => {
  let state: "scanning" | "replacing" = "scanning";
  return (line: string): string | undefined => {
    switch (state) {
      case "scanning":
        if (line.match(/^\\begin{tcolorbox}/)) {
          state = "replacing";
          return line;
        } else {
          return line;
        }

      case "replacing":
        if (line.match(/^\\end{tcolorbox}/)) {
          state = "scanning";
          return line;
        } else if (line.match(/^\\begin{figure}$/)) {
          return "\\begin{figure}[H]";
        } else if (line.match(/^\\begin{codelisting}$/)) {
          return "\\begin{codelisting}[H]";
        } else {
          return line;
        }
    }
  };
};

const kQuartoBibPlaceholderRegex = "%bib-loc-124C8010";
const bibLatexBibligraphyRefsDivProcessor = () => {
  let hasRefsDiv = false;
  return (line: string): string | undefined => {
    if (line === kQuartoBibPlaceholderRegex) {
      if (!hasRefsDiv) {
        hasRefsDiv = true;
        return "\\printbibliography[heading=none]";
      } else {
        // already seen a refs div, just ignore this one
        return undefined;
      }
    } else if (hasRefsDiv && line.match(/^\\printbibliography$/)) {
      return undefined;
    } else {
      return line;
    }
  };
};

const natbibBibligraphyRefsDivProcessor = (bibs?: string[]) => {
  let hasRefsDiv = false;
  return (line: string): string | undefined => {
    if (line === kQuartoBibPlaceholderRegex) {
      if (bibs && !hasRefsDiv) {
        hasRefsDiv = true;
        return `\\renewcommand{\\bibsection}{}\n\\bibliography{${
          bibs.join(",")
        }}`;
      } else {
        // already seen a refs div, just ignore this one
        return undefined;
      }
    } else if (hasRefsDiv && line.match(/^\s*\\bibliography{.*}$/)) {
      return undefined;
    } else {
      return line;
    }
  };
};

// Removes the biblatex \printbibiliography command
const suppressBibLatexBibliographyLineProcessor = () => {
  return (line: string): string | undefined => {
    if (line.match(/^\\printbibliography$/)) {
      return "";
    }
    return line;
  };
};

// Replaces the natbib bibligography declaration with a version
// that will not be printed in the PDF
const suppressNatbibBibliographyLineProcessor = () => {
  return (line: string): string | undefined => {
    return line.replace(/^\s*\\bibliography{(.*)}$/, (_match, bib) => {
      return `\\newsavebox\\mytempbib
\\savebox\\mytempbib{\\parbox{\\textwidth}{\\bibliography{${bib}}}}`;
    });
  };
};

// {?quarto-cite:(id)}
const kQuartoCiteRegex = /{\?quarto-cite:(.*?)}/g;
const bibLatexCiteLineProcessor = () => {
  return (line: string): string | undefined => {
    return line.replaceAll(kQuartoCiteRegex, (_match, citeKey) => {
      return `\\fullcite{${citeKey}}`;
    });
  };
};

const natbibCiteLineProcessor = () => {
  return (line: string): string | undefined => {
    return line.replaceAll(kQuartoCiteRegex, (_match, citeKey) => {
      return `\\bibentry{${citeKey}}`;
    });
  };
};

const sideNoteLineProcessor = () => {
  return (line: string): string | undefined => {
    return line.replaceAll(/\\footnote{/g, "\\sidenote{\\footnotesize ");
  };
};

const longtableBottomCaptionProcessor = () => {
  let scanning = false;
  let capturing = false;
  let caption: string | undefined;

  return (line: string): string | undefined => {
    const isEndOfDocument = !!line.match(/^\\end{document}/);
    if (isEndOfDocument && caption) {
      return `${caption}\n${line}`;
    } else if (scanning) {
      // look for a caption line
      if (capturing) {
        caption = `${caption}\n${line}`;
        capturing = !line.match(/\\tabularnewline$/);
        return undefined;
      } else {
        if (
          line.match(/^\\caption.*?\\tabularnewline$/) ||
          line.match(/^\\caption{.*}\\\\$/)
        ) {
          caption = line;
          return undefined;
        } else if (line.match(/^\\caption.*?/)) {
          caption = line;
          capturing = true;
          return undefined;
        } else if (line.match(/^\\endlastfoot/) && caption) {
          line = `\\tabularnewline\n${caption}\n${line}`;
          caption = undefined;
          return line;
        } else if (line.match(/^\\end{longtable}$/)) {
          scanning = false;
          if (caption) {
            line = caption + "\n" + line;
            caption = undefined;
            return line;
          }
        }
      }
    } else {
      scanning = !!line.match(/^\\begin{longtable}/);
    }

    return line;
  };
};

const kChapterRefNameRegex = /^\\chapter\*?{(.*?)}\\label{references.*?}$/;
const cleanReferencesChapter = () => {
  let refChapterName: string | undefined;
  let refChapterContentsRegex: RegExp | undefined;
  let refChapterMarkRegex: RegExp | undefined;

  return (line: string): string | undefined => {
    const chapterRefMatch = line.match(kChapterRefNameRegex);
    if (chapterRefMatch) {
      refChapterName = chapterRefMatch[1];
      refChapterContentsRegex = new RegExp(
        `\\\\addcontentsline{toc}{chapter}{${refChapterName}}`,
      );
      refChapterMarkRegex = new RegExp(
        `\\\\markboth{${refChapterName}}{${refChapterName}}`,
      );
      // Eat this line
      return undefined;
    } else if (refChapterContentsRegex && line.match(refChapterContentsRegex)) {
      // Eat this line
      return undefined;
    } else if (refChapterMarkRegex && line.match(refChapterMarkRegex)) {
      // Eat this line
      return undefined;
    }
    return line;
  };
};

const indexAndSuppressPandocBibliography = (
  renderedCites: Record<string, string[]>,
) => {
  let readingBibliography = false;
  let currentCiteKey: string | undefined = undefined;

  return (line: string): string | undefined => {
    if (
      !readingBibliography &&
      line.match(/^(\\protect)?\\phantomsection\\label{refs}$/)
    ) {
      readingBibliography = true;
      return undefined;
    } else if (readingBibliography && line.match(/^\\end{CSLReferences}$/)) {
      readingBibliography = false;
      return undefined;
    } else if (readingBibliography) {
      const matches = line.match(/\\bibitem\[\\citeproctext\]{ref\-(.*?)}/);
      if (matches && matches[1]) {
        currentCiteKey = matches[1];
        renderedCites[currentCiteKey] = [line];
      } else if (line.length === 0) {
        currentCiteKey = undefined;
      } else if (currentCiteKey) {
        renderedCites[currentCiteKey].push(line);
      }
    }

    if (readingBibliography) {
      return undefined;
    } else {
      return line;
    }
  };
};

const kInSideCaptionRegex = /^\\sidecaption{/;
const kBeginFigureRegex = /^\\begin{figure}\[.*?\]$/;
const kEndFigureRegex = /^\\end{figure}\%?$/;

const placePandocBibliographyEntries = (
  renderedCites: Record<string, string[]>,
) => {
  let biblioEntryState: "scanning" | "in-figure" | "in-sidecaption" =
    "scanning";
  let pendingCiteKeys: string[] = [];

  return (line: string): string | undefined => {
    switch (biblioEntryState) {
      case "scanning": {
        if (line.match(kBeginFigureRegex)) {
          biblioEntryState = "in-figure";
        }
        break;
      }
      case "in-figure": {
        if (line.match(kInSideCaptionRegex)) {
          biblioEntryState = "in-sidecaption";
        } else {
          if (line.match(kEndFigureRegex)) {
            biblioEntryState = "scanning";
          }
        }
        break;
      }
      case "in-sidecaption": {
        if (line.match(kEndFigureRegex)) {
          biblioEntryState = "scanning";
        }
        break;
      }
      default:
        break;
    }

    if (biblioEntryState === "scanning" && pendingCiteKeys.length > 0) {
      const result = [
        line,
        "\n\\begin{CSLReferences}{2}{0}",
        ...pendingCiteKeys,
        "\\end{CSLReferences}\n",
      ].join("\n");
      pendingCiteKeys = [];
      return result;
    }

    return line.replaceAll(kQuartoCiteRegex, (_match, citeKey) => {
      const citeLines = renderedCites[citeKey];
      if (citeLines) {
        if (biblioEntryState === "in-sidecaption" && citeLines.length > 0) {
          pendingCiteKeys.push(citeLines[0]);
          return ["", ...citeLines.slice(1)].join("\n");
        } else {
          return [
            "\n\\begin{CSLReferences}{2}{0}",
            ...citeLines,
            "\\end{CSLReferences}\n",
          ].join("\n");
        }
      } else {
        return citeKey;
      }
    });
  };
};

const kCodeAnnotationRegex =
  /(.*)\\CommentTok\{(.*?)[^\s]+? +\\textless\{\}(\d+)\\textgreater\{\}.*\}$/gm;
const kCodePlainAnnotationRegex = /(.*)% \((\d+)\)$/g;
const codeAnnotationPostProcessor = () => {
  let lastAnnotation: string | undefined;

  return (line: string): string | undefined => {
    if (line === "\\begin{Shaded}") {
      lastAnnotation = undefined;
    }

    // Replace colorized code
    line = line.replaceAll(
      kCodeAnnotationRegex,
      (_match, prefix: string, comment: string, annotationNumber: string) => {
        if (annotationNumber !== lastAnnotation) {
          lastAnnotation = annotationNumber;
          if (comment.length > 0) {
            // There is something else inside the comment line so
            // We need to recreate the comment line without the annotation
            prefix = `${prefix}\\CommentTok\{${comment}\}`;
          }
          return `${prefix}\\hspace*{\\fill}\\NormalTok{\\circled{${annotationNumber}}}`;
        } else {
          return `${prefix}`;
        }
      },
    );

    // Replace plain code
    line = line.replaceAll(
      kCodePlainAnnotationRegex,
      (_match, prefix: string, annotationNumber: string) => {
        if (annotationNumber !== lastAnnotation) {
          lastAnnotation = annotationNumber;

          const replaceValue = `(${annotationNumber})`;
          const paddingNumber = Math.max(
            0,
            75 - prefix.length - replaceValue.length,
          );
          const padding = " ".repeat(paddingNumber);
          return `${prefix}${padding}${replaceValue}`;
        } else {
          return `${prefix}`;
        }
      },
    );

    return line;
  };
};

const kListAnnotationRegex = /(.*)5CB6E08D-list-annote-(\d+)(.*)/g;
const codeListAnnotationPostProcessor = () => {
  return (line: string): string | undefined => {
    return line.replaceAll(
      kListAnnotationRegex,
      (_match, prefix: string, annotationNumber: string, suffix: string) => {
        return `${prefix}\\circled{${annotationNumber}}${suffix}`;
      },
    );
  };
};

const kbeginLongTablesideCap = `{
\\makeatletter
\\def\\LT@makecaption#1#2#3{%
  \\noalign{\\smash{\\hbox{\\kern\\textwidth\\rlap{\\kern\\marginparsep
  \\parbox[t]{\\marginparwidth}{%
    \\footnotesize{%
      \\vspace{(1.1\\baselineskip)}
    #1{#2: }\\ignorespaces #3}}}}}}%
    }
\\makeatother`;

const kEndLongTableSideCap = "}";

// LaTeX-supported PDF standards (from latex3/latex2e DocumentMetadata)
// See: https://github.com/latex3/latex2e - documentmetadata-support.dtx
const kLatexSupportedStandards = new Set([
  // PDF/A standards (note: a-1a is NOT supported, only a-1b)
  "a-1b",
  "a-2a",
  "a-2b",
  "a-2u",
  "a-3a",
  "a-3b",
  "a-3u",
  "a-4",
  "a-4e",
  "a-4f",
  // PDF/X standards
  "x-4",
  "x-4p",
  "x-5g",
  "x-5n",
  "x-5pg",
  "x-6",
  "x-6n",
  "x-6p",
  // PDF/UA standards (only ua-2 is supported by LaTeX)
  "ua-2",
]);

// Standards that require PDF tagging (document structure)
// - PDF/A level "a" variants require tagged structure per PDF/A spec
// - PDF/UA standards require tagging for universal accessibility
//   (LaTeX does NOT automatically enable tagging for UA standards)
const kTaggingRequiredStandards = new Set([
  "a-2a",
  "a-3a",
  "ua-1",
  "ua-2",
]);

const kVersionPattern = /^(1\.[4-7]|2\.0)$/;

// PDF version required by each standard (maximum version limits)
// LaTeX defaults to PDF 2.0 with \DocumentMetadata, but some standards
// have maximum version requirements that are incompatible with 2.0
// Note: a-1a is intentionally omitted as LaTeX doesn't support it
const kStandardRequiredVersion: Record<string, string> = {
  // PDF/A-1 requires exactly PDF 1.4 (only a-1b supported by LaTeX)
  "a-1b": "1.4",
  // PDF/A-2 and PDF/A-3 have maximum version of 1.7
  "a-2a": "1.7",
  "a-2b": "1.7",
  "a-2u": "1.7",
  "a-3a": "1.7",
  "a-3b": "1.7",
  "a-3u": "1.7",
  // PDF/A-4, PDF/UA-1, PDF/UA-2 all work with PDF 2.0 (the default)
};

function normalizePdfStandardForLatex(
  standards: unknown[],
): { version?: string; standards: string[]; needsTagging: boolean } {
  let version: string | undefined;
  const result: string[] = [];
  let needsTagging = false;

  for (const s of standards) {
    // Convert to string - YAML may parse versions like 2.0 as integer 2
    let str: string;
    if (typeof s === "number") {
      // Handle YAML numeric parsing: integer 2 -> "2.0", float 1.4 -> "1.4"
      str = Number.isInteger(s) ? `${s}.0` : String(s);
    } else if (typeof s === "string") {
      str = s;
    } else {
      continue;
    }
    // Normalize: lowercase, remove any "pdf" prefix
    const normalized = str.toLowerCase().replace(/^pdf[/-]?/, "");

    if (kVersionPattern.test(normalized)) {
      // Use first explicit version (ignore subsequent ones)
      if (!version) {
        version = normalized;
      }
    } else if (kLatexSupportedStandards.has(normalized)) {
      // LaTeX is case-insensitive, pass through lowercase
      result.push(normalized);
      // Check if this standard requires tagging
      if (kTaggingRequiredStandards.has(normalized)) {
        needsTagging = true;
      }
      // Infer required PDF version from standard (if not explicitly set)
      if (!version && kStandardRequiredVersion[normalized]) {
        version = kStandardRequiredVersion[normalized];
      }
    } else {
      warning(
        `PDF standard '${s}' is not supported by LaTeX and will be ignored`,
      );
    }
  }

  return { version, standards: result, needsTagging };
}