feat(file-upload): add chunkr_create_task_req_payload to upload file request

Author: skeptrunedev

clients/ts-sdk/openapi.json

@@ -17,7 +17,7 @@
   "files": [
     "dist"
   ],
-  "version": "0.0.73",
+  "version": "0.0.74",
   "license": "MIT",
   "scripts": {
     "lint": "eslint 'src/**/*.ts'",

clients/ts-sdk/package.json

@@ -11,9 +11,14 @@ import { EXAMPLE_FILE_ID, TRIEVE } from "../../__tests__/constants";
 import fs from "fs";
 import { test } from "../../__tests__/utils";
 
-const file = fs.readFileSync("./src/__tests__/uploadme.pdf");
+const uploadMeFile = fs.readFileSync("./src/__tests__/uploadme.pdf");
+const uploadMeFileEncoded = uploadMeFile.toString("base64");
 
-const fileEncoded = file.toString("base64");
+const villageOfCatskillZoningRegulationsFile = fs.readFileSync(
+  "./src/__tests__/Village_of_Catskill_Zoning_Regulations.pdf",
+);
+const villageOfCatskillZoningRegulationsFileEncoded =
+  villageOfCatskillZoningRegulationsFile.toString("base64");
 
 describe("File Tests", async () => {
   let trieve: TrieveSDK;
@@ -22,13 +27,25 @@ describe("File Tests", async () => {
   });
   test("uploadFile", async () => {
     const data = await trieve.uploadFile({
-      base64_file: fileEncoded,
+      base64_file: uploadMeFileEncoded,
       file_name: "uploadme.pdf",
       group_tracking_id: "file-upload-group",
     });
     expectTypeOf(data).toEqualTypeOf<UploadFileResponseBody>();
   });
 
+  test("uploadFileWithChunkr", async () => {
+    const data = await trieve.uploadFile({
+      base64_file: villageOfCatskillZoningRegulationsFileEncoded,
+      file_name: "Village_of_Catskill_Zoning_Regulations.pdf",
+      group_tracking_id: "village-of-catskill-file-upload-group",
+      chunkr_create_task_req_payload: {
+        pipeline: "Chunkr",
+      },
+    });
+    expectTypeOf(data).toEqualTypeOf<UploadFileResponseBody>();
+  });
+
   test("createPresignedUrlForJsonl", async () => {
     const data = await trieve.createPresignedUrlForCsvJsonl({
       file_name: "flipkart.jsonl",

clients/ts-sdk/src/__tests__/Village_of_Catskill_Zoning_Regulations.pdf

@@ -73,6 +73,29 @@ export type AuthQuery = {
     redirect_uri?: (string) | null;
 };
 
+/**
+ * Controls the processing and generation for the segment.
+ * - `crop_image` controls whether to crop the file's images to the segment's bounding box.
+ * The cropped image will be stored in the segment's `image` field. Use `All` to always crop,
+ * or `Auto` to only crop when needed for post-processing.
+ * - `html` is the HTML output for the segment, generated either through huerstics (`Auto`) or using Chunkr fine-tuned models (`LLM`)
+ * - `llm` is the LLM-generated output for the segment, this uses off-the-shelf models to generate a custom output for the segment
+ * - `markdown` is the Markdown output for the segment, generated either through huerstics (`Auto`) or using Chunkr fine-tuned models (`LLM`)
+ * - `embed_sources` defines which content sources will be included in the chunk's embed field and counted towards the chunk length.
+ * The array's order determines the sequence in which content appears in the embed field (e.g., [Markdown, LLM] means Markdown content
+ * is followed by LLM content). This directly affects what content is available for embedding and retrieval.
+ */
+export type AutoGenerationConfig = {
+    crop_image?: (CroppingStrategy);
+    embed_sources?: Array<EmbedSource>;
+    html?: (GenerationStrategy);
+    /**
+     * Prompt for the LLM mode
+     */
+    llm?: (string) | null;
+    markdown?: (GenerationStrategy);
+};
+
 export type AutocompleteReqPayload = {
     /**
      * Set content_only to true to only returning the chunk_html of the chunks. This is useful for when you want to reduce amount of data over the wire for latency improvement (typically 10-50ms). Default is false.
@@ -395,6 +418,22 @@ export type ChunkMetadataWithScore = {
     weight: number;
 };
 
+/**
+ * Controls the setting for the chunking and post-processing of each chunk.
+ */
+export type ChunkProcessing = {
+    /**
+     * Whether to ignore headers and footers in the chunking process.
+     * This is recommended as headers and footers break reading order across pages.
+     */
+    ignore_headers_and_footers?: boolean;
+    /**
+     * The target number of words in each chunk. If 0, each chunk will contain a single segment.
+     */
+    target_length?: number;
+    tokenizer?: (TokenizerType);
+};
+
 /**
  * Request payload for creating a new chunk
  */
@@ -834,6 +873,26 @@ export type CreateDatasetReqPayload = {
     tracking_id?: (string) | null;
 };
 
+/**
+ * Will use [chunkr.ai](https://chunkr.ai) to process the file when this object is defined. See [docs.chunkr.ai/api-references/task/create-task](https://docs.chunkr.ai/api-references/task/create-task) for detailed information about what each field on this request payload does.
+ */
+export type CreateFormWithoutFile = {
+    chunk_processing?: ((ChunkProcessing) | null);
+    /**
+     * The number of seconds until task is deleted.
+     * Expried tasks can **not** be updated, polled or accessed via web interface.
+     */
+    expires_in?: (number) | null;
+    /**
+     * Whether to use high-resolution images for cropping and post-processing. (Latency penalty: ~7 seconds per page)
+     */
+    high_resolution?: (boolean) | null;
+    ocr_strategy?: ((OcrStrategy) | null);
+    pipeline?: ((PipelineType) | null);
+    segment_processing?: ((SegmentProcessing) | null);
+    segmentation_strategy?: ((SegmentationStrategy) | null);
+};
+
 export type CreateMessageReqPayload = {
     /**
      * The base64 encoded audio input of the user message to attach to the topic and then generate an assistant message in response to.
@@ -1022,6 +1081,13 @@ export type CreateTopicReqPayload = {
     owner_id: string;
 };
 
+/**
+ * Controls the cropping strategy for an item (e.g. segment, chunk, etc.)
+ * - `All` crops all images in the item
+ * - `Auto` crops images only if required for post-processing
+ */
+export type CroppingStrategy = 'All' | 'Auto';
+
 export type Dataset = {
     /**
      * Timestamp of the creation of the dataset
@@ -1353,6 +1419,8 @@ export type EditMessageReqPayload = {
     user_id?: (string) | null;
 };
 
+export type EmbedSource = 'HTML' | 'Markdown' | 'LLM' | 'Content';
+
 export type ErrorResponseBody = {
     message: string;
 };
@@ -1890,6 +1958,8 @@ export type GenerateOffChunksReqPayload = {
     user_id?: (string) | null;
 };
 
+export type GenerationStrategy = 'LLM' | 'Auto';
+
 /**
  * Location that you want to use as the center of the search.
  */
@@ -2294,6 +2364,29 @@ export type LatencyGraphResponse = {
     points: Array<FloatTimePoint>;
 };
 
+/**
+ * Controls the processing and generation for the segment.
+ * - `crop_image` controls whether to crop the file's images to the segment's bounding box.
+ * The cropped image will be stored in the segment's `image` field. Use `All` to always crop,
+ * or `Auto` to only crop when needed for post-processing.
+ * - `html` is the HTML output for the segment, generated either through huerstics (`Auto`) or using Chunkr fine-tuned models (`LLM`)
+ * - `llm` is the LLM-generated output for the segment, this uses off-the-shelf models to generate a custom output for the segment
+ * - `markdown` is the Markdown output for the segment, generated either through huerstics (`Auto`) or using Chunkr fine-tuned models (`LLM`)
+ * - `embed_sources` defines which content sources will be included in the chunk's embed field and counted towards the chunk length.
+ * The array's order determines the sequence in which content appears in the embed field (e.g., [Markdown, LLM] means Markdown content
+ * is followed by LLM content). This directly affects what content is available for embedding and retrieval.
+ */
+export type LlmGenerationConfig = {
+    crop_image?: (CroppingStrategy);
+    embed_sources?: Array<EmbedSource>;
+    html?: (GenerationStrategy);
+    /**
+     * Prompt for the LLM model
+     */
+    llm?: (string) | null;
+    markdown?: (GenerationStrategy);
+};
+
 export type LocationBoundingBox = {
     bottom_right: GeoInfo;
     top_left: GeoInfo;
@@ -2393,6 +2486,13 @@ export type MultiQuery = {
 
 export type NewChunkMetadataTypes = SlimChunkMetadataWithArrayTagSet | ChunkMetadata | ContentChunkMetadata;
 
+/**
+ * Controls the Optical Character Recognition (OCR) strategy.
+ * - `All`: Processes all pages with OCR. (Latency penalty: ~0.5 seconds per page)
+ * - `Auto`: Selectively applies OCR only to pages with missing or low-quality text. When text layer is present the bounding boxes from the text layer are used.
+ */
+export type OcrStrategy = 'All' | 'Auto';
+
 export type OpenGraphMetadata = {
     description?: (string) | null;
     image?: (string) | null;
@@ -2458,6 +2558,9 @@ export type PartnerConfiguration = {
     SLACK_LINK: string;
 };
 
+/**
+ * We plan to deprecate pdf2md in favor of chunkr.ai. This is a legacy option for using a vision LLM to convert a given file into markdown and then ingest it.
+ */
 export type Pdf2MdOptions = {
     /**
      * Split headings is an optional field which allows you to specify whether or not to split headings into separate chunks. Default is false.
@@ -2473,6 +2576,38 @@ export type Pdf2MdOptions = {
     use_pdf2md_ocr: boolean;
 };
 
+/**
+ * Controls the cropping strategy for an item (e.g. segment, chunk, etc.)
+ * - `All` crops all images in the item
+ * - `Auto` crops images only if required for post-processing
+ */
+export type PictureCroppingStrategy = 'All' | 'Auto';
+
+/**
+ * Controls the processing and generation for the segment.
+ * - `crop_image` controls whether to crop the file's images to the segment's bounding box.
+ * The cropped image will be stored in the segment's `image` field. Use `All` to always crop,
+ * or `Auto` to only crop when needed for post-processing.
+ * - `html` is the HTML output for the segment, generated either through huerstics (`Auto`) or using Chunkr fine-tuned models (`LLM`)
+ * - `llm` is the LLM-generated output for the segment, this uses off-the-shelf models to generate a custom output for the segment
+ * - `markdown` is the Markdown output for the segment, generated either through huerstics (`Auto`) or using Chunkr fine-tuned models (`LLM`)
+ * - `embed_sources` defines which content sources will be included in the chunk's embed field and counted towards the chunk length.
+ * The array's order determines the sequence in which content appears in the embed field (e.g., [Markdown, LLM] means Markdown content
+ * is followed by LLM content). This directly affects what content is available for embedding and retrieval.
+ */
+export type PictureGenerationConfig = {
+    crop_image?: (PictureCroppingStrategy);
+    embed_sources?: Array<EmbedSource>;
+    html?: (GenerationStrategy);
+    /**
+     * Prompt for the LLM model
+     */
+    llm?: (string) | null;
+    markdown?: (GenerationStrategy);
+};
+
+export type PipelineType = 'Azure' | 'Chunkr';
+
 export type PopularChat = {
     count: number;
     name: string;
@@ -3545,6 +3680,38 @@ export type SearchesPerUserResponse = {
     points: Array<FloatTimePoint>;
 };
 
+/**
+ * Controls the post-processing of each segment type.
+ *
+ * Allows you to generate HTML and Markdown from chunkr models for each segment type.
+ * By default, the HTML and Markdown are generated manually using the segmentation information except for `Table`, `Formula` and `Picture`.
+ * You can optionally configure custom LLM prompts and models to generate an additional `llm` field with LLM-processed content for each segment type.
+ *
+ * The configuration of which content sources (HTML, Markdown, LLM, Content) of the segment
+ * should be included in the chunk's `embed` field and counted towards the chunk length can be configured through the `embed_sources` setting.
+ */
+export type SegmentProcessing = {
+    Caption?: ((AutoGenerationConfig) | null);
+    Footnote?: ((AutoGenerationConfig) | null);
+    Formula?: ((LlmGenerationConfig) | null);
+    ListItem?: ((AutoGenerationConfig) | null);
+    Page?: ((LlmGenerationConfig) | null);
+    PageFooter?: ((AutoGenerationConfig) | null);
+    PageHeader?: ((AutoGenerationConfig) | null);
+    Picture?: ((PictureGenerationConfig) | null);
+    SectionHeader?: ((AutoGenerationConfig) | null);
+    Table?: ((LlmGenerationConfig) | null);
+    Text?: ((AutoGenerationConfig) | null);
+    Title?: ((AutoGenerationConfig) | null);
+};
+
+/**
+ * Controls the segmentation strategy:
+ * - `LayoutAnalysis`: Analyzes pages for layout elements (e.g., `Table`, `Picture`, `Formula`, etc.) using bounding boxes. Provides fine-grained segmentation and better chunking. (Latency penalty: ~TBD seconds per page).
+ * - `Page`: Treats each page as a single segment. Faster processing, but without layout element detection and only simple chunking.
+ */
+export type SegmentationStrategy = 'LayoutAnalysis' | 'Page';
+
 /**
  * Semantic boosting moves the dense vector of the chunk in the direction of the distance phrase for semantic search. I.e. you can force a cluster by moving every chunk for a PDF closer to its title or push a chunk with a chunk_html of "iphone" 25% closer to the term "flagship" by using the distance phrase "flagship" and a distance factor of 0.25. Conceptually it's drawing a line (euclidean/L2 distance) between the vector for the innerText of the chunk_html and distance_phrase then moving the vector of the chunk_html distance_factor*L2Distance closer to or away from the distance_phrase point along the line between the two points.
  */
@@ -3790,6 +3957,35 @@ export type TagsWithCount = {
     tag: string;
 };
 
+/**
+ * Common tokenizers used for text processing.
+ *
+ * These values represent standard tokenization approaches and popular pre-trained
+ * tokenizers from the Hugging Face ecosystem.
+ */
+export type Tokenizer = 'Word' | 'Cl100kBase' | 'XlmRobertaBase' | 'BertBaseUncased';
+
+/**
+ * Specifies which tokenizer to use for the chunking process.
+ *
+ * This type supports two ways of specifying a tokenizer:
+ * 1. Using a predefined tokenizer from the `Tokenizer` enum
+ * 2. Using any Hugging Face tokenizer by providing its model ID as a string
+ * (e.g. "facebook/bart-large", "Qwen/Qwen-tokenizer", etc.)
+ *
+ * When using a string, any valid Hugging Face tokenizer ID can be specified,
+ * which will be loaded using the Hugging Face tokenizers library.
+ */
+export type TokenizerType = {
+    Enum: Tokenizer;
+} | {
+    /**
+     * Use any Hugging Face tokenizer by specifying its model ID
+     * Examples: "Qwen/Qwen-tokenizer", "facebook/bart-large"
+     */
+    String: string;
+};
+
 /**
  * Function for a LLM tool call
  */
@@ -4193,6 +4389,7 @@ export type UploadFileReqPayload = {
      * Base64 encoded file.
      */
     base64_file: string;
+    chunkr_create_task_req_payload?: ((CreateFormWithoutFile) | null);
     /**
      * Create chunks is a boolean which determines whether or not to create chunks from the file. If false, you can manually chunk the file and send the chunks to the create_chunk endpoint with the file_id to associate chunks with the file. Meant mostly for advanced users.
      */