feature: add the ablity to set custom tags on crawled content

Author: densumesh

clients/ts-sdk/openapi.json

@@ -10996,6 +10996,14 @@
             "description": "The URL to crawl",
             "nullable": true
           },
+          "tags": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "Tags to add to the crawl",
+            "nullable": true
+          },
           "webhook_metadata": {
             "description": "Metadata to send back with the webhook call for each successful page scrape",
             "nullable": true
@@ -11670,6 +11678,13 @@
             "description": "No result message for when there are no chunks found above the score threshold.",
             "nullable": true
           },
+          "number_of_messages_to_include": {
+            "type": "integer",
+            "format": "int64",
+            "description": "Number of messages to include in the context window. If not specified, this defaults to 10.",
+            "nullable": true,
+            "minimum": 0
+          },
           "only_include_docs_used": {
             "type": "boolean",
             "description": "Only include docs used is a boolean that indicates whether or not to only include the docs that were used in the completion. If true, the completion will only include the docs that were used in the completion. If false, the completion will include all of the docs.",
@@ -12779,6 +12794,13 @@
             "description": "No result message for when there are no chunks found above the score threshold.",
             "nullable": true
           },
+          "number_of_messages_to_include": {
+            "type": "integer",
+            "format": "int64",
+            "description": "Number of messages to include in the context window. If not specified, this defaults to 10.",
+            "nullable": true,
+            "minimum": 0
+          },
           "only_include_docs_used": {
             "type": "boolean",
             "description": "Only include docs used is a boolean that indicates whether or not to only include the docs that were used in the completion. If true, the completion will only include the docs that were used in the completion. If false, the completion will include all of the docs.",
@@ -19371,6 +19393,13 @@
             "description": "No result message for when there are no chunks found above the score threshold.",
             "nullable": true
           },
+          "number_of_messages_to_include": {
+            "type": "integer",
+            "format": "int64",
+            "description": "Number of messages to include in the context window. If not specified, this defaults to 10.",
+            "nullable": true,
+            "minimum": 0
+          },
           "only_include_docs_used": {
             "type": "boolean",
             "description": "Only include docs used is a boolean that indicates whether or not to only include the docs that were used in the completion. If true, the completion will only include the docs that were used in the completion. If false, the completion will include all of the docs.",

clients/ts-sdk/src/types.gen.ts

@@ -887,6 +887,10 @@ export type CrawlOptions = {
      * The URL to crawl
      */
     site_url?: (string) | null;
+    /**
+     * Tags to add to the crawl
+     */
+    tags?: Array<(string)> | null;
     /**
      * Metadata to send back with the webhook call for each successful page scrape
      */
@@ -1086,6 +1090,10 @@ export type CreateMessageReqPayload = {
      * No result message for when there are no chunks found above the score threshold.
      */
     no_result_message?: (string) | null;
+    /**
+     * Number of messages to include in the context window. If not specified, this defaults to 10.
+     */
+    number_of_messages_to_include?: (number) | null;
     /**
      * Only include docs used is a boolean that indicates whether or not to only include the docs that were used in the completion. If true, the completion will only include the docs that were used in the completion. If false, the completion will include all of the docs.
      */
@@ -1599,6 +1607,10 @@ export type EditMessageReqPayload = {
      * No result message for when there are no chunks found above the score threshold.
      */
     no_result_message?: (string) | null;
+    /**
+     * Number of messages to include in the context window. If not specified, this defaults to 10.
+     */
+    number_of_messages_to_include?: (number) | null;
     /**
      * Only include docs used is a boolean that indicates whether or not to only include the docs that were used in the completion. If true, the completion will only include the docs that were used in the completion. If false, the completion will include all of the docs.
      */
@@ -3716,6 +3728,10 @@ export type RegenerateMessageReqPayload = {
      * No result message for when there are no chunks found above the score threshold.
      */
     no_result_message?: (string) | null;
+    /**
+     * Number of messages to include in the context window. If not specified, this defaults to 10.
+     */
+    number_of_messages_to_include?: (number) | null;
     /**
      * Only include docs used is a boolean that indicates whether or not to only include the docs that were used in the completion. If true, the completion will only include the docs that were used in the completion. If false, the completion will include all of the docs.
      */

frontends/dashboard/src/components/CrawlingSettings.tsx

@@ -34,6 +34,7 @@ export const defaultCrawlOptions: CrawlOptions = {
   } as ScrapeOptions,
   webhook_urls: [],
   add_chunks_to_dataset: true,
+  tags: [],
 };
 
 export type FlatCrawlOptions = Omit<CrawlOptions, "scrape_options"> & {
@@ -71,6 +72,7 @@ export const unflattenCrawlOptions = (
       },
       webhook_urls: options.webhook_urls,
       add_chunks_to_dataset: options.add_chunks_to_dataset,
+      tags: options.tags,
     };
   } else if (options && options.type == "shopify") {
     return {
@@ -90,6 +92,7 @@ export const unflattenCrawlOptions = (
       },
       webhook_urls: options.webhook_urls,
       add_chunks_to_dataset: options.add_chunks_to_dataset,
+      tags: options.tags,
     };
   } else if (options && options.type == "youtube") {
     return {
@@ -107,6 +110,7 @@ export const unflattenCrawlOptions = (
       },
       webhook_urls: options.webhook_urls,
       add_chunks_to_dataset: options.add_chunks_to_dataset,
+      tags: options.tags,
     };
   }
   return {
@@ -122,6 +126,7 @@ export const unflattenCrawlOptions = (
     scrape_options: null,
     webhook_urls: options.webhook_urls,
     add_chunks_to_dataset: options.add_chunks_to_dataset,
+    tags: options.tags,
   };
 };
 
@@ -752,6 +757,25 @@ const RealCrawlingSettings = (props: RealCrawlingSettingsProps) => {
           />
           <Error error={errors.body_remove_strings} />
         </div>
+        <div>
+          <div class="flex items-center gap-2">
+            <div>Crawl Tags</div>
+            <Tooltip
+              tooltipText="Tags to associate with all chunks created from this crawl. Useful for filtering and organizing crawled content."
+              body={<FaRegularCircleQuestion class="h-3 w-3 text-black" />}
+            />
+          </div>
+          <MultiStringInput
+            placeholder="documentation"
+            addClass="bg-magenta-100/40 px-2 text-sm rounded border border-magenta-300/40"
+            addLabel="Add Tag"
+            onChange={(value) => {
+              setOptions("tags", value);
+            }}
+            value={options.tags || []}
+          />
+          <Error error={errors.tags} />
+        </div>
       </div>
       <Spacer h={18} />
       <div class="mt-5 flex justify-start">

server/src/bin/crawl-worker.rs

@@ -177,10 +177,19 @@ fn create_shopify_chunk_req_payload(
         product_title.clone()
     };
 
+    let mut tag_set = product.tags.clone();
+    tag_set.extend(
+        scrape_request
+            .crawl_options
+            .tags
+            .clone()
+            .unwrap_or_default(),
+    );
+
     Ok(ChunkReqPayload {
         chunk_html: Some(chunk_html),
         link: Some(link),
-        tag_set: Some(product.tags.clone()),
+        tag_set: Some(tag_set),
         num_value: variant.price.clone().unwrap_or_default().parse().ok(),
         metadata: serde_json::to_value(product.clone()).ok(),
         tracking_id: if group_variants {
@@ -282,6 +291,7 @@ async fn parse_chunks_with_firecrawl(
         let page_html = crawl_doc.html.clone().unwrap_or_default();
         let mut page_tags = get_tags(page_link.clone());
         page_tags.push(crawl_req.url.clone());
+        page_tags.extend(crawl_req.crawl_options.tags.clone().unwrap_or_default());
 
         if let Some(spec) = &spec {
             if let Some(ScrapeOptions::OpenApi(ref openapi_options)) =
@@ -815,7 +825,7 @@ async fn parse_youtube_chunks(
                             video.id.video_id,
                             transcript.start.as_secs()
                         )),
-                        tag_set: None,
+                        tag_set: crawl_request.crawl_options.tags.clone(),
                         metadata: Some(json!({
                             "heading": video.snippet.title.clone(),
                             "title": video.snippet.title.clone(),

server/src/data/models.rs

@@ -10030,6 +10030,8 @@ pub struct CrawlOptions {
     pub webhook_metadata: Option<serde_json::Value>,
     /// Add chunks to the dataset that the crawl is created for, defaults to true
     pub add_chunks_to_dataset: Option<bool>,
+    /// Tags to add to the crawl
+    pub tags: Option<Vec<String>>,
 }
 
 #[derive(Serialize, Deserialize, Debug, ToSchema, Clone)]
@@ -10088,6 +10090,7 @@ impl CrawlOptions {
                 .clone()
                 .or(other.webhook_metadata.clone()),
             add_chunks_to_dataset: self.add_chunks_to_dataset.or(other.add_chunks_to_dataset),
+            tags: self.tags.clone().or(other.tags.clone()),
         }
     }
 }