Skip to content

Commit 14d5af3

Browse files
mogerymogery
committed
chore(api/query): change api surface
1 parent 68ced07 commit 14d5af3

37 files changed

Lines changed: 650 additions & 43 deletions

File tree

‎apps/api/src/__tests__/snips/v2/scrape-query.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,30 @@ describe("Query format", () => {
6161
scrapeTimeout,
6262
);
6363

64+
concurrentIf(TEST_PRODUCTION || HAS_AI)(
65+
"returns a direct quote answer when query mode is directQuote",
66+
async () => {
67+
const response = await scrape(
68+
{
69+
url: "https://firecrawl.dev",
70+
formats: [
71+
{
72+
type: "query",
73+
prompt: "What is Firecrawl?",
74+
mode: "directQuote",
75+
},
76+
],
77+
},
78+
identity,
79+
);
80+
81+
expect(response.answer).toBeDefined();
82+
expect(typeof response.answer).toBe("string");
83+
expect(response.answer!.length).toBeGreaterThan(0);
84+
},
85+
scrapeTimeout,
86+
);
87+
6488
concurrentIf(TEST_PRODUCTION || HAS_AI)(
6589
"does not include answer field when query format is not provided",
6690
async () => {

‎apps/api/src/__tests__/snips/v2/types-validation.test.ts‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,25 @@ describe("V2 Types Validation", () => {
8787

8888
const result = scrapeRequestSchema.parse(input);
8989
expect(result.formats).toEqual([
90-
{ type: "query", prompt: "What is Firecrawl?", directQuote: false },
90+
{ type: "query", prompt: "What is Firecrawl?", mode: "freeform" },
91+
]);
92+
});
93+
94+
it("should accept query format directQuote mode", () => {
95+
const input: ScrapeRequestInput = {
96+
url: "https://example.com",
97+
formats: [
98+
{
99+
type: "query",
100+
prompt: "What is Firecrawl?",
101+
mode: "directQuote",
102+
},
103+
],
104+
};
105+
106+
const result = scrapeRequestSchema.parse(input);
107+
expect(result.formats).toEqual([
108+
{ type: "query", prompt: "What is Firecrawl?", mode: "directQuote" },
91109
]);
92110
});
93111

@@ -1003,10 +1021,44 @@ describe("V2 Types Validation", () => {
10031021

10041022
const result = searchRequestSchema.parse(input);
10051023
expect(result.scrapeOptions?.formats).toEqual([
1006-
{ type: "query", prompt: "What is Firecrawl?", directQuote: false },
1024+
{ type: "query", prompt: "What is Firecrawl?", mode: "freeform" },
10071025
]);
10081026
});
10091027

1028+
it("should reject search scrapeOptions query format with invalid mode", () => {
1029+
const input: SearchRequestInput = {
1030+
query: "test",
1031+
scrapeOptions: {
1032+
formats: [
1033+
{
1034+
type: "query",
1035+
prompt: "What is Firecrawl?",
1036+
mode: "quoted",
1037+
} as any,
1038+
],
1039+
},
1040+
};
1041+
1042+
expect(() => searchRequestSchema.parse(input)).toThrow();
1043+
});
1044+
1045+
it("should reject search scrapeOptions query format with directQuote boolean", () => {
1046+
const input: SearchRequestInput = {
1047+
query: "test",
1048+
scrapeOptions: {
1049+
formats: [
1050+
{
1051+
type: "query",
1052+
prompt: "What is Firecrawl?",
1053+
directQuote: true,
1054+
} as any,
1055+
],
1056+
},
1057+
};
1058+
1059+
expect(() => searchRequestSchema.parse(input)).toThrow();
1060+
});
1061+
10101062
it("should reject search scrapeOptions query prompt over 10000 characters", () => {
10111063
const input: SearchRequestInput = {
10121064
query: "test",

‎apps/api/src/controllers/v2/types.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ type AttributesFormatWithOptions = z.output<typeof attributesFormatWithOptions>;
403403
const queryFormatWithOptions = z.strictObject({
404404
type: z.literal("query"),
405405
prompt: z.string().max(10000),
406-
directQuote: z.boolean().optional().default(false),
406+
mode: z.enum(["freeform", "directQuote"]).optional().default("freeform"),
407407
});
408408

409409
type QueryFormatWithOptions = z.output<typeof queryFormatWithOptions>;

‎apps/api/src/scraper/scrapeURL/transformers/query.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ export async function performQuery(
507507

508508
let answer: string | null;
509509

510-
if (queryFormat.directQuote) {
510+
if (queryFormat.mode === "directQuote") {
511511
answer = await performDirectQuoteQuery(
512512
meta,
513513
document,

‎apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,21 @@ public void JsonFormat_HasCorrectType()
244244
Assert.Contains("\"schema\"", json);
245245
}
246246

247+
[Fact]
248+
public void QueryFormat_HasCorrectMode()
249+
{
250+
var format = new QueryFormat
251+
{
252+
Prompt = "What is Firecrawl?",
253+
Mode = QueryFormat.DirectQuoteMode
254+
};
255+
256+
var json = JsonSerializer.Serialize(format, JsonOptions);
257+
Assert.Contains("\"type\":\"query\"", json);
258+
Assert.Contains("\"prompt\":\"What is Firecrawl?\"", json);
259+
Assert.Contains("\"mode\":\"directQuote\"", json);
260+
}
261+
247262
[Fact]
248263
public void WebhookConfig_SerializesCorrectly()
249264
{

‎apps/dot-net-sdk/Firecrawl/Firecrawl.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
<!-- NuGet package metadata -->
1010
<PackageId>firecrawl-sdk</PackageId>
11-
<Version>1.2.0</Version>
11+
<Version>1.2.1</Version>
1212
<Authors>Firecrawl</Authors>
1313
<Company>Firecrawl</Company>
1414
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>

‎apps/dot-net-sdk/Firecrawl/Models/QueryFormat.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using System.Text.Json.Serialization;
2+
3+
namespace Firecrawl.Models;
4+
5+
/// <summary>
6+
/// Query format specification for use in ScrapeOptions.Formats.
7+
/// </summary>
8+
public class QueryFormat
9+
{
10+
public const string FreeformMode = "freeform";
11+
public const string DirectQuoteMode = "directQuote";
12+
13+
[JsonPropertyName("type")]
14+
public string Type { get; } = "query";
15+
16+
[JsonPropertyName("prompt")]
17+
public required string Prompt { get; set; }
18+
19+
[JsonPropertyName("mode")]
20+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
21+
public string? Mode { get; set; }
22+
}

‎apps/elixir-sdk/mix.exs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
defmodule Firecrawl.MixProject do
22
use Mix.Project
33

4-
@version "1.2.0"
4+
@version "1.2.1"
55
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
66

77
def project do

‎apps/go-sdk/options.go‎

Lines changed: 61 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,40 @@
11
package firecrawl
22

3+
import "encoding/json"
4+
5+
// QueryFormatMode selects how query answers are generated.
6+
type QueryFormatMode string
7+
8+
const (
9+
QueryModeFreeform QueryFormatMode = "freeform"
10+
QueryModeDirectQuote QueryFormatMode = "directQuote"
11+
)
12+
13+
// QueryFormat asks a question about page content.
14+
type QueryFormat struct {
15+
Prompt string `json:"prompt"`
16+
Mode QueryFormatMode `json:"mode,omitempty"`
17+
}
18+
19+
// MarshalJSON always emits the API-required query format type.
20+
func (q QueryFormat) MarshalJSON() ([]byte, error) {
21+
type queryFormat struct {
22+
Type string `json:"type"`
23+
Prompt string `json:"prompt"`
24+
Mode QueryFormatMode `json:"mode,omitempty"`
25+
}
26+
27+
return json.Marshal(queryFormat{
28+
Type: "query",
29+
Prompt: q.Prompt,
30+
Mode: q.Mode,
31+
})
32+
}
33+
334
// ScrapeOptions configures a single-page scrape request.
435
type ScrapeOptions struct {
5-
Formats []string `json:"formats,omitempty"`
36+
Formats []string `json:"-"`
37+
FormatOptions []interface{} `json:"-"`
638
Headers map[string]string `json:"headers,omitempty"`
739
IncludeTags []string `json:"includeTags,omitempty"`
840
ExcludeTags []string `json:"excludeTags,omitempty"`
@@ -24,6 +56,25 @@ type ScrapeOptions struct {
2456
JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
2557
}
2658

59+
// MarshalJSON preserves string formats while allowing object formats such as QueryFormat.
60+
func (o ScrapeOptions) MarshalJSON() ([]byte, error) {
61+
type scrapeOptions ScrapeOptions
62+
payload := struct {
63+
scrapeOptions
64+
Formats interface{} `json:"formats,omitempty"`
65+
}{
66+
scrapeOptions: scrapeOptions(o),
67+
}
68+
69+
if len(o.FormatOptions) > 0 {
70+
payload.Formats = o.FormatOptions
71+
} else if len(o.Formats) > 0 {
72+
payload.Formats = o.Formats
73+
}
74+
75+
return json.Marshal(payload)
76+
}
77+
2778
// CrawlOptions configures a crawl request.
2879
type CrawlOptions struct {
2980
Prompt *string `json:"prompt,omitempty"`
@@ -34,7 +85,7 @@ type CrawlOptions struct {
3485
IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`
3586
DeduplicateSimilarURLs *bool `json:"deduplicateSimilarURLs,omitempty"`
3687
Limit *int `json:"limit,omitempty"`
37-
CrawlEntireDomain *bool `json:"crawlEntireDomain,omitempty"`
88+
CrawlEntireDomain *bool `json:"crawlEntireDomain,omitempty"`
3889
AllowExternalLinks *bool `json:"allowExternalLinks,omitempty"`
3990
AllowSubdomains *bool `json:"allowSubdomains,omitempty"`
4091
Delay *int `json:"delay,omitempty"`
@@ -87,14 +138,14 @@ type SearchOptions struct {
87138

88139
// AgentOptions configures an agent request.
89140
type AgentOptions struct {
90-
URLs []string `json:"urls,omitempty"`
91-
Prompt string `json:"prompt"`
92-
Schema map[string]interface{} `json:"schema,omitempty"`
93-
Integration *string `json:"integration,omitempty"`
94-
MaxCredits *int `json:"maxCredits,omitempty"`
95-
StrictConstrainToURLs *bool `json:"strictConstrainToURLs,omitempty"`
96-
Model *string `json:"model,omitempty"`
97-
Webhook *WebhookConfig `json:"webhook,omitempty"`
141+
URLs []string `json:"urls,omitempty"`
142+
Prompt string `json:"prompt"`
143+
Schema map[string]interface{} `json:"schema,omitempty"`
144+
Integration *string `json:"integration,omitempty"`
145+
MaxCredits *int `json:"maxCredits,omitempty"`
146+
StrictConstrainToURLs *bool `json:"strictConstrainToURLs,omitempty"`
147+
Model *string `json:"model,omitempty"`
148+
Webhook *WebhookConfig `json:"webhook,omitempty"`
98149
}
99150

100151
// LocationConfig specifies geolocation for requests.

‎apps/go-sdk/options_test.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package firecrawl
2+
3+
import (
4+
"encoding/json"
5+
"strings"
6+
"testing"
7+
)
8+
9+
func TestScrapeOptionsSerializesQueryFormatMode(t *testing.T) {
10+
payload, err := json.Marshal(ScrapeOptions{
11+
FormatOptions: []interface{}{
12+
QueryFormat{
13+
Prompt: "What is Firecrawl?",
14+
Mode: QueryModeDirectQuote,
15+
},
16+
},
17+
})
18+
if err != nil {
19+
t.Fatalf("Marshal ScrapeOptions: %v", err)
20+
}
21+
22+
jsonBody := string(payload)
23+
for _, want := range []string{
24+
`"formats":[{"type":"query","prompt":"What is Firecrawl?","mode":"directQuote"}]`,
25+
} {
26+
if !strings.Contains(jsonBody, want) {
27+
t.Fatalf("serialized query format = %s, want to contain %s", jsonBody, want)
28+
}
29+
}
30+
}
31+
32+
func TestScrapeOptionsPreservesStringFormats(t *testing.T) {
33+
payload, err := json.Marshal(ScrapeOptions{
34+
Formats: []string{"markdown"},
35+
})
36+
if err != nil {
37+
t.Fatalf("Marshal ScrapeOptions: %v", err)
38+
}
39+
40+
if !strings.Contains(string(payload), `"formats":["markdown"]`) {
41+
t.Fatalf("serialized string formats = %s", payload)
42+
}
43+
}

0 commit comments

Comments
 (0)