|
5 | 5 | -- 4) Marks document processed/failed |
6 | 6 |
|
7 | 7 | SELECT df.start( |
8 | | - $$SELECT COALESCE( |
9 | | - ( |
10 | | - SELECT jsonb_build_object('id', id, 'content', content)::text |
11 | | - FROM demo.af_documents |
12 | | - WHERE status = 'pending' |
13 | | - ORDER BY id |
14 | | - LIMIT 1 |
15 | | - ), |
16 | | - 'null' |
17 | | - )$$ |=> 'doc' |
18 | | - ~> df.if( |
19 | | - $$SELECT (($doc)::jsonb) IS NOT NULL$$, |
20 | | - ( |
| 8 | + -- 1. Fetch one pending document |
| 9 | + $$SELECT id, content FROM demo.af_documents |
| 10 | + WHERE status = 'pending' |
| 11 | + ORDER BY id |
| 12 | + LIMIT 1 |
| 13 | + $$ |=> 'doc' |
| 14 | + |
| 15 | + ~> df.if_rows('doc', |
| 16 | + |
| 17 | + -- 2. Build & send HTTP request |
21 | 18 | ($$SELECT jsonb_build_object( |
22 | | - 'document_id', ((($doc)::jsonb->>'id')::bigint), |
23 | | - 'text', (($doc)::jsonb->>'content'), |
24 | | - 'max_tokens', 20, |
25 | | - 'overlap_tokens', 5, |
26 | | - 'language', 'en' |
27 | | - )::text$$ |=> 'requestbody') |
28 | | - ~> (df.http( |
29 | | - '{azure_function_base_url}/api/chunk_text', |
30 | | - 'POST', |
31 | | - '$requestbody', |
32 | | - ('{"Content-Type":"application/json","x-functions-key":"{azure_function_key}"}')::jsonb, |
33 | | - 60 |
34 | | - ) |=> 'chunkresponse') |
| 19 | + 'document_id', $doc.id, |
| 20 | + 'text', $doc.content, |
| 21 | + 'max_tokens', 20, |
| 22 | + 'overlap_tokens', 5, |
| 23 | + 'language', 'en' |
| 24 | + )::text$$ |=> 'requestbody') |
| 25 | + |
| 26 | + ~> ( |
| 27 | + df.http( |
| 28 | + '{azure_function_base_url}/api/chunk_text', |
| 29 | + 'POST', |
| 30 | + '$requestbody', |
| 31 | + '{"Content-Type":"application/json","x-functions-key":"{azure_function_key}"}'::jsonb, |
| 32 | + 60 |
| 33 | + ) |=> 'resp') |
| 34 | + |
| 35 | + -- 3. Parse response fields |
| 36 | + ~> ($$SELECT ($resp::jsonb->>'ok')::boolean AS ok, |
| 37 | + $resp::jsonb->>'body' AS body$$ |=> 'r') |
| 38 | + |
| 39 | + -- 4. Branch on success/failure |
35 | 40 | ~> df.if( |
36 | | - $$SELECT ($chunkresponse::jsonb->>'ok')::boolean$$, |
37 | | - ( |
38 | | - $$WITH payload AS ( |
39 | | - SELECT (($chunkresponse::jsonb->>'body')::jsonb) AS body_json |
40 | | - ) |
41 | | - INSERT INTO demo.af_document_chunks (document_id, chunk_index, chunk_text, token_count) |
42 | | - SELECT |
43 | | - (payload.body_json->>'document_id')::bigint, |
44 | | - (c->>'chunk_index')::int, |
45 | | - c->>'text', |
46 | | - (c->>'token_count')::int |
47 | | - FROM payload, jsonb_array_elements(payload.body_json->'chunks') AS c$$ |
48 | | - ~> $$UPDATE demo.af_documents |
49 | | - SET status = 'processed', |
50 | | - processed_at = now(), |
51 | | - total_tokens = ((($chunkresponse::jsonb->>'body')::jsonb->>'total_tokens')::int), |
52 | | - last_error = NULL |
53 | | - WHERE id = ((($doc)::jsonb->>'id')::bigint)$$ |
54 | | - ), |
| 41 | + $$SELECT $r.ok$$, |
| 42 | + |
| 43 | + -- Success: insert chunks, then mark processed |
| 44 | + $$INSERT INTO demo.af_document_chunks (document_id, chunk_index, chunk_text, token_count) |
| 45 | + SELECT $doc.id, |
| 46 | + (c->>'chunk_index')::int, |
| 47 | + c->>'text', |
| 48 | + (c->>'token_count')::int |
| 49 | + FROM jsonb_array_elements(($r.body)::jsonb->'chunks') AS c$$ |
| 50 | + ~> $$UPDATE demo.af_documents |
| 51 | + SET status = 'processed', |
| 52 | + processed_at = now(), |
| 53 | + total_tokens = (($r.body)::jsonb->>'total_tokens')::int, |
| 54 | + last_error = NULL |
| 55 | + WHERE id = $doc.id$$, |
| 56 | + |
| 57 | + -- Failure: mark failed |
55 | 58 | $$UPDATE demo.af_documents |
56 | | - SET status = 'failed', |
57 | | - processed_at = now(), |
58 | | - last_error = 'HTTP ' || ($chunkresponse::jsonb->>'status') || ': ' || coalesce(($chunkresponse::jsonb->>'body'), '') |
59 | | - WHERE id = ((($doc)::jsonb->>'id')::bigint)$$ |
60 | | - ) |
61 | | - ), |
62 | | - $$SELECT 'no pending documents'$$ |
| 59 | + SET status = 'failed', |
| 60 | + processed_at = now(), |
| 61 | + last_error = 'HTTP ' || ($resp::jsonb->>'status') || ': ' || coalesce($r.body, '') |
| 62 | + WHERE id = $doc.id$$ |
| 63 | + ), |
| 64 | + |
| 65 | + -- No pending documents |
| 66 | + $$SELECT 'no pending documents'$$ |
63 | 67 | ), |
64 | 68 | 'azure-functions-chunk-text' |
65 | 69 | ) AS instance_id; |
0 commit comments