Skip to content

Commit d6edc20

Browse files
authored
Merge pull request #4 from StacLabs/max-body-size
Configure max payload size
2 parents ad6927f + ab00bd1 commit d6edc20

4 files changed

Lines changed: 189 additions & 6 deletions

File tree

CHANGELOG.md

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,13 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## [0.1.2] - 2026-03-30
9+
10+
### Added
11+
- **Configurable Payload Limits**: Introduced the `MAX_BODY_SIZE_MB` environment variable, allowing operators to tune the maximum allowed size for incoming STAC batches (defaults to 150MB).
12+
813
## [0.1.1] - 2026-03-29
14+
915
### Added
1016
- **Intelligent Batch Logging**: Enhanced the `/validate` endpoint with high-visibility logging for both single-item and bulk `ItemCollection` requests.
1117
- **Error Aggregation**: Implemented a frequency-based error summarizer for large batches. Instead of flooding logs, the service now identifies and counts unique failure reasons (e.g., "Top failure reason (99/100): 'datetime' is required").
@@ -14,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1420
- **Project Tooling**: Added a `Makefile` and `go test` suite to standardize the contributor workflow and ensure build stability. [#3](https://github.com/StacLabs/gostac-validator/pull/3)
1521

1622
## [0.1.0] - 2026-03-29
23+
1724
### Added
1825
- Core STAC validation engine with PCRE regex (`^(?!eo:)`) support via `regexp2`.
1926
- Lossless float decoding for STAC geographic coordinates.
@@ -22,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
2229
- High-performance CLI tool for local STAC validation.
2330
- Dockerfile for microservice deployment.
2431

25-
[Unreleased]: https://github.com/StacLabs/gostac-validator/compare/v0.1.1...HEAD
32+
[Unreleased]: https://github.com/StacLabs/gostac-validator/compare/v0.1.2...HEAD
33+
[0.1.2]: https://github.com/StacLabs/gostac-validator/compare/v0.1.1...v0.1.2
2634
[0.1.1]: https://github.com/StacLabs/gostac-validator/compare/v0.1.0...v0.1.1
2735
[0.1.0]: https://github.com/StacLabs/gostac-validator/releases/tag/v0.1.0

README.md

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ It drops validation times from ~2,500ms (network-bound) down to **~0.24ms** (RAM
2020
1. [Installation & Build](#installation--build)
2121
2. [CLI Usage](#cli-usage)
2222
3. [Microservice Usage](#microservice-usage)
23-
4. [Benchmarking & Performance](#benchmarking--performance)
24-
5. [Architecture Overview](#architecture-overview)
23+
4. [Configuration](#configuration)
24+
5. [Benchmarking & Performance](#benchmarking--performance)
25+
6. [Architecture Overview](#architecture-overview)
2526

2627
---
2728

@@ -78,10 +79,22 @@ The CLI tool is perfect for local testing, CI/CD pipelines, or ad-hoc validation
7879

7980
The HTTP server uses a thread-safe `sync.Map` to cache compiled schemas. It is designed to sit behind an ingestor API (like FastAPI) and process thousands of concurrent validation requests safely.
8081

82+
## Configuration
83+
84+
The STAC Validator Microservice can be tuned via environment variables to handle different infrastructure requirements and payload sizes.
85+
86+
| Variable | Description | Default |
87+
| :--- | :--- | :--- |
88+
| `ADDR` | The network address and port for the server to listen on. | `:8080` |
89+
| `MAX_BODY_SIZE_MB` | The maximum allowed size of an incoming POST request in Megabytes. | `150` |
90+
8191
### Start the server:
8292
```bash
93+
# Start with default 150MB limit
8394
./stac-server
84-
# 🚀 STAC Validation Server running on http://localhost:8080/validate
95+
96+
# Start with a custom 512MB limit for large batches
97+
MAX_BODY_SIZE_MB=512 ./stac-server
8598
```
8699

87100
### Test via cURL:

internal/server/handlers.go

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99
"encoding/json"
1010
"log"
1111
"net/http"
12+
"os"
13+
"strconv"
1214
"sync"
1315
"time"
1416

@@ -165,9 +167,29 @@ func (h *Handler) Health(w http.ResponseWriter, r *http.Request) {
165167
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
166168
}
167169

168-
// readBody handles reading the request with a 150MB limit.
170+
// getBodyLimit returns the limit in bytes from MAX_BODY_SIZE_MB env var,
171+
// defaulting to 150MB.
172+
func getBodyLimit() int64 {
173+
const defaultLimitMB = 150
174+
limitStr := os.Getenv("MAX_BODY_SIZE_MB")
175+
if limitStr == "" {
176+
return defaultLimitMB << 20
177+
}
178+
179+
limitMB, err := strconv.ParseInt(limitStr, 10, 64)
180+
if err != nil {
181+
// Log the error or just return default
182+
return defaultLimitMB << 20
183+
}
184+
185+
return limitMB << 20
186+
}
187+
188+
// readBody handles reading the request with a configurable limit.
169189
func readBody(w http.ResponseWriter, r *http.Request) ([]byte, error) {
170-
const maxBytes = 150 << 20
190+
// Dynamically fetch the limit
191+
maxBytes := getBodyLimit()
192+
171193
r.Body = http.MaxBytesReader(w, r.Body, maxBytes)
172194
var buf bytes.Buffer
173195
if _, err := buf.ReadFrom(r.Body); err != nil {

internal/server/server_test.go

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
package server
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"net/http"
7+
"net/http/httptest"
8+
"os"
9+
"testing"
10+
)
11+
12+
func TestGetBodyLimit(t *testing.T) {
13+
// Table-driven tests for environment variable parsing
14+
tests := []struct {
15+
name string
16+
envValue string
17+
expected int64
18+
}{
19+
{
20+
name: "Default value when unset",
21+
envValue: "",
22+
expected: 150 << 20, // 150 MB
23+
},
24+
{
25+
name: "Valid custom value",
26+
envValue: "500",
27+
expected: 500 << 20, // 500 MB
28+
},
29+
{
30+
name: "Invalid string falls back to default",
31+
envValue: "not_a_number",
32+
expected: 150 << 20, // 150 MB
33+
},
34+
}
35+
36+
for _, tt := range tests {
37+
t.Run(tt.name, func(t *testing.T) {
38+
// Setup environment
39+
if tt.envValue != "" {
40+
os.Setenv("MAX_BODY_SIZE_MB", tt.envValue)
41+
} else {
42+
os.Unsetenv("MAX_BODY_SIZE_MB")
43+
}
44+
45+
// Clean up after test
46+
t.Cleanup(func() {
47+
os.Unsetenv("MAX_BODY_SIZE_MB")
48+
})
49+
50+
got := getBodyLimit()
51+
if got != tt.expected {
52+
t.Errorf("getBodyLimit() = %d bytes, want %d bytes", got, tt.expected)
53+
}
54+
})
55+
}
56+
}
57+
58+
func TestReadBody_LimitExceeded(t *testing.T) {
59+
// Set the limit artificially low (1 MB)
60+
os.Setenv("MAX_BODY_SIZE_MB", "1")
61+
t.Cleanup(func() { os.Unsetenv("MAX_BODY_SIZE_MB") })
62+
63+
// Create a 2MB dummy payload
64+
bigPayload := bytes.Repeat([]byte("a"), 2<<20)
65+
66+
req := httptest.NewRequest(http.MethodPost, "/validate", bytes.NewReader(bigPayload))
67+
w := httptest.NewRecorder()
68+
69+
_, err := readBody(w, req)
70+
if err == nil {
71+
t.Fatal("Expected an error when reading body larger than limit, got nil")
72+
}
73+
74+
// http.MaxBytesReader returns an error containing "request body too large"
75+
if err.Error() != "http: request body too large" {
76+
t.Errorf("Expected 'request body too large' error, got: %v", err)
77+
}
78+
}
79+
80+
func TestValidateHandler_PayloadTooLarge(t *testing.T) {
81+
// Set the limit artificially low (1 MB)
82+
os.Setenv("MAX_BODY_SIZE_MB", "1")
83+
t.Cleanup(func() { os.Unsetenv("MAX_BODY_SIZE_MB") })
84+
85+
// We can pass nil for the validator because the body limit check
86+
// happens before the handler attempts to use the validator!
87+
handler := NewHandler(nil)
88+
89+
// Create a 2MB dummy payload
90+
bigPayload := bytes.Repeat([]byte(`{"type":"Feature"}`), 100000)
91+
92+
req := httptest.NewRequest(http.MethodPost, "/validate", bytes.NewReader(bigPayload))
93+
w := httptest.NewRecorder()
94+
95+
handler.Validate(w, req)
96+
97+
res := w.Result()
98+
defer res.Body.Close()
99+
100+
// Ensure the server rejected it as a Bad Request
101+
if res.StatusCode != http.StatusBadRequest {
102+
t.Errorf("Expected status %d, got %d", http.StatusBadRequest, res.StatusCode)
103+
}
104+
105+
// Verify the JSON error message
106+
var errResponse map[string]string
107+
if err := json.NewDecoder(res.Body).Decode(&errResponse); err != nil {
108+
t.Fatalf("Failed to decode response JSON: %v", err)
109+
}
110+
111+
expectedErrFragment := "could not read request body"
112+
if errStr, ok := errResponse["error"]; !ok || len(errStr) < len(expectedErrFragment) || errStr[:len(expectedErrFragment)] != expectedErrFragment {
113+
t.Errorf("Expected error starting with '%s', got '%s'", expectedErrFragment, errStr)
114+
}
115+
}
116+
117+
func TestHealthHandler(t *testing.T) {
118+
handler := NewHandler(nil)
119+
120+
req := httptest.NewRequest(http.MethodGet, "/health", nil)
121+
w := httptest.NewRecorder()
122+
123+
handler.Health(w, req)
124+
125+
res := w.Result()
126+
defer res.Body.Close()
127+
128+
if res.StatusCode != http.StatusOK {
129+
t.Errorf("Expected status 200, got %d", res.StatusCode)
130+
}
131+
132+
var response map[string]string
133+
if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
134+
t.Fatalf("Failed to decode response: %v", err)
135+
}
136+
137+
if status, ok := response["status"]; !ok || status != "ok" {
138+
t.Errorf("Expected status='ok', got %v", response)
139+
}
140+
}

0 commit comments

Comments
 (0)