Skip to content

Commit a2d700e

Browse files
creed-brattoncreed-bratton
committed
feat: add IPACodesURL option to fetch fresh IPA codes at runtime
ParserConfig.IPACodesURL, when set, causes NewParser to download a fresh Italian IPA codes list from the given URL instead of using the embedded snapshot. The format matches the Agid export (one code per line), so the Agid URL can be passed directly. The validator is now per-Parser rather than a package-level global, which also makes concurrent parsers with different IPA code sets safe. As a side effect, MakeIsOrganisationURI now builds its inner validator once per Parser (via closure) instead of on every validation call. The embedded list remains the default; WASM callers omit the option and continue to use it as before.
1 parent b2aed58 commit a2d700e

9 files changed

Lines changed: 173 additions & 68 deletions

File tree

‎fields.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ func validateFieldsV0(publiccode PublicCode, parser *Parser, network bool, baseU
9292
// to use uppercase on an invalid country.
9393
if publiccodev0.IntendedAudience.Countries != nil {
9494
for i, c := range *publiccodev0.IntendedAudience.Countries {
95-
if sharedValidate.Var(c, "iso3166_1_alpha2_lower_or_upper") == nil && c == strings.ToLower(c) {
95+
if parser.validate.Var(c, "iso3166_1_alpha2_lower_or_upper") == nil && c == strings.ToLower(c) {
9696
vr = append(vr, ValidationWarning{
9797
fmt.Sprintf("intendedAudience.countries[%d]", i),
9898
fmt.Sprintf("Lowercase country codes are DEPRECATED. Use uppercase instead ('%s')", strings.ToUpper(c)),
@@ -104,7 +104,7 @@ func validateFieldsV0(publiccode PublicCode, parser *Parser, network bool, baseU
104104

105105
if publiccodev0.IntendedAudience.UnsupportedCountries != nil {
106106
for i, c := range *publiccodev0.IntendedAudience.UnsupportedCountries {
107-
if sharedValidate.Var(c, "iso3166_1_alpha2_lower_or_upper") == nil && c == strings.ToLower(c) {
107+
if parser.validate.Var(c, "iso3166_1_alpha2_lower_or_upper") == nil && c == strings.ToLower(c) {
108108
vr = append(vr, ValidationWarning{
109109
fmt.Sprintf("intendedAudience.unsupportedCountries[%d]", i),
110110
fmt.Sprintf("Lowercase country codes are DEPRECATED. Use uppercase instead ('%s')", strings.ToUpper(c)),
@@ -242,7 +242,7 @@ func validateFieldsV0(publiccode PublicCode, parser *Parser, network bool, baseU
242242
}
243243

244244
if it.Riuso.CodiceIPA != "" {
245-
if sharedValidate.Var(it.Riuso.CodiceIPA, "is_italian_ipa_code") == nil {
245+
if parser.validate.Var(it.Riuso.CodiceIPA, "is_italian_ipa_code") == nil {
246246
vr = append(vr, ValidationWarning{
247247
"IT.riuso.codiceIPA",
248248
fmt.Sprintf(
@@ -324,7 +324,7 @@ func validateFieldsV1(publiccode PublicCode, parser *Parser, network bool, baseU
324324
// to use uppercase on an invalid country.
325325
if publiccodev1.IntendedAudience.Countries != nil {
326326
for i, c := range *publiccodev1.IntendedAudience.Countries {
327-
if sharedValidate.Var(c, "iso3166_1_alpha2_lower_or_upper") == nil && c == strings.ToLower(c) {
327+
if parser.validate.Var(c, "iso3166_1_alpha2_lower_or_upper") == nil && c == strings.ToLower(c) {
328328
vr = append(vr, ValidationWarning{
329329
fmt.Sprintf("intendedAudience.countries[%d]", i),
330330
fmt.Sprintf("Lowercase country codes are DEPRECATED. Use uppercase instead ('%s')", strings.ToUpper(c)),
@@ -336,7 +336,7 @@ func validateFieldsV1(publiccode PublicCode, parser *Parser, network bool, baseU
336336

337337
if publiccodev1.IntendedAudience.UnsupportedCountries != nil {
338338
for i, c := range *publiccodev1.IntendedAudience.UnsupportedCountries {
339-
if sharedValidate.Var(c, "iso3166_1_alpha2_lower_or_upper") == nil && c == strings.ToLower(c) {
339+
if parser.validate.Var(c, "iso3166_1_alpha2_lower_or_upper") == nil && c == strings.ToLower(c) {
340340
vr = append(vr, ValidationWarning{
341341
fmt.Sprintf("intendedAudience.unsupportedCountries[%d]", i),
342342
fmt.Sprintf("Lowercase country codes are DEPRECATED. Use uppercase instead ('%s')", strings.ToUpper(c)),

‎parser.go‎

Lines changed: 63 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package publiccode
22

33
import (
4+
"bufio"
45
"bytes"
56
"context"
67
"errors"
@@ -30,21 +31,37 @@ import (
3031
publiccodeValidator "github.com/italia/publiccode-parser-go/v5/validators"
3132
)
3233

33-
// Build Validator and Translator once at package init.
34-
var (
35-
sharedValidate *validator.Validate
36-
sharedTrans ut.Translator
37-
)
34+
// fetchIPACodes downloads the IPA codes list from the given URL and returns it
35+
// as a set. The format is expected to match the Agid export: one code per line.
36+
func fetchIPACodes(client *http.Client, rawURL string) (map[string]struct{}, error) {
37+
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
38+
if err != nil {
39+
return nil, fmt.Errorf("building IPA codes request for %q: %w", rawURL, err)
40+
}
3841

39-
func init() {
40-
sharedValidate = publiccodeValidator.New()
42+
resp, err := client.Do(req)
43+
if err != nil {
44+
return nil, fmt.Errorf("fetching IPA codes from %q: %w", rawURL, err)
45+
}
4146

42-
enLocale := en.New()
43-
uni := ut.New(enLocale, enLocale)
47+
defer resp.Body.Close()
48+
49+
if resp.StatusCode != http.StatusOK {
50+
return nil, fmt.Errorf("fetching IPA codes from %q: unexpected HTTP status %d", rawURL, resp.StatusCode) //nolint:err113,lll // dynamic status code
51+
}
52+
53+
codes := make(map[string]struct{}, 24000)
54+
scanner := bufio.NewScanner(resp.Body)
55+
56+
for scanner.Scan() {
57+
codes[strings.ToLower(scanner.Text())] = struct{}{}
58+
}
59+
60+
if err := scanner.Err(); err != nil {
61+
return nil, fmt.Errorf("reading IPA codes from %q: %w", rawURL, err)
62+
}
4463

45-
sharedTrans, _ = uni.GetTranslator("en")
46-
_ = en_translations.RegisterDefaultTranslations(sharedValidate, sharedTrans)
47-
_ = publiccodeValidator.RegisterLocalErrorMessages(sharedValidate, sharedTrans)
64+
return codes, nil
4865
}
4966

5067
var reMapKey = regexp.MustCompile(`\[([[:alpha:]]+)\]`)
@@ -75,6 +92,15 @@ type ParserConfig struct {
7592
// Timeout is the maximum duration for each HTTP request during external checks.
7693
// Defaults to 30s if zero.
7794
Timeout time.Duration
95+
96+
// IPACodesURL, if set, causes the parser to fetch a fresh list of Italian
97+
// Public Administration codes from this URL at creation time, instead of
98+
// using the embedded snapshot. The expected format matches the Agid export:
99+
// one code per line (https://www.indicepa.gov.it).
100+
//
101+
// Leave empty (default) to use the embedded snapshot, which is updated
102+
// periodically via the repo's automated workflow.
103+
IPACodesURL string
78104
}
79105

80106
const defaultHTTPTimeout = 30 * time.Second
@@ -88,6 +114,8 @@ type Parser struct {
88114
baseURL *url.URL
89115
client *http.Client
90116
httpclient *httpclient.Client
117+
validate *validator.Validate
118+
trans ut.Translator
91119
}
92120

93121
// Domain is a single code hosting service.
@@ -112,13 +140,34 @@ func NewParser(config ParserConfig) (*Parser, error) {
112140

113141
httpClient := &http.Client{Timeout: timeout}
114142
vcsurl.Client = httpClient
143+
144+
ipaCodes := publiccodeValidator.DefaultIPACodes()
145+
146+
if config.IPACodesURL != "" {
147+
var err error
148+
if ipaCodes, err = fetchIPACodes(httpClient, config.IPACodesURL); err != nil {
149+
return nil, err
150+
}
151+
}
152+
153+
validate := publiccodeValidator.New(ipaCodes)
154+
155+
enLocale := en.New()
156+
uni := ut.New(enLocale, enLocale)
157+
158+
trans, _ := uni.GetTranslator("en")
159+
_ = en_translations.RegisterDefaultTranslations(validate, trans)
160+
_ = publiccodeValidator.RegisterLocalErrorMessages(validate, trans)
161+
115162
p := Parser{
116163
disableNetwork: config.DisableNetwork,
117164
disableExternalChecks: config.DisableExternalChecks,
118165
domain: config.Domain,
119166
branch: config.Branch,
120167
client: httpClient,
121168
httpclient: httpclient.NewClient(httpClient),
169+
validate: validate,
170+
trans: trans,
122171
}
123172

124173
if config.BaseURL != "" {
@@ -289,7 +338,7 @@ func (p *Parser) parseStream(in io.Reader, fileURL *url.URL) (PublicCode, error)
289338
ve = append(ve, decodeResults...)
290339
}
291340

292-
err = sharedValidate.Struct(publiccode)
341+
err = p.validate.Struct(publiccode)
293342
if err != nil {
294343
var validationErrs validator.ValidationErrors
295344
if errors.As(err, &validationErrs) {
@@ -301,7 +350,7 @@ func (p *Parser) parseStream(in io.Reader, fileURL *url.URL) (PublicCode, error)
301350

302351
ve = append(ve, ValidationError{
303352
Key: key,
304-
Description: err.Translate(sharedTrans),
353+
Description: err.Translate(p.trans),
305354
Line: line,
306355
Column: column,
307356
})

‎parser_extra_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,50 @@ func TestParseStreamSyntaxError(t *testing.T) {
133133
}
134134
}
135135

136+
// TestIPACodesURLFetch verifies that WithIPACodesURL fetches and uses the
137+
// provided list, making a code from the served file valid.
138+
func TestIPACodesURLFetch(t *testing.T) {
139+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
140+
_, _ = w.Write([]byte("TESTCODE\n"))
141+
}))
142+
defer srv.Close()
143+
144+
p, err := NewParser(ParserConfig{IPACodesURL: srv.URL})
145+
if err != nil {
146+
t.Fatalf("unexpected error: %v", err)
147+
}
148+
149+
if err := p.validate.Var("TESTCODE", "is_italian_ipa_code"); err != nil {
150+
t.Errorf("expected TESTCODE to be valid with custom list: %v", err)
151+
}
152+
153+
if err := p.validate.Var("pcm", "is_italian_ipa_code"); err == nil {
154+
t.Error("expected 'pcm' to be invalid when not in custom list")
155+
}
156+
}
157+
158+
// TestIPACodesURLFetchError verifies that an unreachable IPACodesURL returns an
159+
// error from NewParser.
160+
func TestIPACodesURLFetchError(t *testing.T) {
161+
_, err := NewParser(ParserConfig{IPACodesURL: "http://127.0.0.1:1/ipa_codes.txt"})
162+
if err == nil {
163+
t.Fatal("expected error for unreachable IPACodesURL")
164+
}
165+
}
166+
167+
// TestIPACodesDefaultEmbedded verifies that the embedded list is used when
168+
// IPACodesURL is empty.
169+
func TestIPACodesDefaultEmbedded(t *testing.T) {
170+
p, err := NewParser(ParserConfig{})
171+
if err != nil {
172+
t.Fatalf("unexpected error: %v", err)
173+
}
174+
175+
if err := p.validate.Var("pcm", "is_italian_ipa_code"); err != nil {
176+
t.Errorf("expected 'pcm' to be valid with embedded list: %v", err)
177+
}
178+
}
179+
136180
func TestParseStreamReaderError(t *testing.T) {
137181
p, _ := NewParser(ParserConfig{DisableNetwork: true})
138182
_, err := p.ParseStream(errReader{})

‎validators/common.go‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -80,37 +80,42 @@ func isURL(fl validator.FieldLevel) bool {
8080
panic(fmt.Sprintf("Bad field type for %T. Must implement fmt.Stringer", fl.Field().Interface()))
8181
}
8282

83-
func isOrganisationURI(fl validator.FieldLevel) bool {
84-
field := fl.Field().String()
83+
// MakeIsOrganisationURI returns a validator.Func that validates an organisation URI,
84+
// including Italian PA URNs (urn:x-italian-pa:<codiceIPA>), using the provided
85+
// IPA codes set. The inner validator is built once and captured in the closure.
86+
func MakeIsOrganisationURI(codes map[string]struct{}) validator.Func {
87+
inner := validator.New(validator.WithRequiredStructEnabled())
88+
_ = inner.RegisterValidation("is_italian_ipa_code", MakeIsItalianIpaCode(codes))
8589

86-
u, err := url.ParseRequestURI(field)
87-
if err != nil {
88-
return false
89-
}
90+
return func(fl validator.FieldLevel) bool {
91+
field := fl.Field().String()
9092

91-
// Validate URNs as well
92-
if strings.EqualFold(u.Scheme, "urn") {
93-
err := sharedValidator.Var(field, "urn_rfc2141")
93+
u, err := url.ParseRequestURI(field)
9494
if err != nil {
9595
return false
9696
}
9797

98-
if strings.HasPrefix(strings.ToLower(u.Opaque), "x-italian-pa:") {
99-
ipa := u.Opaque[len("x-italian-pa:"):]
98+
// Validate URNs as well
99+
if strings.EqualFold(u.Scheme, "urn") {
100+
if err := inner.Var(field, "urn_rfc2141"); err != nil {
101+
return false
102+
}
103+
104+
if strings.HasPrefix(strings.ToLower(u.Opaque), "x-italian-pa:") {
105+
ipa := u.Opaque[len("x-italian-pa:"):]
100106

101-
_, ok := ipaCodes[strings.ToLower(ipa)]
107+
return inner.Var(ipa, "is_italian_ipa_code") == nil
108+
}
102109

103-
return ok
110+
return true
104111
}
105112

106-
return true
107-
}
113+
if u.Scheme == "" || u.Host == "" {
114+
return false
115+
}
108116

109-
if u.Scheme == "" || u.Host == "" {
110-
return false
117+
return true
111118
}
112-
113-
return true
114119
}
115120

116121
// Custom validator to work around https://github.com/go-playground/validator/issues/1260

‎validators/common_test.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
)
1010

1111
func TestBCP47KeysValidMap(t *testing.T) {
12-
v := New()
12+
v := New(DefaultIPACodes())
1313

1414
type S struct {
1515
M map[string]string `validate:"bcp47_keys"`
@@ -22,7 +22,7 @@ func TestBCP47KeysValidMap(t *testing.T) {
2222
}
2323

2424
func TestBCP47KeysInvalidMap(t *testing.T) {
25-
v := New()
25+
v := New(DefaultIPACodes())
2626

2727
type S struct {
2828
M map[string]string `validate:"bcp47_keys"`
@@ -35,7 +35,7 @@ func TestBCP47KeysInvalidMap(t *testing.T) {
3535
}
3636

3737
func TestIsHTTPURLValid(t *testing.T) {
38-
v := New()
38+
v := New(DefaultIPACodes())
3939

4040
type S struct {
4141
U *testURL `validate:"omitnil,url_http_url"`
@@ -49,7 +49,7 @@ func TestIsHTTPURLValid(t *testing.T) {
4949
}
5050

5151
func TestIsHTTPURLInvalid(t *testing.T) {
52-
v := New()
52+
v := New(DefaultIPACodes())
5353

5454
type S struct {
5555
U *testURL `validate:"omitnil,url_http_url"`
@@ -63,7 +63,7 @@ func TestIsHTTPURLInvalid(t *testing.T) {
6363
}
6464

6565
func TestIsURLValid(t *testing.T) {
66-
v := New()
66+
v := New(DefaultIPACodes())
6767

6868
type S struct {
6969
U *testURL `validate:"omitnil,url_url"`
@@ -77,7 +77,7 @@ func TestIsURLValid(t *testing.T) {
7777
}
7878

7979
func TestIsURLInvalid(t *testing.T) {
80-
v := New()
80+
v := New(DefaultIPACodes())
8181

8282
type S struct {
8383
U *testURL `validate:"omitnil,url_url"`
@@ -139,7 +139,7 @@ func TestIsHTTPURLPanicNonStringer(t *testing.T) {
139139
}
140140
}()
141141

142-
v := New()
142+
v := New(DefaultIPACodes())
143143

144144
type S struct {
145145
U int `validate:"url_http_url"`
@@ -155,7 +155,7 @@ func TestIsURLPanicNonStringer(t *testing.T) {
155155
}
156156
}()
157157

158-
v := New()
158+
v := New(DefaultIPACodes())
159159

160160
type S struct {
161161
U int `validate:"url_url"`
@@ -171,7 +171,7 @@ func TestBCP47KeysPanicNonMap(t *testing.T) {
171171
}
172172
}()
173173

174-
v := New()
174+
v := New(DefaultIPACodes())
175175

176176
type S struct {
177177
M int `validate:"bcp47_keys"`

0 commit comments

Comments
 (0)